kylin git commit: minor, replace HBaseResourceStore.getByScan() with a simpler Get request

2015-12-30 Thread liyang 
Repository: kylin
Updated Branches:
  refs/heads/2.x-staging 95d952da5 -> 9e25b2374


minor, replace HBaseResourceStore.getByScan() with a simpler Get request


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/9e25b237
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/9e25b237
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/9e25b237

Branch: refs/heads/2.x-staging
Commit: 9e25b237415fed2f94f4c5518285dd71931cc84c
Parents: 95d952d
Author: Li, Yang 
Authored: Thu Dec 31 11:46:34 2015 +0800
Committer: Li, Yang 
Committed: Thu Dec 31 11:47:27 2015 +0800

--
 .../kylin/storage/hbase/HBaseResourceStore.java | 28 
 1 file changed, 11 insertions(+), 17 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/9e25b237/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
index f1659c3..10ef6c0 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
@@ -34,6 +34,7 @@ import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.client.Delete;
+import org.apache.hadoop.hbase.client.Get;
 import org.apache.hadoop.hbase.client.HConnection;
 import org.apache.hadoop.hbase.client.HTableInterface;
 import org.apache.hadoop.hbase.client.Put;
@@ -147,7 +148,7 @@ public class HBaseResourceStore extends ResourceStore {
 
 @Override
 protected boolean existsImpl(String resPath) throws IOException {
-Result r = getByScan(resPath, false, false);
+Result r = getFromHTable(resPath, false, false);
 return r != null;
 }
 
@@ -250,7 +251,7 @@ public class HBaseResourceStore extends ResourceStore {
 
 @Override
 protected RawResource getResourceImpl(String resPath) throws IOException {
-Result r = getByScan(resPath, true, true);
+Result r = getFromHTable(resPath, true, true);
 if (r == null)
 return null;
 else
@@ -259,7 +260,7 @@ public class HBaseResourceStore extends ResourceStore {
 
 @Override
 protected long getResourceTimestampImpl(String resPath) throws IOException 
{
-return getTimestamp(getByScan(resPath, false, true));
+return getTimestamp(getFromHTable(resPath, false, true));
 }
 
 @Override
@@ -320,30 +321,23 @@ public class HBaseResourceStore extends ResourceStore {
 return getAllInOneTableName() + "(key='" + resPath + "')@" + 
kylinConfig.getMetadataUrl();
 }
 
-private Result getByScan(String path, boolean fetchContent, boolean 
fetchTimestamp) throws IOException {
-byte[] startRow = Bytes.toBytes(path);
-byte[] endRow = plusZero(startRow);
+private Result getFromHTable(String path, boolean fetchContent, boolean 
fetchTimestamp) throws IOException {
+byte[] rowkey = Bytes.toBytes(path);
 
-Scan scan = new Scan(startRow, endRow);
-scan.setCaching(1);
-scan.setMaxResultSize(kylinConfig.getHBaseScanMaxResultSize());
+Get get = new Get(rowkey);
 
 if (!fetchContent && !fetchTimestamp) {
-scan.setFilter(new KeyOnlyFilter());
+get.setCheckExistenceOnly(true);
 } else {
 if (fetchContent)
-scan.addColumn(B_FAMILY, B_COLUMN);
+get.addColumn(B_FAMILY, B_COLUMN);
 if (fetchTimestamp)
-scan.addColumn(B_FAMILY, B_COLUMN_TS);
+get.addColumn(B_FAMILY, B_COLUMN_TS);
 }
 
 HTableInterface table = 
getConnection().getTable(getAllInOneTableName());
 try {
-ResultScanner scanner = table.getScanner(scan);
-Result result = null;
-for (Result r : scanner) {
-result = r;
-}
+Result result = table.get(get);
 return result == null || result.isEmpty() ? null : result;
 } finally {
 IOUtils.closeQuietly(table);

kylin git commit: minor, correct typo in error message

2015-12-30 Thread liyang 
Repository: kylin
Updated Branches:
  refs/heads/2.0-rc 33f48f6a7 -> 71c8033c4


minor, correct typo in error message


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/71c8033c
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/71c8033c
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/71c8033c

Branch: refs/heads/2.0-rc
Commit: 71c8033c4eb6c0f098ca1592c7123f5ed47ad121
Parents: 33f48f6
Author: Li, Yang 
Authored: Thu Dec 31 14:37:16 2015 +0800
Committer: Li, Yang 
Committed: Thu Dec 31 14:37:16 2015 +0800

--
 core-cube/.settings/org.eclipse.core.resources.prefs   | 1 -
 .../kylin/metadata/measure/serializer/DataTypeSerializer.java  | 2 +-
 2 files changed, 1 insertion(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/71c8033c/core-cube/.settings/org.eclipse.core.resources.prefs
--
diff --git a/core-cube/.settings/org.eclipse.core.resources.prefs 
b/core-cube/.settings/org.eclipse.core.resources.prefs
index 04cfa2c..365bbd6 100644
--- a/core-cube/.settings/org.eclipse.core.resources.prefs
+++ b/core-cube/.settings/org.eclipse.core.resources.prefs
@@ -2,5 +2,4 @@ eclipse.preferences.version=1
 encoding//src/main/java=UTF-8
 encoding//src/main/resources=UTF-8
 encoding//src/test/java=UTF-8
-encoding//src/test/resources=UTF-8
 encoding/=UTF-8

http://git-wip-us.apache.org/repos/asf/kylin/blob/71c8033c/core-metadata/src/main/java/org/apache/kylin/metadata/measure/serializer/DataTypeSerializer.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/measure/serializer/DataTypeSerializer.java
 
b/core-metadata/src/main/java/org/apache/kylin/metadata/measure/serializer/DataTypeSerializer.java
index d542098..bceae52 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/metadata/measure/serializer/DataTypeSerializer.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/metadata/measure/serializer/DataTypeSerializer.java
@@ -71,7 +71,7 @@ abstract public class DataTypeSerializer implements 
BytesSerializer {
 
 Class clz = implementations.get(type.getName());
 if (clz == null)
-throw new RuntimeException("No MeasureSerializer for type " + 
type);
+throw new RuntimeException("No DataTypeSerializer for type " + 
type);
 
 try {
 return (DataTypeSerializer) 
clz.getConstructor(DataType.class).newInstance(type);

kylin git commit: KYLIN-976 Minor document enrichment.

2015-12-30 Thread liyang 
Repository: kylin
Updated Branches:
  refs/heads/2.x-staging a95cef67d -> 95d952da5


KYLIN-976 Minor document enrichment.


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/95d952da
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/95d952da
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/95d952da

Branch: refs/heads/2.x-staging
Commit: 95d952da5b9d8dbb60005235c5c2b42c13a29c78
Parents: a95cef6
Author: Li, Yang 
Authored: Thu Dec 31 10:53:47 2015 +0800
Committer: Li, Yang 
Committed: Thu Dec 31 10:54:23 2015 +0800

--
 .../kylin/metadata/datatype/DataTypeSerializer.java | 12 +++-
 1 file changed, 7 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/95d952da/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DataTypeSerializer.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DataTypeSerializer.java
 
b/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DataTypeSerializer.java
index a66776c..df5513c 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DataTypeSerializer.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DataTypeSerializer.java
@@ -66,21 +66,23 @@ abstract public class DataTypeSerializer implements 
BytesSerializer {
 }
 }
 
-/** peek into buffer and return the length of serialization */
+/** Peek into buffer and return the length of serialization which is 
previously written by this.serialize().
+ *  The current position of input buffer is guaranteed to be at the 
beginning of the serialization.
+ *  The implementation must not alter the buffer position by its return. */
 abstract public int peekLength(ByteBuffer in);
 
-/** return the max number of bytes to the longest serialization */
+/** Return the max number of bytes to the longest possible serialization */
 abstract public int maxLength();
 
-/** get an estimate of size in bytes of the serialized data */
+/** Get an estimate of size in bytes of the serialized data */
 abstract public int getStorageBytesEstimate();
 
-/** an optional convenient method that converts a string to this data type 
(for dimensions) */
+/** An optional convenient method that converts a string to this data type 
(for dimensions) */
 public T valueOf(String str) {
 throw new UnsupportedOperationException();
 }
 
-/** convert from obj to string */
+/** Convert from obj to string */
 public String toString(T value) {
 if (value == null)
 return "NULL";

kylin git commit: KYLIN-1266 Drop some unused dependencies

2015-12-30 Thread liyang 
Repository: kylin
Updated Branches:
  refs/heads/2.0-rc 2b222043d -> 6d7d2d71a


KYLIN-1266 Drop some unused dependencies


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/6d7d2d71
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/6d7d2d71
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/6d7d2d71

Branch: refs/heads/2.0-rc
Commit: 6d7d2d71ac749d79fceb7158b7e714843fab5263
Parents: 2b22204
Author: Li, Yang 
Authored: Wed Dec 30 16:36:43 2015 +0800
Committer: Li, Yang 
Committed: Wed Dec 30 16:37:29 2015 +0800

--
 assembly/pom.xml|  24 ++-
 core-common/pom.xml |  18 +-
 .../kylin/common/util/CollectionUtils.java  |  29 +++
 .../org/apache/kylin/common/util/Primes.java| 184 +++
 .../org/apache/kylin/common/util/BasicTest.java |   4 +-
 .../cube/upgrade/v2/CubeMetadataUpgradeV2.java  |   2 +-
 core-storage/pom.xml|   2 -
 cube/pom.xml|   4 -
 metadata/pom.xml|   4 -
 pom.xml |  12 --
 server/pom.xml  |  10 +-
 .../rest/security/MockAclHBaseStorage.java  |   2 +-
 .../rest/security/RealAclHBaseStorage.java  |   2 +-
 .../apache/kylin/rest/service/QueryService.java |   4 +-
 source-kafka/pom.xml|   1 +
 storage-hbase/pom.xml   |   5 -
 .../storage/hbase/steps/CreateHTableJob.java|   2 +-
 17 files changed, 243 insertions(+), 66 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/6d7d2d71/assembly/pom.xml
--
diff --git a/assembly/pom.xml b/assembly/pom.xml
index df4a07f..cd7fc2b 100644
--- a/assembly/pom.xml
+++ b/assembly/pom.xml
@@ -84,11 +84,6 @@
 
 
 org.apache.hadoop
-hadoop-mapreduce-client-core
-provided
-
-
-org.apache.hadoop
 hadoop-minicluster
 test
 
@@ -110,11 +105,6 @@
 
 
 org.apache.hbase
-hbase-client
-provided
-
-
-org.apache.hbase
 hbase-server
 provided
 
@@ -165,6 +155,13 @@
 
 
 
+
+org.apache.kafka
+kafka_2.10
+${kafka.version}
+
+
+
 
 
 
@@ -184,6 +181,13 @@
 false
 
true
 job
+
+   
+   io.netty:*
+   
org.apache.zookeeper:*
+   net.sf.ehcache:*
+   
+
 
 
 *:*

http://git-wip-us.apache.org/repos/asf/kylin/blob/6d7d2d71/core-common/pom.xml
--
diff --git a/core-common/pom.xml b/core-common/pom.xml
index d02ddd3..72d8808 100644
--- a/core-common/pom.xml
+++ b/core-common/pom.xml
@@ -49,18 +49,10 @@
 commons-lang3
 
 
-org.apache.commons
-commons-math3
-
-
 commons-io
 commons-io
 
 
-commons-configuration
-commons-configuration
-
-
 commons-cli
 commons-cli
 
@@ -69,11 +61,6 @@
 commons-email
 
 
-org.apache.commons
-commons-collections4
-${commons-collections4.version}
-
-
 commons-httpclient
 commons-httpclient
 
@@ -114,6 +101,11 @@
 
 
 
+org.apache.commons
+commons-math3
+test
+
+
 junit
 junit
 test

http://git-wip-us.apache.org/repos/asf/kylin/blob/6d7d2d71/core-common/src/main/java/org/apache/kylin/common/util/CollectionUtils.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/util/CollectionUtils.java 
b/core-common/src/main/java/org/apache/kylin/common/util/CollectionUtils.java
new file mode 100644
index 000..c4b7d1d
--- /dev/null
+++ 
b/core-common/src/main/java/org/apache/kylin/common/util/CollectionUtils.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software

[2/2] kylin git commit: KYLIN-1266 Add back commons-collections, but a shared version not collections4

2015-12-30 Thread liyang 
KYLIN-1266 Add back commons-collections, but a shared version not collections4


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/8d76b760
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/8d76b760
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/8d76b760

Branch: refs/heads/2.x-staging
Commit: 8d76b7604f113f5c77669fb222eaa6e958f62b98
Parents: 6e78e4e
Author: Li, Yang 
Authored: Wed Dec 30 16:55:00 2015 +0800
Committer: Li, Yang 
Committed: Wed Dec 30 16:55:00 2015 +0800

--
 core-common/pom.xml |  4 +++
 .../kylin/common/util/CollectionUtils.java  | 29 
 .../validation/rule/AggregationGroupRule.java   |  2 +-
 pom.xml |  6 
 4 files changed, 11 insertions(+), 30 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/8d76b760/core-common/pom.xml
--
diff --git a/core-common/pom.xml b/core-common/pom.xml
index 72d8808..07a7ee1 100644
--- a/core-common/pom.xml
+++ b/core-common/pom.xml
@@ -49,6 +49,10 @@
 commons-lang3
 
 
+commons-collections
+commons-collections
+
+
 commons-io
 commons-io
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/8d76b760/core-common/src/main/java/org/apache/kylin/common/util/CollectionUtils.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/util/CollectionUtils.java 
b/core-common/src/main/java/org/apache/kylin/common/util/CollectionUtils.java
deleted file mode 100644
index c4b7d1d..000
--- 
a/core-common/src/main/java/org/apache/kylin/common/util/CollectionUtils.java
+++ /dev/null
@@ -1,29 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * 
- * http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.common.util;
-
-import java.util.Collection;
-
-public class CollectionUtils {
-
-public static boolean isEmpty(final Collection coll) {
-return coll == null || coll.isEmpty();
-}
-
-}

http://git-wip-us.apache.org/repos/asf/kylin/blob/8d76b760/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/AggregationGroupRule.java
--
diff --git 
a/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/AggregationGroupRule.java
 
b/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/AggregationGroupRule.java
index 40555c2..88a 100644
--- 
a/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/AggregationGroupRule.java
+++ 
b/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/AggregationGroupRule.java
@@ -22,7 +22,7 @@ import java.util.Arrays;
 import java.util.Set;
 import java.util.TreeSet;
 
-import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.collections.CollectionUtils;
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.cube.model.AggregationGroup;
 import org.apache.kylin.cube.model.CubeDesc;

http://git-wip-us.apache.org/repos/asf/kylin/blob/8d76b760/pom.xml
--
diff --git a/pom.xml b/pom.xml
index 9311691..74bac02 100644
--- a/pom.xml
+++ b/pom.xml
@@ -64,6 +64,7 @@
 1.2
 2.6
 3.1
+3.2.1
 2.4
 1.0.15
 3.1
@@ -329,6 +330,11 @@
 ${commons-lang3.version}
 
 
+commons-collections
+commons-collections
+${commons-collections.version}
+
+
 org.apache.commons
 commons-math3
 ${commons-math3.version}

kylin git commit: KYLIN-1266 Add back commons-collections, but a shared version not collections4

2015-12-30 Thread liyang 
Repository: kylin
Updated Branches:
  refs/heads/2.0-rc 6d7d2d71a -> e0748f5b4


KYLIN-1266 Add back commons-collections, but a shared version not collections4


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/e0748f5b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/e0748f5b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/e0748f5b

Branch: refs/heads/2.0-rc
Commit: e0748f5b4cf97d671250c967666a02bb9d03a700
Parents: 6d7d2d7
Author: Li, Yang 
Authored: Wed Dec 30 16:55:00 2015 +0800
Committer: Li, Yang 
Committed: Wed Dec 30 17:00:07 2015 +0800

--
 core-common/pom.xml |  4 +++
 .../kylin/common/util/CollectionUtils.java  | 29 
 .../cube/upgrade/v2/CubeMetadataUpgradeV2.java  |  2 +-
 pom.xml |  6 
 4 files changed, 11 insertions(+), 30 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/e0748f5b/core-common/pom.xml
--
diff --git a/core-common/pom.xml b/core-common/pom.xml
index 72d8808..07a7ee1 100644
--- a/core-common/pom.xml
+++ b/core-common/pom.xml
@@ -49,6 +49,10 @@
 commons-lang3
 
 
+commons-collections
+commons-collections
+
+
 commons-io
 commons-io
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/e0748f5b/core-common/src/main/java/org/apache/kylin/common/util/CollectionUtils.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/util/CollectionUtils.java 
b/core-common/src/main/java/org/apache/kylin/common/util/CollectionUtils.java
deleted file mode 100644
index c4b7d1d..000
--- 
a/core-common/src/main/java/org/apache/kylin/common/util/CollectionUtils.java
+++ /dev/null
@@ -1,29 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * 
- * http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.common.util;
-
-import java.util.Collection;
-
-public class CollectionUtils {
-
-public static boolean isEmpty(final Collection coll) {
-return coll == null || coll.isEmpty();
-}
-
-}

http://git-wip-us.apache.org/repos/asf/kylin/blob/e0748f5b/core-cube/src/main/java/org/apache/kylin/cube/upgrade/v2/CubeMetadataUpgradeV2.java
--
diff --git 
a/core-cube/src/main/java/org/apache/kylin/cube/upgrade/v2/CubeMetadataUpgradeV2.java
 
b/core-cube/src/main/java/org/apache/kylin/cube/upgrade/v2/CubeMetadataUpgradeV2.java
index 4b823e6..f11d95c 100644
--- 
a/core-cube/src/main/java/org/apache/kylin/cube/upgrade/v2/CubeMetadataUpgradeV2.java
+++ 
b/core-cube/src/main/java/org/apache/kylin/cube/upgrade/v2/CubeMetadataUpgradeV2.java
@@ -23,12 +23,12 @@ import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 
+import org.apache.commons.collections.CollectionUtils;
 import org.apache.commons.lang.ArrayUtils;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.persistence.ResourceStore;
-import org.apache.kylin.common.util.CollectionUtils;
 import org.apache.kylin.cube.CubeDescManager;
 import org.apache.kylin.cube.CubeInstance;
 import org.apache.kylin.cube.CubeManager;

http://git-wip-us.apache.org/repos/asf/kylin/blob/e0748f5b/pom.xml
--
diff --git a/pom.xml b/pom.xml
index c936774..7a709a6 100644
--- a/pom.xml
+++ b/pom.xml
@@ -64,6 +64,7 @@
 1.2
 2.6
 3.1
+3.2.1
 2.4
 1.0.15
 3.1
@@ -329,6 +330,11 @@
 ${commons-lang3.version}
 
 
+commons-collections
+commons-collections
+${commons-collections.version}
+
+

kylin git commit: KYLIN-1270 improve TimedJsonStreamParser to support week_start, month_start, quarter_start, year_start

2015-12-30 Thread mahongbin 
Repository: kylin
Updated Branches:
  refs/heads/2.0-rc e0748f5b4 -> 5c83a14a4


KYLIN-1270 improve TimedJsonStreamParser to support week_start, month_start, 
quarter_start, year_start


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/5c83a14a
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/5c83a14a
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/5c83a14a

Branch: refs/heads/2.0-rc
Commit: 5c83a14a4bea5e1496d49ed362593aaaf84922ed
Parents: e0748f5
Author: honma 
Authored: Wed Dec 30 21:33:37 2015 +0800
Committer: honma 
Committed: Wed Dec 30 21:35:43 2015 +0800

--
 .../org/apache/kylin/common/util/TimeUtil.java  | 44 +++--
 .../apache/kylin/common/util/TimeUtilTest.java  | 21 +++-
 .../source/kafka/TimedJsonStreamParser.java | 51 ++--
 3 files changed, 94 insertions(+), 22 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/5c83a14a/core-common/src/main/java/org/apache/kylin/common/util/TimeUtil.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/util/TimeUtil.java 
b/core-common/src/main/java/org/apache/kylin/common/util/TimeUtil.java
index c79e88b..17868a6 100644
--- a/core-common/src/main/java/org/apache/kylin/common/util/TimeUtil.java
+++ b/core-common/src/main/java/org/apache/kylin/common/util/TimeUtil.java
@@ -1,12 +1,13 @@
 package org.apache.kylin.common.util;
 
+import java.util.Calendar;
+import java.util.TimeZone;
+
 /**
  */
 public class TimeUtil {
-public enum NormalizedTimeUnit {
-MINUTE, HOUR, DAY
-}
 
+private static TimeZone gmt = TimeZone.getTimeZone("GMT");
 private static long ONE_MINUTE_TS = 60 * 1000;
 private static long ONE_HOUR_TS = 60 * ONE_MINUTE_TS;
 private static long ONE_DAY_TS = 24 * ONE_HOUR_TS;
@@ -23,7 +24,40 @@ public class TimeUtil {
 return ts / ONE_DAY_TS * ONE_DAY_TS;
 }
 
-public static long getNextPeriodStart(long ts, long period) {
-return ((ts + period - 1) / period) * period;
+public static long getWeekStart(long ts) {
+Calendar calendar = Calendar.getInstance(gmt);
+calendar.setTimeInMillis(getDayStart(ts));
+calendar.add(Calendar.DAY_OF_WEEK, calendar.getFirstDayOfWeek() - 
calendar.get(Calendar.DAY_OF_WEEK));
+return calendar.getTimeInMillis();
 }
+
+public static long getMonthStart(long ts) {
+Calendar calendar = Calendar.getInstance(gmt);
+calendar.setTimeInMillis(ts);
+int year = calendar.get(Calendar.YEAR);
+int month = calendar.get(Calendar.MONTH);
+calendar.clear();
+calendar.set(year, month, 1);
+return calendar.getTimeInMillis();
+}
+
+public static long getQuarterStart(long ts) {
+Calendar calendar = Calendar.getInstance(gmt);
+calendar.setTimeInMillis(ts);
+int year = calendar.get(Calendar.YEAR);
+int month = calendar.get(Calendar.MONTH);
+calendar.clear();
+calendar.set(year, month / 3 * 3, 1);
+return calendar.getTimeInMillis();
+}
+
+public static long getYearStart(long ts) {
+Calendar calendar = Calendar.getInstance(gmt);
+calendar.setTimeInMillis(ts);
+int year = calendar.get(Calendar.YEAR);
+calendar.clear();
+calendar.set(year, 0, 1);
+return calendar.getTimeInMillis();
+}
+
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/5c83a14a/core-common/src/test/java/org/apache/kylin/common/util/TimeUtilTest.java
--
diff --git 
a/core-common/src/test/java/org/apache/kylin/common/util/TimeUtilTest.java 
b/core-common/src/test/java/org/apache/kylin/common/util/TimeUtilTest.java
index d81d49a..3fdf6aa 100644
--- a/core-common/src/test/java/org/apache/kylin/common/util/TimeUtilTest.java
+++ b/core-common/src/test/java/org/apache/kylin/common/util/TimeUtilTest.java
@@ -3,14 +3,19 @@ package org.apache.kylin.common.util;
 import java.text.ParseException;
 import java.text.SimpleDateFormat;
 import java.util.Calendar;
+import java.util.TimeZone;
 
-import org.apache.kylin.common.util.TimeUtil.NormalizedTimeUnit;
 import org.junit.Assert;
 import org.junit.Test;
 
 /**
  */
 public class TimeUtilTest {
+
+public enum NormalizedTimeUnit {
+MINUTE, HOUR, DAY, WEEK, MONTH, QUARTER, YEAR
+}
+
 public static long normalizeTime(long timeMillis, NormalizedTimeUnit unit) 
{
 Calendar a = Calendar.getInstance();
 Calendar b = Calendar.getInstance();
@@ -28,6 +33,7 @@ public class TimeUtilTest {
 @Test
 public void basicTest() throws ParseException {
 java.text.DateFormat dateFormat