[1/2] kylin git commit: KYLIN-2722 Introduce a new measure for dropwizard metrics framework, called active reservoir, for actively pushing metrics to reporters [Forced Update!]

2017-10-19 Thread liyang 
Repository: kylin
Updated Branches:
  refs/heads/master 33b1e281e -> 2b32aa4ca (forced update)


http://git-wip-us.apache.org/repos/asf/kylin/blob/2b32aa4c/metrics-reporter-hive/src/main/java/org/apache/kylin/metrics/lib/impl/hive/HiveReservoirReporter.java
--
diff --git 
a/metrics-reporter-hive/src/main/java/org/apache/kylin/metrics/lib/impl/hive/HiveReservoirReporter.java
 
b/metrics-reporter-hive/src/main/java/org/apache/kylin/metrics/lib/impl/hive/HiveReservoirReporter.java
new file mode 100644
index 000..5af2bf9
--- /dev/null
+++ 
b/metrics-reporter-hive/src/main/java/org/apache/kylin/metrics/lib/impl/hive/HiveReservoirReporter.java
@@ -0,0 +1,139 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.metrics.lib.impl.hive;
+
+import java.util.List;
+import java.util.Properties;
+
+import org.apache.kylin.metrics.lib.ActiveReservoir;
+import org.apache.kylin.metrics.lib.ActiveReservoirListener;
+import org.apache.kylin.metrics.lib.ActiveReservoirReporter;
+import org.apache.kylin.metrics.lib.Record;
+import org.apache.kylin.metrics.lib.impl.ReporterBuilder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * A reporter which listens for new records and publishes them to hive.
+ */
+public class HiveReservoirReporter extends ActiveReservoirReporter {
+
+public static final String HIVE_REPORTER_SUFFIX = "HIVE";
+public static final HiveSink sink = new HiveSink();
+protected static final Logger logger = 
LoggerFactory.getLogger(HiveReservoirReporter.class);
+private final ActiveReservoir activeReservoir;
+private final HiveReservoirListener listener;
+
+public HiveReservoirReporter(ActiveReservoir activeReservoir, Properties 
props) throws Exception {
+this.activeReservoir = activeReservoir;
+this.listener = new HiveReservoirListener(props);
+}
+
+/**
+ * Returns a new {@link Builder} for {@link HiveReservoirReporter}.
+ *
+ * @param activeReservoir the registry to report
+ * @return a {@link Builder} instance for a {@link HiveReservoirReporter}
+ */
+public static Builder forRegistry(ActiveReservoir activeReservoir) {
+return new Builder(activeReservoir);
+}
+
+public static String getTableFromSubject(String subject) {
+return sink.getTableFromSubject(subject);
+}
+
+/**
+ * Starts the reporter.
+ */
+public void start() {
+activeReservoir.addListener(listener);
+}
+
+/**
+ * Stops the reporter.
+ */
+public void stop() {
+activeReservoir.removeListener(listener);
+}
+
+/**
+ * Stops the reporter.
+ */
+@Override
+public void close() {
+stop();
+}
+
+/**
+ * A builder for {@link HiveReservoirReporter} instances.
+ */
+public static class Builder extends ReporterBuilder {
+
+private Builder(ActiveReservoir activeReservoir) {
+super(activeReservoir);
+}
+
+private void setFixedProperties() {
+}
+
+/**
+ * Builds a {@link HiveReservoirReporter} with the given properties.
+ *
+ * @return a {@link HiveReservoirReporter}
+ */
+public HiveReservoirReporter build() throws Exception {
+setFixedProperties();
+return new HiveReservoirReporter(registry, props);
+}
+}
+
+private class HiveReservoirListener implements ActiveReservoirListener {
+
+HiveProducer producer;
+
+private HiveReservoirListener(Properties props) throws Exception {
+producer = new HiveProducer(props);
+}
+
+public boolean onRecordUpdate(final List records) {
+try {
+producer.send(records);
+} catch (Exception e) {
+logger.error(e.getMessage(), e);
+return false;
+}
+return true;
+}
+
+public boolean onRecordUpdate(final Record record) {
+try {
+producer.send(record);
+} catch (Exception e) {
+logger.error(e.getMessage(), e);
+

[2/2] kylin git commit: KYLIN-2722 Introduce a new measure for dropwizard metrics framework, called active reservoir, for actively pushing metrics to reporters

2017-10-19 Thread liyang 
KYLIN-2722 Introduce a new measure for dropwizard metrics framework, called 
active reservoir, for actively pushing metrics to reporters

This closes #77

Signed-off-by: Li Yang 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/2b32aa4c
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/2b32aa4c
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/2b32aa4c

Branch: refs/heads/master
Commit: 2b32aa4ca91ccdb4a885c4c509ae173bea5658d5
Parents: b53e54f
Author: Zhong 
Authored: Tue Aug 8 22:50:54 2017 +0800
Committer: Li Yang 
Committed: Fri Oct 20 07:05:39 2017 +0800

--
 .../apache/kylin/common/KylinConfigBase.java|  10 +-
 core-metrics/pom.xml|  51 
 .../kylin/metrics/lib/ActiveReservoir.java  |  40 +++
 .../metrics/lib/ActiveReservoirFilter.java  |  44 +++
 .../metrics/lib/ActiveReservoirListener.java|  30 ++
 .../metrics/lib/ActiveReservoirReporter.java|  51 
 .../org/apache/kylin/metrics/lib/Record.java|  51 
 .../java/org/apache/kylin/metrics/lib/Sink.java |  23 ++
 .../lib/impl/AbstractActiveReservoir.java   |  68 +
 .../metrics/lib/impl/BaseScheduledReporter.java | 103 +++
 .../metrics/lib/impl/BlockingReservoir.java | 167 
 .../metrics/lib/impl/InstantReservoir.java  |  76 ++
 .../kylin/metrics/lib/impl/MetricsSystem.java   | 164 +++
 .../kylin/metrics/lib/impl/RecordEvent.java | 272 +++
 .../metrics/lib/impl/RecordEventTimeDetail.java |  77 ++
 .../metrics/lib/impl/RecordEventWrapper.java|  61 +
 .../kylin/metrics/lib/impl/ReporterBuilder.java |  48 
 .../kylin/metrics/lib/impl/StubReservoir.java   |  54 
 .../metrics/lib/impl/StubReservoirReporter.java |  51 
 .../apache/kylin/metrics/lib/impl/StubSink.java |  30 ++
 .../metrics/lib/impl/TimePropertyEnum.java  |  49 
 metrics-reporter-hive/pom.xml   |  53 
 .../metrics/lib/impl/hive/HiveProducer.java | 201 ++
 .../lib/impl/hive/HiveProducerRecord.java   | 196 +
 .../lib/impl/hive/HiveReservoirReporter.java| 139 ++
 .../kylin/metrics/lib/impl/hive/HiveSink.java   |  30 ++
 metrics-reporter-kafka/pom.xml  |  46 
 .../kafka/KafkaActiveReserviorListener.java | 115 
 .../lib/impl/kafka/KafkaReservoirReporter.java  | 139 ++
 .../kylin/metrics/lib/impl/kafka/KafkaSink.java |  29 ++
 pom.xml |  25 ++
 31 files changed, 2490 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/2b32aa4c/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 34d8b7c..f3cf6c0 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -1295,9 +1295,9 @@ abstract public class KylinConfigBase implements 
Serializable {
 return 
Integer.valueOf(this.getOptional("kylin.restclient.connection.max-total", 
"200"));
 }
 
-/**
- * metric
- */
+// 

+// Metrics
+// 

 public String getCoadhaleMetricsReportClassesNames() {
 return getOptional("kylin.metrics.reporter-classes",
 
"org.apache.kylin.common.metrics.metrics2.JsonFileMetricsReporter,org.apache.kylin.common.metrics.metrics2.JmxMetricsReporter");
@@ -1315,4 +1315,8 @@ abstract public class KylinConfigBase implements 
Serializable {
 return getOptional("kylin.metrics.perflogger-class", 
"org.apache.kylin.common.metrics.perflog.PerfLogger");
 }
 
+public String getMetricsActiveReservoirDefaultClass() {
+return getOptional("kylin.metrics.active-reservoir-default-class",
+"org.apache.kylin.metrics.lib.impl.StubReservoir");
+}
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/2b32aa4c/core-metrics/pom.xml
--
diff --git a/core-metrics/pom.xml b/core-metrics/pom.xml
new file mode 100644
index 000..e436c97
--- /dev/null
+++ b/core-metrics/pom.xml
@@ -0,0 +1,51 @@
+
+
+
+http://www.w3.org/2001/XMLSchema-instance; 
xmlns="http://maven.apache.org/POM/4.0.0;
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0

[1/2] kylin git commit: KYLIN-2722 Introduce a new measure for dropwizard metrics framework, called active reservoir, for actively pushing metrics to reporters

2017-10-19 Thread liyang 
Repository: kylin
Updated Branches:
  refs/heads/master b53e54f31 -> 33b1e281e


http://git-wip-us.apache.org/repos/asf/kylin/blob/33b1e281/metrics-reporter-hive/src/main/java/org/apache/kylin/metrics/lib/impl/hive/HiveReservoirReporter.java
--
diff --git 
a/metrics-reporter-hive/src/main/java/org/apache/kylin/metrics/lib/impl/hive/HiveReservoirReporter.java
 
b/metrics-reporter-hive/src/main/java/org/apache/kylin/metrics/lib/impl/hive/HiveReservoirReporter.java
new file mode 100644
index 000..5af2bf9
--- /dev/null
+++ 
b/metrics-reporter-hive/src/main/java/org/apache/kylin/metrics/lib/impl/hive/HiveReservoirReporter.java
@@ -0,0 +1,139 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.metrics.lib.impl.hive;
+
+import java.util.List;
+import java.util.Properties;
+
+import org.apache.kylin.metrics.lib.ActiveReservoir;
+import org.apache.kylin.metrics.lib.ActiveReservoirListener;
+import org.apache.kylin.metrics.lib.ActiveReservoirReporter;
+import org.apache.kylin.metrics.lib.Record;
+import org.apache.kylin.metrics.lib.impl.ReporterBuilder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * A reporter which listens for new records and publishes them to hive.
+ */
+public class HiveReservoirReporter extends ActiveReservoirReporter {
+
+public static final String HIVE_REPORTER_SUFFIX = "HIVE";
+public static final HiveSink sink = new HiveSink();
+protected static final Logger logger = 
LoggerFactory.getLogger(HiveReservoirReporter.class);
+private final ActiveReservoir activeReservoir;
+private final HiveReservoirListener listener;
+
+public HiveReservoirReporter(ActiveReservoir activeReservoir, Properties 
props) throws Exception {
+this.activeReservoir = activeReservoir;
+this.listener = new HiveReservoirListener(props);
+}
+
+/**
+ * Returns a new {@link Builder} for {@link HiveReservoirReporter}.
+ *
+ * @param activeReservoir the registry to report
+ * @return a {@link Builder} instance for a {@link HiveReservoirReporter}
+ */
+public static Builder forRegistry(ActiveReservoir activeReservoir) {
+return new Builder(activeReservoir);
+}
+
+public static String getTableFromSubject(String subject) {
+return sink.getTableFromSubject(subject);
+}
+
+/**
+ * Starts the reporter.
+ */
+public void start() {
+activeReservoir.addListener(listener);
+}
+
+/**
+ * Stops the reporter.
+ */
+public void stop() {
+activeReservoir.removeListener(listener);
+}
+
+/**
+ * Stops the reporter.
+ */
+@Override
+public void close() {
+stop();
+}
+
+/**
+ * A builder for {@link HiveReservoirReporter} instances.
+ */
+public static class Builder extends ReporterBuilder {
+
+private Builder(ActiveReservoir activeReservoir) {
+super(activeReservoir);
+}
+
+private void setFixedProperties() {
+}
+
+/**
+ * Builds a {@link HiveReservoirReporter} with the given properties.
+ *
+ * @return a {@link HiveReservoirReporter}
+ */
+public HiveReservoirReporter build() throws Exception {
+setFixedProperties();
+return new HiveReservoirReporter(registry, props);
+}
+}
+
+private class HiveReservoirListener implements ActiveReservoirListener {
+
+HiveProducer producer;
+
+private HiveReservoirListener(Properties props) throws Exception {
+producer = new HiveProducer(props);
+}
+
+public boolean onRecordUpdate(final List records) {
+try {
+producer.send(records);
+} catch (Exception e) {
+logger.error(e.getMessage(), e);
+return false;
+}
+return true;
+}
+
+public boolean onRecordUpdate(final Record record) {
+try {
+producer.send(record);
+} catch (Exception e) {
+logger.error(e.getMessage(), e);
+return

[kylin] Git Push Summary

2017-10-19 Thread liyang 
Repository: kylin
Updated Branches:
  refs/heads/pr77 [deleted] 556298656

[2/2] kylin git commit: KYLIN-2722 Introduce a new measure for dropwizard metrics framework, called active reservoir, for actively pushing metrics to reporters

2017-10-19 Thread liyang 
KYLIN-2722 Introduce a new measure for dropwizard metrics framework, called 
active reservoir, for actively pushing metrics to reporters

Signed-off-by: Li Yang 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/33b1e281
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/33b1e281
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/33b1e281

Branch: refs/heads/master
Commit: 33b1e281e5193a4a4e48ea9c4f9fd8929e0ede34
Parents: b53e54f
Author: Zhong 
Authored: Tue Aug 8 22:50:54 2017 +0800
Committer: Li Yang 
Committed: Fri Oct 20 07:04:50 2017 +0800

--
 .../apache/kylin/common/KylinConfigBase.java|  10 +-
 core-metrics/pom.xml|  51 
 .../kylin/metrics/lib/ActiveReservoir.java  |  40 +++
 .../metrics/lib/ActiveReservoirFilter.java  |  44 +++
 .../metrics/lib/ActiveReservoirListener.java|  30 ++
 .../metrics/lib/ActiveReservoirReporter.java|  51 
 .../org/apache/kylin/metrics/lib/Record.java|  51 
 .../java/org/apache/kylin/metrics/lib/Sink.java |  23 ++
 .../lib/impl/AbstractActiveReservoir.java   |  68 +
 .../metrics/lib/impl/BaseScheduledReporter.java | 103 +++
 .../metrics/lib/impl/BlockingReservoir.java | 167 
 .../metrics/lib/impl/InstantReservoir.java  |  76 ++
 .../kylin/metrics/lib/impl/MetricsSystem.java   | 164 +++
 .../kylin/metrics/lib/impl/RecordEvent.java | 272 +++
 .../metrics/lib/impl/RecordEventTimeDetail.java |  77 ++
 .../metrics/lib/impl/RecordEventWrapper.java|  61 +
 .../kylin/metrics/lib/impl/ReporterBuilder.java |  48 
 .../kylin/metrics/lib/impl/StubReservoir.java   |  54 
 .../metrics/lib/impl/StubReservoirReporter.java |  51 
 .../apache/kylin/metrics/lib/impl/StubSink.java |  30 ++
 .../metrics/lib/impl/TimePropertyEnum.java  |  49 
 metrics-reporter-hive/pom.xml   |  53 
 .../metrics/lib/impl/hive/HiveProducer.java | 201 ++
 .../lib/impl/hive/HiveProducerRecord.java   | 196 +
 .../lib/impl/hive/HiveReservoirReporter.java| 139 ++
 .../kylin/metrics/lib/impl/hive/HiveSink.java   |  30 ++
 metrics-reporter-kafka/pom.xml  |  46 
 .../kafka/KafkaActiveReserviorListener.java | 115 
 .../lib/impl/kafka/KafkaReservoirReporter.java  | 139 ++
 .../kylin/metrics/lib/impl/kafka/KafkaSink.java |  29 ++
 pom.xml |  25 ++
 31 files changed, 2490 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/33b1e281/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 34d8b7c..f3cf6c0 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -1295,9 +1295,9 @@ abstract public class KylinConfigBase implements 
Serializable {
 return 
Integer.valueOf(this.getOptional("kylin.restclient.connection.max-total", 
"200"));
 }
 
-/**
- * metric
- */
+// 

+// Metrics
+// 

 public String getCoadhaleMetricsReportClassesNames() {
 return getOptional("kylin.metrics.reporter-classes",
 
"org.apache.kylin.common.metrics.metrics2.JsonFileMetricsReporter,org.apache.kylin.common.metrics.metrics2.JmxMetricsReporter");
@@ -1315,4 +1315,8 @@ abstract public class KylinConfigBase implements 
Serializable {
 return getOptional("kylin.metrics.perflogger-class", 
"org.apache.kylin.common.metrics.perflog.PerfLogger");
 }
 
+public String getMetricsActiveReservoirDefaultClass() {
+return getOptional("kylin.metrics.active-reservoir-default-class",
+"org.apache.kylin.metrics.lib.impl.StubReservoir");
+}
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/33b1e281/core-metrics/pom.xml
--
diff --git a/core-metrics/pom.xml b/core-metrics/pom.xml
new file mode 100644
index 000..e436c97
--- /dev/null
+++ b/core-metrics/pom.xml
@@ -0,0 +1,51 @@
+
+
+
+http://www.w3.org/2001/XMLSchema-instance; 
xmlns="http://maven.apache.org/POM/4.0.0;
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 
http://maven.apache.org/xsd/maven-4.0.0.xsd;>
+4.0.0
+
+

kylin git commit: minor, update doc link on README.md

2017-10-19 Thread lidong 
Repository: kylin
Updated Branches:
  refs/heads/master 37e03a43e -> b53e54f31


minor, update doc link on README.md


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b53e54f3
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b53e54f3
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b53e54f3

Branch: refs/heads/master
Commit: b53e54f31f88993e998f7b6656d7874ec5f62ead
Parents: 37e03a4
Author: lidongsjtu 
Authored: Thu Oct 19 23:33:53 2017 +0800
Committer: lidongsjtu 
Committed: Thu Oct 19 23:33:53 2017 +0800

--
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/b53e54f3/README.md
--
diff --git a/README.md b/README.md
index 06e5ba4..df8fd3a 100644
--- a/README.md
+++ b/README.md
@@ -12,7 +12,7 @@ For more details, see the website 
[http://kylin.apache.org](http://kylin.apache.
 
 Documentation
 =
-Please refer to 
[http://kylin.apache.org/docs16/](http://kylin.apache.org/docs16/).
+Please refer to 
[http://kylin.apache.org/docs21/](http://kylin.apache.org/docs21/).
 
 Get Help

kylin git commit: KYLIN-2949: The kylin does not get right authorities when the LDAP authentication service was used in RedHat OS.

2017-10-19 Thread lidong 
Repository: kylin
Updated Branches:
  refs/heads/2.2.x 53cad89b3 -> fb2580808


KYLIN-2949: The kylin does not get right authorities when the LDAP 
authentication service was used in RedHat OS.

Signed-off-by: lidongsjtu 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/fb258080
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/fb258080
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/fb258080

Branch: refs/heads/2.2.x
Commit: fb25808088eb1aa857c674b42f969fdcc0a35d7c
Parents: 53cad89
Author: peng.jianhua 
Authored: Thu Oct 19 18:40:23 2017 +0800
Committer: lidongsjtu 
Committed: Thu Oct 19 22:35:37 2017 +0800

--
 .../java/org/apache/kylin/rest/security/AuthoritiesPopulator.java   | 1 +
 1 file changed, 1 insertion(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/fb258080/server-base/src/main/java/org/apache/kylin/rest/security/AuthoritiesPopulator.java
--
diff --git 
a/server-base/src/main/java/org/apache/kylin/rest/security/AuthoritiesPopulator.java
 
b/server-base/src/main/java/org/apache/kylin/rest/security/AuthoritiesPopulator.java
index 592791c..2b9d8c9 100644
--- 
a/server-base/src/main/java/org/apache/kylin/rest/security/AuthoritiesPopulator.java
+++ 
b/server-base/src/main/java/org/apache/kylin/rest/security/AuthoritiesPopulator.java
@@ -63,6 +63,7 @@ public class AuthoritiesPopulator extends 
DefaultLdapAuthoritiesPopulator {
 
 @Override
 public Set getGroupMembershipRoles(String userDn, String 
username) {
+setGroupSearchFilter("(|(member={0})(memberUid={1}))");
 Set authorities = 
super.getGroupMembershipRoles(userDn, username);
 
 Set userAuthorities = new 
HashSet();

kylin git commit: KYLIN-2949: The kylin does not get right authorities when the LDAP authentication service was used in RedHat OS.

2017-10-19 Thread lidong 
Repository: kylin
Updated Branches:
  refs/heads/master 27ba8c755 -> 37e03a43e


KYLIN-2949: The kylin does not get right authorities when the LDAP 
authentication service was used in RedHat OS.

Signed-off-by: lidongsjtu 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/37e03a43
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/37e03a43
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/37e03a43

Branch: refs/heads/master
Commit: 37e03a43e8ce78fec9a6b18e9c7c4863e153d987
Parents: 27ba8c7
Author: peng.jianhua 
Authored: Thu Oct 19 18:40:23 2017 +0800
Committer: lidongsjtu 
Committed: Thu Oct 19 21:25:25 2017 +0800

--
 .../java/org/apache/kylin/rest/security/AuthoritiesPopulator.java   | 1 +
 1 file changed, 1 insertion(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/37e03a43/server-base/src/main/java/org/apache/kylin/rest/security/AuthoritiesPopulator.java
--
diff --git 
a/server-base/src/main/java/org/apache/kylin/rest/security/AuthoritiesPopulator.java
 
b/server-base/src/main/java/org/apache/kylin/rest/security/AuthoritiesPopulator.java
index 592791c..2b9d8c9 100644
--- 
a/server-base/src/main/java/org/apache/kylin/rest/security/AuthoritiesPopulator.java
+++ 
b/server-base/src/main/java/org/apache/kylin/rest/security/AuthoritiesPopulator.java
@@ -63,6 +63,7 @@ public class AuthoritiesPopulator extends 
DefaultLdapAuthoritiesPopulator {
 
 @Override
 public Set getGroupMembershipRoles(String userDn, String 
username) {
+setGroupSearchFilter("(|(member={0})(memberUid={1}))");
 Set authorities = 
super.getGroupMembershipRoles(userDn, username);
 
 Set userAuthorities = new 
HashSet();

[4/4] kylin git commit: KYLIN-2722 Introduce a new measure for dropwizard metrics framework, called active reservoir, for actively pushing metrics to reporters

2017-10-19 Thread liyang 
KYLIN-2722 Introduce a new measure for dropwizard metrics framework, called 
active reservoir, for actively pushing metrics to reporters

Signed-off-by: Li Yang 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/55629865
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/55629865
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/55629865

Branch: refs/heads/pr77
Commit: 556298656362b0932c30c93ad86e1823ebcdc4f5
Parents: 27ba8c7
Author: Zhong 
Authored: Tue Aug 8 22:50:54 2017 +0800
Committer: Li Yang 
Committed: Thu Oct 19 21:39:09 2017 +0800

--
 .../apache/kylin/common/KylinConfigBase.java|  10 +-
 core-metrics/pom.xml|  51 
 .../kylin/metrics/lib/ActiveReservoir.java  |  40 +++
 .../metrics/lib/ActiveReservoirFilter.java  |  44 +++
 .../metrics/lib/ActiveReservoirListener.java|  30 ++
 .../metrics/lib/ActiveReservoirReporter.java|  51 
 .../org/apache/kylin/metrics/lib/Record.java|  51 
 .../java/org/apache/kylin/metrics/lib/Sink.java |  23 ++
 .../lib/impl/AbstractActiveReservoir.java   |  68 +
 .../metrics/lib/impl/BaseScheduledReporter.java | 103 +++
 .../metrics/lib/impl/BlockingReservoir.java | 167 
 .../metrics/lib/impl/InstantReservoir.java  |  76 ++
 .../kylin/metrics/lib/impl/MetricsSystem.java   | 164 +++
 .../kylin/metrics/lib/impl/RecordEvent.java | 272 +++
 .../metrics/lib/impl/RecordEventTimeDetail.java |  77 ++
 .../metrics/lib/impl/RecordEventWrapper.java|  61 +
 .../kylin/metrics/lib/impl/ReporterBuilder.java |  48 
 .../kylin/metrics/lib/impl/StubReservoir.java   |  54 
 .../metrics/lib/impl/StubReservoirReporter.java |  51 
 .../apache/kylin/metrics/lib/impl/StubSink.java |  30 ++
 .../metrics/lib/impl/TimePropertyEnum.java  |  49 
 metrics-reporter-hive/pom.xml   |  53 
 .../metrics/lib/impl/hive/HiveProducer.java | 201 ++
 .../lib/impl/hive/HiveProducerRecord.java   | 196 +
 .../lib/impl/hive/HiveReservoirReporter.java| 139 ++
 .../kylin/metrics/lib/impl/hive/HiveSink.java   |  30 ++
 metrics-reporter-kafka/pom.xml  |  46 
 .../kafka/KafkaActiveReserviorListener.java | 115 
 .../lib/impl/kafka/KafkaReservoirReporter.java  | 139 ++
 .../kylin/metrics/lib/impl/kafka/KafkaSink.java |  29 ++
 pom.xml |  25 ++
 31 files changed, 2490 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/55629865/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 34d8b7c..f3cf6c0 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -1295,9 +1295,9 @@ abstract public class KylinConfigBase implements 
Serializable {
 return 
Integer.valueOf(this.getOptional("kylin.restclient.connection.max-total", 
"200"));
 }
 
-/**
- * metric
- */
+// 

+// Metrics
+// 

 public String getCoadhaleMetricsReportClassesNames() {
 return getOptional("kylin.metrics.reporter-classes",
 
"org.apache.kylin.common.metrics.metrics2.JsonFileMetricsReporter,org.apache.kylin.common.metrics.metrics2.JmxMetricsReporter");
@@ -1315,4 +1315,8 @@ abstract public class KylinConfigBase implements 
Serializable {
 return getOptional("kylin.metrics.perflogger-class", 
"org.apache.kylin.common.metrics.perflog.PerfLogger");
 }
 
+public String getMetricsActiveReservoirDefaultClass() {
+return getOptional("kylin.metrics.active-reservoir-default-class",
+"org.apache.kylin.metrics.lib.impl.StubReservoir");
+}
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/55629865/core-metrics/pom.xml
--
diff --git a/core-metrics/pom.xml b/core-metrics/pom.xml
new file mode 100644
index 000..e436c97
--- /dev/null
+++ b/core-metrics/pom.xml
@@ -0,0 +1,51 @@
+
+
+
+http://www.w3.org/2001/XMLSchema-instance; 
xmlns="http://maven.apache.org/POM/4.0.0;
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 
http://maven.apache.org/xsd/maven-4.0.0.xsd;>
+4.0.0
+
+

[3/4] kylin git commit: KYLIN-2722 Introduce a new measure for dropwizard metrics framework, called active reservoir, for actively pushing metrics to reporters

2017-10-19 Thread liyang 
http://git-wip-us.apache.org/repos/asf/kylin/blob/55629865/metrics-reporter-hive/src/main/java/org/apache/kylin/metrics/lib/impl/hive/HiveReservoirReporter.java
--
diff --git 
a/metrics-reporter-hive/src/main/java/org/apache/kylin/metrics/lib/impl/hive/HiveReservoirReporter.java
 
b/metrics-reporter-hive/src/main/java/org/apache/kylin/metrics/lib/impl/hive/HiveReservoirReporter.java
new file mode 100644
index 000..5af2bf9
--- /dev/null
+++ 
b/metrics-reporter-hive/src/main/java/org/apache/kylin/metrics/lib/impl/hive/HiveReservoirReporter.java
@@ -0,0 +1,139 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.metrics.lib.impl.hive;
+
+import java.util.List;
+import java.util.Properties;
+
+import org.apache.kylin.metrics.lib.ActiveReservoir;
+import org.apache.kylin.metrics.lib.ActiveReservoirListener;
+import org.apache.kylin.metrics.lib.ActiveReservoirReporter;
+import org.apache.kylin.metrics.lib.Record;
+import org.apache.kylin.metrics.lib.impl.ReporterBuilder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * A reporter which listens for new records and publishes them to hive.
+ */
+public class HiveReservoirReporter extends ActiveReservoirReporter {
+
+public static final String HIVE_REPORTER_SUFFIX = "HIVE";
+public static final HiveSink sink = new HiveSink();
+protected static final Logger logger = 
LoggerFactory.getLogger(HiveReservoirReporter.class);
+private final ActiveReservoir activeReservoir;
+private final HiveReservoirListener listener;
+
+public HiveReservoirReporter(ActiveReservoir activeReservoir, Properties 
props) throws Exception {
+this.activeReservoir = activeReservoir;
+this.listener = new HiveReservoirListener(props);
+}
+
+/**
+ * Returns a new {@link Builder} for {@link HiveReservoirReporter}.
+ *
+ * @param activeReservoir the registry to report
+ * @return a {@link Builder} instance for a {@link HiveReservoirReporter}
+ */
+public static Builder forRegistry(ActiveReservoir activeReservoir) {
+return new Builder(activeReservoir);
+}
+
+public static String getTableFromSubject(String subject) {
+return sink.getTableFromSubject(subject);
+}
+
+/**
+ * Starts the reporter.
+ */
+public void start() {
+activeReservoir.addListener(listener);
+}
+
+/**
+ * Stops the reporter.
+ */
+public void stop() {
+activeReservoir.removeListener(listener);
+}
+
+/**
+ * Stops the reporter.
+ */
+@Override
+public void close() {
+stop();
+}
+
+/**
+ * A builder for {@link HiveReservoirReporter} instances.
+ */
+public static class Builder extends ReporterBuilder {
+
+private Builder(ActiveReservoir activeReservoir) {
+super(activeReservoir);
+}
+
+private void setFixedProperties() {
+}
+
+/**
+ * Builds a {@link HiveReservoirReporter} with the given properties.
+ *
+ * @return a {@link HiveReservoirReporter}
+ */
+public HiveReservoirReporter build() throws Exception {
+setFixedProperties();
+return new HiveReservoirReporter(registry, props);
+}
+}
+
+private class HiveReservoirListener implements ActiveReservoirListener {
+
+HiveProducer producer;
+
+private HiveReservoirListener(Properties props) throws Exception {
+producer = new HiveProducer(props);
+}
+
+public boolean onRecordUpdate(final List records) {
+try {
+producer.send(records);
+} catch (Exception e) {
+logger.error(e.getMessage(), e);
+return false;
+}
+return true;
+}
+
+public boolean onRecordUpdate(final Record record) {
+try {
+producer.send(record);
+} catch (Exception e) {
+logger.error(e.getMessage(), e);
+return false;
+}
+return true;
+}
+
+public void

[1/4] kylin git commit: KYLIN-2944 HLLCSerializer, RawSerializer, PercentileSerializer returns shared object in serialize [Forced Update!]

2017-10-19 Thread liyang 
Repository: kylin
Updated Branches:
  refs/heads/pr77 eaff61982 -> 556298656 (forced update)


KYLIN-2944 HLLCSerializer, RawSerializer, PercentileSerializer returns shared 
object in serialize


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/3efa9b4f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/3efa9b4f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/3efa9b4f

Branch: refs/heads/pr77
Commit: 3efa9b4fe17623af1d42ba09b389dabef73d789e
Parents: 8891b1c
Author: shaofengshi 
Authored: Tue Oct 17 22:30:42 2017 +0800
Committer: shaofengshi 
Committed: Wed Oct 18 09:14:40 2017 +0800

--
 .../main/java/org/apache/kylin/measure/hllc/HLLCSerializer.java   | 2 +-
 .../org/apache/kylin/measure/percentile/PercentileSerializer.java | 2 +-
 .../src/main/java/org/apache/kylin/measure/raw/RawSerializer.java | 3 +--
 3 files changed, 3 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/3efa9b4f/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLCSerializer.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLCSerializer.java 
b/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLCSerializer.java
index df0cfaf..ddf8281 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLCSerializer.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLCSerializer.java
@@ -56,7 +56,7 @@ public class HLLCSerializer extends 
DataTypeSerializer {
 
 @Override
 public HLLCounter deserialize(ByteBuffer in) {
-HLLCounter hllc = current();
+HLLCounter hllc = new HLLCounter(precision);
 try {
 hllc.readRegisters(in);
 } catch (IOException e) {

http://git-wip-us.apache.org/repos/asf/kylin/blob/3efa9b4f/core-metadata/src/main/java/org/apache/kylin/measure/percentile/PercentileSerializer.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/measure/percentile/PercentileSerializer.java
 
b/core-metadata/src/main/java/org/apache/kylin/measure/percentile/PercentileSerializer.java
index d7e4204..35230a2 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/measure/percentile/PercentileSerializer.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/measure/percentile/PercentileSerializer.java
@@ -68,7 +68,7 @@ public class PercentileSerializer extends 
DataTypeSerializer
 
 @Override
 public PercentileCounter deserialize(ByteBuffer in) {
-PercentileCounter counter = current();
+PercentileCounter counter = new PercentileCounter(compression);
 counter.readRegisters(in);
 return counter;
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/3efa9b4f/core-metadata/src/main/java/org/apache/kylin/measure/raw/RawSerializer.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/measure/raw/RawSerializer.java 
b/core-metadata/src/main/java/org/apache/kylin/measure/raw/RawSerializer.java
index 68a0273..ea3f376 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/measure/raw/RawSerializer.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/measure/raw/RawSerializer.java
@@ -90,8 +90,7 @@ public class RawSerializer extends 
DataTypeSerializer {
 
 @Override
 public List deserialize(ByteBuffer in) {
-List values = current();
-values.clear();
+List values = new ArrayList<>();
 int size = BytesUtil.readVInt(in);
 if (size >= 0) {
 for (int i = 0; i < size; i++) {

[2/4] kylin git commit: KYLIN-2947 Changed the Pop-up box when no project selected

2017-10-19 Thread liyang 
KYLIN-2947 Changed the Pop-up box when no project selected


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/27ba8c75
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/27ba8c75
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/27ba8c75

Branch: refs/heads/pr77
Commit: 27ba8c75518757fb6503c64915c204cb99e9d089
Parents: 3efa9b4
Author: peng.jianhua 
Authored: Thu Oct 19 16:48:58 2017 +0800
Committer: luguosheng1314 <550175...@qq.com>
Committed: Thu Oct 19 17:34:31 2017 +0800

--
 webapp/app/js/controllers/admin.js | 5 +
 1 file changed, 5 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/27ba8c75/webapp/app/js/controllers/admin.js
--
diff --git a/webapp/app/js/controllers/admin.js 
b/webapp/app/js/controllers/admin.js
index 0437942..bbba593 100644
--- a/webapp/app/js/controllers/admin.js
+++ b/webapp/app/js/controllers/admin.js
@@ -116,6 +116,11 @@ KylinApp.controller('AdminCtrl', function ($scope, 
AdminService, CacheService, T
   }
 
   $scope.calCardinality = function (tableName) {
+var _project = ProjectModel.selectedProject;
+  if (_project == null){
+SweetAlert.swal('', "No project selected.", 'info');
+  return;
+}
 $modal.open({
   templateUrl: 'calCardinality.html',
   controller: CardinalityGenCtrl,

kylin git commit: KYLIN-2947 Changed the Pop-up box when no project selected

2017-10-19 Thread boblu 
Repository: kylin
Updated Branches:
  refs/heads/master 3efa9b4fe -> 27ba8c755


KYLIN-2947 Changed the Pop-up box when no project selected


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/27ba8c75
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/27ba8c75
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/27ba8c75

Branch: refs/heads/master
Commit: 27ba8c75518757fb6503c64915c204cb99e9d089
Parents: 3efa9b4
Author: peng.jianhua 
Authored: Thu Oct 19 16:48:58 2017 +0800
Committer: luguosheng1314 <550175...@qq.com>
Committed: Thu Oct 19 17:34:31 2017 +0800

--
 webapp/app/js/controllers/admin.js | 5 +
 1 file changed, 5 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/27ba8c75/webapp/app/js/controllers/admin.js
--
diff --git a/webapp/app/js/controllers/admin.js 
b/webapp/app/js/controllers/admin.js
index 0437942..bbba593 100644
--- a/webapp/app/js/controllers/admin.js
+++ b/webapp/app/js/controllers/admin.js
@@ -116,6 +116,11 @@ KylinApp.controller('AdminCtrl', function ($scope, 
AdminService, CacheService, T
   }
 
   $scope.calCardinality = function (tableName) {
+var _project = ProjectModel.selectedProject;
+  if (_project == null){
+SweetAlert.swal('', "No project selected.", 'info');
+  return;
+}
 $modal.open({
   templateUrl: 'calCardinality.html',
   controller: CardinalityGenCtrl,