[04/50] [abbrv] kylin git commit: KYLIN-2072 Cleanup old streaming code

2016-10-18 Thread lidong
KYLIN-2072 Cleanup old streaming code

Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/5aee0226
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/5aee0226
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/5aee0226

Branch: refs/heads/master-cdh5.7
Commit: 5aee022612c6fa40c41e8c00063714b79b6d5237
Parents: cb2b12b
Author: shaofengshi 
Authored: Sun Oct 9 13:10:50 2016 +0800
Committer: shaofengshi 
Committed: Mon Oct 10 13:32:44 2016 +0800

--
 assembly/pom.xml|   4 -
 .../kylin/job/streaming/KafkaDataLoader.java|  79 
 build/bin/cleanup_streaming_files.sh|  42 --
 build/bin/kylin.sh  |  61 ---
 build/bin/streaming_build.sh|  33 --
 build/bin/streaming_check.sh|  29 --
 build/bin/streaming_fillgap.sh  |  26 --
 build/bin/streaming_rolllog.sh  |  29 --
 .../metadata/streaming/StreamingConfig.java |  85 
 .../metadata/streaming/StreamingManager.java| 248 
 .../.settings/org.eclipse.core.resources.prefs  |   6 -
 .../.settings/org.eclipse.jdt.core.prefs| 386 ---
 .../.settings/org.eclipse.jdt.ui.prefs  |   7 -
 engine-streaming/pom.xml| 121 --
 .../kylin/engine/streaming/BootstrapConfig.java |  71 
 .../kylin/engine/streaming/IStreamingInput.java |  30 --
 .../engine/streaming/IStreamingOutput.java  |  34 --
 .../streaming/OneOffStreamingBuilder.java   |  71 
 .../engine/streaming/StreamingBatchBuilder.java |  43 ---
 .../kylin/engine/streaming/StreamingConfig.java |  85 
 .../engine/streaming/StreamingManager.java  | 248 
 .../kylin/engine/streaming/cli/MonitorCLI.java  |  88 -
 .../engine/streaming/cli/StreamingCLI.java  | 114 --
 .../streaming/cube/StreamingCubeBuilder.java| 168 
 .../diagnose/StreamingLogAnalyzer.java  |  96 -
 .../streaming/monitor/StreamingMonitor.java | 172 -
 .../engine/streaming/util/StreamingUtils.java   |  51 ---
 .../kylin/provision/BuildCubeWithStream.java|   4 +-
 pom.xml |   6 -
 .../rest/controller/StreamingController.java|   2 +-
 .../kylin/rest/controller/TableController.java  |   2 +-
 .../apache/kylin/rest/service/BasicService.java |   2 +-
 .../kylin/rest/service/StreamingService.java|   2 +-
 source-kafka/pom.xml|   6 -
 .../kafka/ByteBufferBackedInputStream.java  |  52 ---
 .../apache/kylin/source/kafka/KafkaSource.java  |   2 +-
 .../kylin/source/kafka/KafkaStreamingInput.java | 227 ---
 .../source/kafka/TimedJsonStreamParser.java |   1 +
 .../kafka/diagnose/KafkaInputAnalyzer.java  | 312 ---
 .../source/kafka/diagnose/KafkaVerify.java  | 101 -
 .../source/kafka/diagnose/TimeHistogram.java|  85 
 .../kafka/util/ByteBufferBackedInputStream.java |  52 +++
 .../kylin/source/kafka/util/KafkaRequester.java | 191 -
 .../kylin/source/kafka/util/KafkaUtils.java | 173 -
 .../test/java/TimedJsonStreamParserTest.java|   4 +-
 storage-hbase/pom.xml   |   4 -
 .../hbase/steps/HBaseStreamingOutput.java   |  98 -
 .../apache/kylin/tool/CubeMetaExtractor.java|   4 +-
 48 files changed, 397 insertions(+), 3360 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/5aee0226/assembly/pom.xml
--
diff --git a/assembly/pom.xml b/assembly/pom.xml
index 0c80afc..e6f83a8 100644
--- a/assembly/pom.xml
+++ b/assembly/pom.xml
@@ -47,10 +47,6 @@
 org.apache.kylin
 kylin-engine-mr
 
-
-org.apache.kylin
-kylin-engine-streaming
-
 
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/5aee0226/assembly/src/test/java/org/apache/kylin/job/streaming/KafkaDataLoader.java
--
diff --git 
a/assembly/src/test/java/org/apache/kylin/job/streaming/KafkaDataLoader.java 
b/assembly/src/test/java/org/apache/kylin/job/streaming/KafkaDataLoader.java
deleted file mode 100644
index 454f6cf..000
--- a/assembly/src/test/java/org/apache/kylin/job/streaming/KafkaDataLoader.java
+++ /dev/null
@@ -1,79 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *

[04/50] [abbrv] kylin git commit: KYLIN-2072 Cleanup old streaming code

2016-10-18 Thread lidong
http://git-wip-us.apache.org/repos/asf/kylin/blob/5aee0226/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/StreamingManager.java
--
diff --git 
a/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/StreamingManager.java
 
b/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/StreamingManager.java
deleted file mode 100644
index 271bf41..000
--- 
a/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/StreamingManager.java
+++ /dev/null
@@ -1,248 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.engine.streaming;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.concurrent.ConcurrentHashMap;
-
-import org.apache.commons.lang3.StringUtils;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.persistence.JsonSerializer;
-import org.apache.kylin.common.persistence.ResourceStore;
-import org.apache.kylin.common.persistence.Serializer;
-import org.apache.kylin.metadata.MetadataConstants;
-import org.apache.kylin.metadata.cachesync.Broadcaster;
-import org.apache.kylin.metadata.cachesync.Broadcaster.Event;
-import org.apache.kylin.metadata.cachesync.CaseInsensitiveStringCache;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- */
-public class StreamingManager {
-
-private static final Logger logger = 
LoggerFactory.getLogger(StreamingManager.class);
-
-// static cached instances
-private static final ConcurrentHashMap 
CACHE = new ConcurrentHashMap();
-
-public static final Serializer STREAMING_SERIALIZER = new 
JsonSerializer(StreamingConfig.class);
-
-private KylinConfig config;
-
-// name ==> StreamingConfig
-private CaseInsensitiveStringCache streamingMap;
-
-public static void clearCache() {
-CACHE.clear();
-}
-
-private StreamingManager(KylinConfig config) throws IOException {
-this.config = config;
-this.streamingMap = new 
CaseInsensitiveStringCache(config, "streaming");
-
-// touch lower level metadata before registering my listener
-reloadAllStreaming();
-Broadcaster.getInstance(config).registerListener(new 
StreamingSyncListener(), "streaming");
-}
-
-private class StreamingSyncListener extends Broadcaster.Listener {
-@Override
-public void onClearAll(Broadcaster broadcaster) throws IOException {
-clearCache();
-}
-
-@Override
-public void onEntityChange(Broadcaster broadcaster, String entity, 
Event event, String cacheKey) throws IOException {
-if (event == Event.DROP)
-removeStreamingLocal(cacheKey);
-else
-reloadStreamingConfigLocal(cacheKey);
-}
-}
-
-private ResourceStore getStore() {
-return ResourceStore.getStore(this.config);
-}
-
-public static StreamingManager getInstance(KylinConfig config) {
-StreamingManager r = CACHE.get(config);
-if (r != null) {
-return r;
-}
-
-synchronized (StreamingManager.class) {
-r = CACHE.get(config);
-if (r != null) {
-return r;
-}
-try {
-r = new StreamingManager(config);
-CACHE.put(config, r);
-if (CACHE.size() > 1) {
-logger.warn("More than one singleton exist");
-}
-return r;
-} catch (IOException e) {
-throw new IllegalStateException("Failed to init 
StreamingManager from " + config, e);
-}
-}
-}
-
-private static String formatStreamingConfigPath(String name) {
-return ResourceStore.STREAMING_RESOURCE_ROOT + "/" + name + ".json";
-}
-
-private static String formatStreamingOutputPath(String streaming, int 
partition) {
-return ResourceStore.STREAMING_OUTPUT_RESOURCE_ROOT + "/" + streaming 
+ "_" + partition + ".json";
-}
-
-private static String formatStreamingOutputPath(String streaming, 
List partitions) {
-retu