fix GEARPUMP-118 change package name to org.apache Author: huafengw <[email protected]>
Closes #14 from huafengw/rename. Project: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/commit/83b36ef7 Tree: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/tree/83b36ef7 Diff: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/diff/83b36ef7 Branch: refs/heads/master Commit: 83b36ef76de786534fe92a6d75149d4d561915f4 Parents: 9bb9ca5 Author: huafengw <[email protected]> Authored: Tue May 10 10:41:43 2016 +0800 Committer: manuzhang <[email protected]> Committed: Tue May 10 10:41:43 2016 +0800 ---------------------------------------------------------------------- conf/gear.conf | 28 +- conf/log4j.properties | 4 +- .../netty/ITransportMessageSerializer.java | 31 - .../gearpump/transport/netty/MessageBatch.java | 152 ----- .../transport/netty/MessageDecoder.java | 104 ---- .../transport/netty/MessageEncoder.java | 34 -- .../netty/NettyRenameThreadFactory.java | 53 -- .../gearpump/transport/netty/TaskMessage.java | 56 -- .../transport/netty/WrappedChannelBuffer.java | 116 ---- .../main/java/io/gearpump/util/AkkaHelper.java | 40 -- .../io/gearpump/util/HadoopFSLogAppender.java | 28 - .../util/RecreateRollingFileAppender.java | 64 -- .../netty/ITransportMessageSerializer.java | 31 + .../gearpump/transport/netty/MessageBatch.java | 152 +++++ .../transport/netty/MessageDecoder.java | 104 ++++ .../transport/netty/MessageEncoder.java | 34 ++ .../netty/NettyRenameThreadFactory.java | 53 ++ .../gearpump/transport/netty/TaskMessage.java | 56 ++ .../transport/netty/WrappedChannelBuffer.java | 116 ++++ .../org/apache/gearpump/util/AkkaHelper.java | 40 ++ .../gearpump/util/HadoopFSLogAppender.java | 28 + .../util/RecreateRollingFileAppender.java | 64 ++ core/src/main/resources/geardefault.conf | 34 +- core/src/main/resources/log4j.properties | 4 +- core/src/main/scala/io/gearpump/Message.scala | 32 - .../io/gearpump/cluster/AppDescription.scala | 145 ----- .../io/gearpump/cluster/ClusterConfig.scala | 184 ------ .../gearpump/cluster/ClusterConfigSource.scala | 63 -- .../io/gearpump/cluster/ClusterMessage.scala | 287 --------- .../scala/io/gearpump/cluster/UserConfig.scala | 162 ------ .../appmaster/AppMasterRuntimeEnvironment.scala | 195 ------- .../appmaster/AppMasterRuntimeInfo.scala | 38 -- .../cluster/appmaster/ApplicationState.scala | 47 -- .../cluster/appmaster/ExecutorSystem.scala | 43 -- .../appmaster/ExecutorSystemLauncher.scala | 114 ---- .../appmaster/ExecutorSystemScheduler.scala | 190 ------ .../appmaster/MasterConnectionKeeper.scala | 98 ---- .../gearpump/cluster/client/ClientContext.scala | 185 ------ .../gearpump/cluster/client/MasterClient.scala | 81 --- .../gearpump/cluster/main/ArgumentsParser.scala | 139 ----- .../cluster/master/AppMasterLauncher.scala | 147 ----- .../gearpump/cluster/master/MasterProxy.scala | 134 ----- .../gearpump/cluster/master/MasterSummary.scala | 51 -- .../gearpump/cluster/scheduler/Resource.scala | 80 --- .../worker/ExecutorProcessLauncher.scala | 56 -- .../io/gearpump/cluster/worker/WorkerId.scala | 63 -- .../gearpump/cluster/worker/WorkerSummary.scala | 47 -- .../io/gearpump/jarstore/JarStoreService.scala | 86 --- .../io/gearpump/metrics/AkkaReporter.scala | 78 --- .../scala/io/gearpump/metrics/Counter.scala | 42 -- .../scala/io/gearpump/metrics/Histogram.scala | 43 -- .../io/gearpump/metrics/JvmMetricsSet.scala | 43 -- .../main/scala/io/gearpump/metrics/Meter.scala | 44 -- .../scala/io/gearpump/metrics/Metrics.scala | 160 ----- .../io/gearpump/metrics/MetricsAggregator.scala | 32 - .../metrics/MetricsReporterService.scala | 109 ---- core/src/main/scala/io/gearpump/package.scala | 24 - .../partitioner/BroadcastPartitioner.scala | 36 -- .../partitioner/CoLocationPartitioner.scala | 31 - .../gearpump/partitioner/HashPartitioner.scala | 32 - .../io/gearpump/partitioner/Partitioner.scala | 109 ---- .../ShuffleGroupingPartitioner.scala | 45 -- .../partitioner/ShufflePartitioner.scala | 44 -- .../io/gearpump/security/Authenticator.scala | 68 --- .../security/ConfigFileBasedAuthenticator.scala | 120 ---- .../io/gearpump/security/PasswordUtil.scala | 101 ---- .../FastKryoSerializationFramework.scala | 46 -- .../serializer/FastKryoSerializer.scala | 83 --- .../serializer/GearpumpSerialization.scala | 60 -- .../serializer/SerializationFramework.scala | 47 -- .../io/gearpump/serializer/Serializer.scala | 28 - .../scala/io/gearpump/transport/Express.scala | 135 ----- .../scala/io/gearpump/transport/HostPort.scala | 32 - .../io/gearpump/transport/netty/Client.scala | 248 -------- .../io/gearpump/transport/netty/Context.scala | 113 ---- .../io/gearpump/transport/netty/IContext.scala | 41 -- .../gearpump/transport/netty/NettyConfig.scala | 39 -- .../io/gearpump/transport/netty/NettyUtil.scala | 67 --- .../io/gearpump/transport/netty/Server.scala | 134 ----- .../io/gearpump/util/ActorSystemBooter.scala | 152 ----- .../main/scala/io/gearpump/util/ActorUtil.scala | 130 ----- .../main/scala/io/gearpump/util/AkkaApp.scala | 45 -- .../main/scala/io/gearpump/util/Constants.scala | 177 ------ .../main/scala/io/gearpump/util/FileUtils.scala | 53 -- .../src/main/scala/io/gearpump/util/Graph.scala | 497 ---------------- .../gearpump/util/HistoryMetricsService.scala | 404 ------------- .../main/scala/io/gearpump/util/LogUtil.scala | 125 ---- .../io/gearpump/util/ProcessLogRedirector.scala | 53 -- .../scala/io/gearpump/util/ReferenceEqual.scala | 33 -- .../scala/io/gearpump/util/RestartPolicy.scala | 41 -- .../scala/io/gearpump/util/RichProcess.scala | 33 -- .../io/gearpump/util/TimeOutScheduler.scala | 43 -- core/src/main/scala/io/gearpump/util/Util.scala | 181 ------ .../scala/org/apache/gearpump/Message.scala | 32 + .../gearpump/cluster/AppDescription.scala | 145 +++++ .../apache/gearpump/cluster/ClusterConfig.scala | 180 ++++++ .../gearpump/cluster/ClusterConfigSource.scala | 63 ++ .../gearpump/cluster/ClusterMessage.scala | 288 +++++++++ .../apache/gearpump/cluster/UserConfig.scala | 162 ++++++ .../appmaster/AppMasterRuntimeEnvironment.scala | 195 +++++++ .../appmaster/AppMasterRuntimeInfo.scala | 38 ++ .../cluster/appmaster/ApplicationState.scala | 47 ++ .../cluster/appmaster/ExecutorSystem.scala | 43 ++ .../appmaster/ExecutorSystemLauncher.scala | 114 ++++ .../appmaster/ExecutorSystemScheduler.scala | 191 ++++++ .../appmaster/MasterConnectionKeeper.scala | 98 ++++ .../gearpump/cluster/client/ClientContext.scala | 185 ++++++ .../gearpump/cluster/client/MasterClient.scala | 81 +++ .../gearpump/cluster/main/ArgumentsParser.scala | 139 +++++ .../cluster/master/AppMasterLauncher.scala | 148 +++++ .../gearpump/cluster/master/MasterProxy.scala | 134 +++++ .../gearpump/cluster/master/MasterSummary.scala | 51 ++ .../gearpump/cluster/scheduler/Resource.scala | 79 +++ .../worker/ExecutorProcessLauncher.scala | 56 ++ .../gearpump/cluster/worker/WorkerId.scala | 63 ++ .../gearpump/cluster/worker/WorkerSummary.scala | 47 ++ .../gearpump/jarstore/JarStoreService.scala | 86 +++ .../apache/gearpump/metrics/AkkaReporter.scala | 78 +++ .../org/apache/gearpump/metrics/Counter.scala | 42 ++ .../org/apache/gearpump/metrics/Histogram.scala | 43 ++ .../apache/gearpump/metrics/JvmMetricsSet.scala | 43 ++ .../org/apache/gearpump/metrics/Meter.scala | 44 ++ .../org/apache/gearpump/metrics/Metrics.scala | 160 +++++ .../gearpump/metrics/MetricsAggregator.scala | 32 + .../metrics/MetricsReporterService.scala | 109 ++++ .../scala/org/apache/gearpump/package.scala | 24 + .../partitioner/BroadcastPartitioner.scala | 36 ++ .../partitioner/CoLocationPartitioner.scala | 31 + .../gearpump/partitioner/HashPartitioner.scala | 32 + .../gearpump/partitioner/Partitioner.scala | 109 ++++ .../ShuffleGroupingPartitioner.scala | 45 ++ .../partitioner/ShufflePartitioner.scala | 44 ++ .../gearpump/security/Authenticator.scala | 68 +++ .../security/ConfigFileBasedAuthenticator.scala | 120 ++++ .../apache/gearpump/security/PasswordUtil.scala | 102 ++++ .../FastKryoSerializationFramework.scala | 46 ++ .../serializer/FastKryoSerializer.scala | 83 +++ .../serializer/GearpumpSerialization.scala | 60 ++ .../serializer/SerializationFramework.scala | 44 ++ .../apache/gearpump/serializer/Serializer.scala | 28 + .../org/apache/gearpump/transport/Express.scala | 135 +++++ .../apache/gearpump/transport/HostPort.scala | 32 + .../gearpump/transport/netty/Client.scala | 249 ++++++++ .../gearpump/transport/netty/Context.scala | 114 ++++ .../gearpump/transport/netty/IContext.scala | 41 ++ .../gearpump/transport/netty/NettyConfig.scala | 39 ++ .../gearpump/transport/netty/NettyUtil.scala | 67 +++ .../gearpump/transport/netty/Server.scala | 135 +++++ .../gearpump/util/ActorSystemBooter.scala | 152 +++++ .../org/apache/gearpump/util/ActorUtil.scala | 131 +++++ .../org/apache/gearpump/util/AkkaApp.scala | 45 ++ .../org/apache/gearpump/util/Constants.scala | 177 ++++++ .../org/apache/gearpump/util/FileUtils.scala | 53 ++ .../scala/org/apache/gearpump/util/Graph.scala | 496 ++++++++++++++++ .../gearpump/util/HistoryMetricsService.scala | 404 +++++++++++++ .../org/apache/gearpump/util/LogUtil.scala | 125 ++++ .../gearpump/util/ProcessLogRedirector.scala | 53 ++ .../apache/gearpump/util/ReferenceEqual.scala | 33 ++ .../apache/gearpump/util/RestartPolicy.scala | 41 ++ .../org/apache/gearpump/util/RichProcess.scala | 33 ++ .../apache/gearpump/util/TimeOutScheduler.scala | 43 ++ .../scala/org/apache/gearpump/util/Util.scala | 181 ++++++ core/src/test/resources/log4j.properties | 4 +- core/src/test/resources/test.conf | 6 +- .../test/scala/io/gearpump/TestProbeUtil.scala | 37 -- .../io/gearpump/cluster/MasterHarness.scala | 160 ----- .../scala/io/gearpump/cluster/TestUtil.scala | 36 -- .../AppMasterRuntimeEnvironmentSpec.scala | 140 ----- .../appmaster/ExecutorSystemLauncherSpec.scala | 105 ---- .../appmaster/ExecutorSystemSchedulerSpec.scala | 140 ----- .../appmaster/MasterConnectionKeeperSpec.scala | 118 ---- .../cluster/main/ArgumentParserSpec.scala | 63 -- .../cluster/master/AppMasterLauncherSpec.scala | 100 ---- .../cluster/master/ApplicationStateSpec.scala | 36 -- .../cluster/master/MasterProxySpec.scala | 24 - .../io/gearpump/cluster/master/MasterSpec.scala | 22 - .../scala/io/gearpump/metrics/MetricsSpec.scala | 122 ---- .../gearpump/partitioner/PartitionerSpec.scala | 55 -- .../ConfigFileBasedAuthenticatorSpec.scala | 54 -- .../io/gearpump/security/PasswordUtilSpec.scala | 38 -- .../io/gearpump/serializer/SerializerSpec.scala | 91 --- .../transport/MockTransportSerializer.scala | 43 -- .../scala/io/gearpump/transport/NettySpec.scala | 62 -- .../gearpump/util/ActorSystemBooterSpec.scala | 118 ---- .../scala/io/gearpump/util/ActorUtilSpec.scala | 40 -- .../scala/io/gearpump/util/ConfigsSpec.scala | 99 ---- .../scala/io/gearpump/util/FileUtilsSpec.scala | 62 -- .../test/scala/io/gearpump/util/GraphSpec.scala | 213 ------- .../io/gearpump/util/TimeOutSchedulerSpec.scala | 75 --- .../test/scala/io/gearpump/util/UtilSpec.scala | 46 -- .../org/apache/gearpump/TestProbeUtil.scala | 37 ++ .../apache/gearpump/cluster/MasterHarness.scala | 160 +++++ .../org/apache/gearpump/cluster/TestUtil.scala | 36 ++ .../AppMasterRuntimeEnvironmentSpec.scala | 140 +++++ .../appmaster/ExecutorSystemLauncherSpec.scala | 106 ++++ .../appmaster/ExecutorSystemSchedulerSpec.scala | 141 +++++ .../appmaster/MasterConnectionKeeperSpec.scala | 118 ++++ .../cluster/main/ArgumentParserSpec.scala | 63 ++ .../cluster/master/AppMasterLauncherSpec.scala | 101 ++++ .../cluster/master/ApplicationStateSpec.scala | 36 ++ .../cluster/master/MasterProxySpec.scala | 24 + .../gearpump/cluster/master/MasterSpec.scala | 22 + .../apache/gearpump/metrics/MetricsSpec.scala | 122 ++++ .../gearpump/partitioner/PartitionerSpec.scala | 55 ++ .../ConfigFileBasedAuthenticatorSpec.scala | 54 ++ .../gearpump/security/PasswordUtilSpec.scala | 38 ++ .../gearpump/serializer/SerializerSpec.scala | 91 +++ .../transport/MockTransportSerializer.scala | 42 ++ .../apache/gearpump/transport/NettySpec.scala | 63 ++ .../gearpump/util/ActorSystemBooterSpec.scala | 118 ++++ .../apache/gearpump/util/ActorUtilSpec.scala | 40 ++ .../org/apache/gearpump/util/ConfigsSpec.scala | 99 ++++ .../apache/gearpump/util/FileUtilsSpec.scala | 62 ++ .../org/apache/gearpump/util/GraphSpec.scala | 213 +++++++ .../gearpump/util/TimeOutSchedulerSpec.scala | 75 +++ .../org/apache/gearpump/util/UtilSpec.scala | 46 ++ .../io.gearpump.jarstore.JarStoreService | 20 - ...org.apache.gearpump.jarstore.JarStoreService | 20 + .../io/gearpump/cluster/DaemonMessage.scala | 51 -- .../cluster/embedded/EmbeddedCluster.scala | 95 --- .../io/gearpump/cluster/main/AppSubmitter.scala | 106 ---- .../scala/io/gearpump/cluster/main/Gear.scala | 81 --- .../scala/io/gearpump/cluster/main/Info.scala | 53 -- .../scala/io/gearpump/cluster/main/Kill.scala | 50 -- .../scala/io/gearpump/cluster/main/Local.scala | 90 --- .../io/gearpump/cluster/main/MainRunner.scala | 43 -- .../scala/io/gearpump/cluster/main/Master.scala | 236 -------- .../scala/io/gearpump/cluster/main/Replay.scala | 48 -- .../scala/io/gearpump/cluster/main/Worker.scala | 71 --- .../io/gearpump/cluster/master/AppManager.scala | 341 ----------- .../cluster/master/InMemoryKVService.scala | 122 ---- .../io/gearpump/cluster/master/Master.scala | 313 ---------- .../cluster/scheduler/PriorityScheduler.scala | 155 ----- .../gearpump/cluster/scheduler/Scheduler.scala | 78 --- .../worker/DefaultExecutorProcessLauncher.scala | 41 -- .../io/gearpump/cluster/worker/Worker.scala | 580 ------------------ .../jarstore/dfs/DFSJarStoreService.scala | 76 --- .../gearpump/jarstore/local/LocalJarStore.scala | 64 -- .../jarstore/local/LocalJarStoreService.scala | 81 --- .../scala/io/gearpump/util/FileDirective.scala | 140 ----- .../scala/io/gearpump/util/FileServer.scala | 167 ------ .../apache/gearpump/cluster/DaemonMessage.scala | 51 ++ .../cluster/embedded/EmbeddedCluster.scala | 95 +++ .../gearpump/cluster/main/AppSubmitter.scala | 106 ++++ .../org/apache/gearpump/cluster/main/Gear.scala | 81 +++ .../org/apache/gearpump/cluster/main/Info.scala | 53 ++ .../org/apache/gearpump/cluster/main/Kill.scala | 50 ++ .../apache/gearpump/cluster/main/Local.scala | 90 +++ .../gearpump/cluster/main/MainRunner.scala | 43 ++ .../apache/gearpump/cluster/main/Master.scala | 236 ++++++++ .../apache/gearpump/cluster/main/Replay.scala | 48 ++ .../apache/gearpump/cluster/main/Worker.scala | 71 +++ .../gearpump/cluster/master/AppManager.scala | 341 +++++++++++ .../cluster/master/InMemoryKVService.scala | 122 ++++ .../apache/gearpump/cluster/master/Master.scala | 314 ++++++++++ .../cluster/scheduler/PriorityScheduler.scala | 156 +++++ .../gearpump/cluster/scheduler/Scheduler.scala | 79 +++ .../worker/DefaultExecutorProcessLauncher.scala | 41 ++ .../apache/gearpump/cluster/worker/Worker.scala | 581 +++++++++++++++++++ .../jarstore/dfs/DFSJarStoreService.scala | 76 +++ .../gearpump/jarstore/local/LocalJarStore.scala | 64 ++ .../jarstore/local/LocalJarStoreService.scala | 81 +++ .../apache/gearpump/util/FileDirective.scala | 140 +++++ .../org/apache/gearpump/util/FileServer.scala | 167 ++++++ .../io.gearpump.jarstore.JarStoreService | 20 - ...org.apache.gearpump.jarstore.JarStoreService | 20 + .../scala/io/gearpump/cluster/MiniCluster.scala | 74 --- .../io/gearpump/cluster/main/MainSpec.scala | 189 ------ .../cluster/main/MasterWatcherSpec.scala | 44 -- .../cluster/master/AppManagerSpec.scala | 181 ------ .../cluster/master/InMemoryKVServiceSpec.scala | 69 --- .../scheduler/PrioritySchedulerSpec.scala | 231 -------- .../io/gearpump/cluster/worker/WorkerSpec.scala | 129 ---- .../scala/io/gearpump/util/FileServerSpec.scala | 120 ---- .../apache/gearpump/cluster/MiniCluster.scala | 74 +++ .../apache/gearpump/cluster/main/MainSpec.scala | 189 ++++++ .../cluster/main/MasterWatcherSpec.scala | 44 ++ .../cluster/master/AppManagerSpec.scala | 181 ++++++ .../cluster/master/InMemoryKVServiceSpec.scala | 69 +++ .../scheduler/PrioritySchedulerSpec.scala | 232 ++++++++ .../gearpump/cluster/worker/WorkerSpec.scala | 129 ++++ .../apache/gearpump/util/FileServerSpec.scala | 120 ++++ docs/deployment-configuration.md | 4 +- docs/deployment-resource-isolation.md | 2 +- docs/deployment-ui-authentication.md | 10 +- docs/dev-connectors.md | 6 +- docs/dev-custom-serializer.md | 26 +- docs/dev-non-streaming-example.md | 2 +- docs/dev-rest-api.md | 100 ++-- docs/dev-write-1st-app.md | 6 +- docs/submit-your-1st-application.md | 2 +- examples/distributedshell/README.md | 4 +- .../distributedshell/DistShellAppMaster.scala | 95 --- .../distributedshell/DistributedShell.scala | 41 -- .../DistributedShellClient.scala | 57 -- .../distributedshell/ShellExecutor.scala | 47 -- .../distributedshell/DistShellAppMaster.scala | 95 +++ .../distributedshell/DistributedShell.scala | 41 ++ .../DistributedShellClient.scala | 57 ++ .../distributedshell/ShellExecutor.scala | 47 ++ .../DistShellAppMasterSpec.scala | 78 --- .../DistributedShellClientSpec.scala | 66 --- .../distributedshell/DistributedShellSpec.scala | 56 -- .../ShellCommandResultAggregatorSpec.scala | 38 -- .../distributedshell/ShellExecutorSpec.scala | 66 --- .../DistShellAppMasterSpec.scala | 78 +++ .../DistributedShellClientSpec.scala | 66 +++ .../distributedshell/DistributedShellSpec.scala | 56 ++ .../ShellCommandResultAggregatorSpec.scala | 38 ++ .../distributedshell/ShellExecutorSpec.scala | 66 +++ examples/distributeservice/README.md | 4 +- .../DistServiceAppMaster.scala | 100 ---- .../distributeservice/DistServiceExecutor.scala | 89 --- .../distributeservice/DistributeService.scala | 41 -- .../DistributeServiceClient.scala | 90 --- .../distributeservice/FileServer.scala | 138 ----- .../DistServiceAppMaster.scala | 100 ++++ .../distributeservice/DistServiceExecutor.scala | 89 +++ .../distributeservice/DistributeService.scala | 41 ++ .../DistributeServiceClient.scala | 90 +++ .../distributeservice/FileServer.scala | 138 +++++ .../DistServiceAppMasterSpec.scala | 87 --- .../DistServiceAppMasterSpec.scala | 87 +++ examples/pagerank/README.md | 2 +- .../src/main/resources/geardefault.conf | 6 +- .../pagerank/PageRankApplication.scala | 77 --- .../pagerank/PageRankController.scala | 79 --- .../experiments/pagerank/PageRankWorker.scala | 83 --- .../pagerank/example/PageRankExample.scala | 42 -- .../pagerank/PageRankApplication.scala | 77 +++ .../pagerank/PageRankController.scala | 79 +++ .../experiments/pagerank/PageRankWorker.scala | 83 +++ .../pagerank/example/PageRankExample.scala | 42 ++ .../streaming/examples/complexdag/Dag.scala | 113 ---- .../streaming/examples/complexdag/Node.scala | 34 -- .../streaming/examples/complexdag/Sink.scala | 45 -- .../streaming/examples/complexdag/Source.scala | 38 -- .../streaming/examples/complexdag/Dag.scala | 113 ++++ .../streaming/examples/complexdag/Node.scala | 34 ++ .../streaming/examples/complexdag/Sink.scala | 45 ++ .../streaming/examples/complexdag/Source.scala | 38 ++ .../streaming/examples/complexdag/DagSpec.scala | 56 -- .../examples/complexdag/NodeSpec.scala | 42 -- .../examples/complexdag/SinkSpec.scala | 43 -- .../examples/complexdag/SourceSpec.scala | 45 -- .../streaming/examples/complexdag/DagSpec.scala | 55 ++ .../examples/complexdag/NodeSpec.scala | 41 ++ .../examples/complexdag/SinkSpec.scala | 43 ++ .../examples/complexdag/SourceSpec.scala | 44 ++ examples/streaming/fsio/README.md | 2 +- .../streaming/examples/fsio/HadoopConfig.scala | 61 -- .../examples/fsio/SeqFileStreamProcessor.scala | 96 --- .../examples/fsio/SeqFileStreamProducer.scala | 68 --- .../examples/fsio/SequenceFileIO.scala | 67 --- .../streaming/examples/fsio/HadoopConfig.scala | 61 ++ .../examples/fsio/SeqFileStreamProcessor.scala | 96 +++ .../examples/fsio/SeqFileStreamProducer.scala | 68 +++ .../examples/fsio/SequenceFileIO.scala | 67 +++ .../examples/fsio/HadoopConfigSpec.scala | 40 -- .../fsio/SeqFileStreamProcessorSpec.scala | 94 --- .../fsio/SeqFileStreamProducerSpec.scala | 87 --- .../examples/fsio/SequenceFileIOSpec.scala | 83 --- .../examples/fsio/HadoopConfigSpec.scala | 40 ++ .../fsio/SeqFileStreamProcessorSpec.scala | 94 +++ .../fsio/SeqFileStreamProducerSpec.scala | 87 +++ .../examples/fsio/SequenceFileIOSpec.scala | 83 +++ examples/streaming/kafka/README.md | 2 +- .../examples/kafka/KafkaReadWrite.scala | 80 --- .../kafka/wordcount/KafkaWordCount.scala | 79 --- .../examples/kafka/wordcount/Split.scala | 38 -- .../examples/kafka/wordcount/Sum.scala | 43 -- .../examples/kafka/KafkaReadWrite.scala | 80 +++ .../kafka/wordcount/KafkaWordCount.scala | 79 +++ .../examples/kafka/wordcount/Split.scala | 38 ++ .../examples/kafka/wordcount/Sum.scala | 43 ++ .../kafka/wordcount/KafkaWordCountSpec.scala | 71 --- .../examples/kafka/wordcount/SplitSpec.scala | 40 -- .../examples/kafka/wordcount/SumSpec.scala | 58 -- .../kafka/wordcount/KafkaWordCountSpec.scala | 71 +++ .../examples/kafka/wordcount/SplitSpec.scala | 40 ++ .../examples/kafka/wordcount/SumSpec.scala | 58 ++ examples/streaming/sol/README.md | 2 +- .../gearpump/streaming/examples/sol/SOL.scala | 68 --- .../examples/sol/SOLStreamProcessor.scala | 66 --- .../examples/sol/SOLStreamProducer.scala | 78 --- .../gearpump/streaming/examples/sol/SOL.scala | 68 +++ .../examples/sol/SOLStreamProcessor.scala | 66 +++ .../examples/sol/SOLStreamProducer.scala | 78 +++ .../streaming/examples/sol/SOLSpec.scala | 70 --- .../examples/sol/SOLStreamProcessorSpec.scala | 42 -- .../examples/sol/SOLStreamProducerSpec.scala | 43 -- .../streaming/examples/sol/SOLSpec.scala | 70 +++ .../examples/sol/SOLStreamProcessorSpec.scala | 42 ++ .../examples/sol/SOLStreamProducerSpec.scala | 43 ++ .../state/src/main/resources/state.conf | 2 +- .../examples/state/MessageCountApp.scala | 100 ---- .../examples/state/WindowAverageApp.scala | 72 --- .../state/processor/CountProcessor.scala | 41 -- .../processor/NumberGeneratorProcessor.scala | 42 -- .../processor/WindowAverageProcessor.scala | 54 -- .../examples/state/MessageCountApp.scala | 100 ++++ .../examples/state/WindowAverageApp.scala | 72 +++ .../state/processor/CountProcessor.scala | 41 ++ .../processor/NumberGeneratorProcessor.scala | 42 ++ .../processor/WindowAverageProcessor.scala | 54 ++ .../examples/state/MessageCountAppSpec.scala | 79 --- .../examples/state/WindowAverageAppSpec.scala | 76 --- .../state/processor/CountProcessorSpec.scala | 84 --- .../NumberGeneratorProcessorSpec.scala | 61 -- .../processor/WindowAverageProcessorSpec.scala | 86 --- .../examples/state/MessageCountAppSpec.scala | 79 +++ .../examples/state/WindowAverageAppSpec.scala | 76 +++ .../state/processor/CountProcessorSpec.scala | 84 +++ .../NumberGeneratorProcessorSpec.scala | 61 ++ .../processor/WindowAverageProcessorSpec.scala | 86 +++ examples/streaming/stockcrawler/README.md | 4 +- .../src/main/resources/geardefault.conf | 2 +- .../streaming/examples/stock/Analyzer.scala | 170 ------ .../streaming/examples/stock/Crawler.scala | 60 -- .../streaming/examples/stock/Data.scala | 61 -- .../streaming/examples/stock/QueryServer.scala | 134 ----- .../streaming/examples/stock/StockMarket.scala | 155 ----- .../streaming/examples/stock/main/Stock.scala | 86 --- .../streaming/examples/stock/Analyzer.scala | 170 ++++++ .../streaming/examples/stock/Crawler.scala | 60 ++ .../streaming/examples/stock/Data.scala | 61 ++ .../streaming/examples/stock/QueryServer.scala | 134 +++++ .../streaming/examples/stock/StockMarket.scala | 155 +++++ .../streaming/examples/stock/main/Stock.scala | 86 +++ .../src/main/resources/geardefault.conf | 4 +- .../streaming/examples/transport/Data.scala | 32 - .../examples/transport/DataSource.scala | 56 -- .../examples/transport/QueryServer.scala | 154 ----- .../examples/transport/Transport.scala | 69 --- .../examples/transport/VelocityInspector.scala | 123 ---- .../examples/transport/generator/MockCity.scala | 88 --- .../generator/PassRecordGenerator.scala | 69 --- .../streaming/examples/transport/Data.scala | 32 + .../examples/transport/DataSource.scala | 56 ++ .../examples/transport/QueryServer.scala | 154 +++++ .../examples/transport/Transport.scala | 69 +++ .../examples/transport/VelocityInspector.scala | 123 ++++ .../examples/transport/generator/MockCity.scala | 88 +++ .../generator/PassRecordGenerator.scala | 69 +++ .../examples/transport/DataSourceSpec.scala | 45 -- .../examples/transport/TransportSpec.scala | 69 --- .../transport/generator/MockCitySpec.scala | 31 - .../generator/PassRecordGeneratorSpec.scala | 34 -- .../examples/transport/DataSourceSpec.scala | 45 ++ .../examples/transport/TransportSpec.scala | 69 +++ .../transport/generator/MockCitySpec.scala | 31 + .../generator/PassRecordGeneratorSpec.scala | 34 ++ .../streaming/examples/wordcountjava/Split.java | 54 -- .../streaming/examples/wordcountjava/Sum.java | 54 -- .../examples/wordcountjava/WordCount.java | 91 --- .../examples/wordcountjava/dsl/WordCount.java | 84 --- .../streaming/examples/wordcountjava/Split.java | 54 ++ .../streaming/examples/wordcountjava/Sum.java | 54 ++ .../examples/wordcountjava/WordCount.java | 91 +++ .../examples/wordcountjava/dsl/WordCount.java | 85 +++ .../examples/wordcountjava/WordCountSpec.scala | 59 -- .../examples/wordcountjava/WordCountSpec.scala | 59 ++ .../streaming/examples/wordcount/Split.scala | 66 --- .../streaming/examples/wordcount/Sum.scala | 66 --- .../examples/wordcount/WordCount.scala | 86 --- .../examples/wordcount/dsl/WordCount.scala | 45 -- .../streaming/examples/wordcount/Split.scala | 66 +++ .../streaming/examples/wordcount/Sum.scala | 66 +++ .../examples/wordcount/WordCount.scala | 86 +++ .../examples/wordcount/dsl/WordCount.scala | 45 ++ .../examples/wordcount/SplitSpec.scala | 62 -- .../streaming/examples/wordcount/SumSpec.scala | 56 -- .../examples/wordcount/WordCountSpec.scala | 69 --- .../examples/wordcount/SplitSpec.scala | 62 ++ .../streaming/examples/wordcount/SumSpec.scala | 56 ++ .../examples/wordcount/WordCountSpec.scala | 69 +++ .../main/scala/akka/stream/ModuleGraph.scala | 4 +- .../akka/stream/gearpump/example/Test3.scala | 2 +- .../akka/stream/gearpump/example/Test4.scala | 2 +- .../akka/stream/gearpump/example/Test6.scala | 2 +- .../stream/gearpump/example/WikipediaApp.scala | 4 +- .../stream/gearpump/graph/GraphCutter.scala | 2 +- .../akka/stream/gearpump/graph/LocalGraph.scala | 2 +- .../stream/gearpump/graph/RemoteGraph.scala | 8 +- .../akka/stream/gearpump/graph/SubGraph.scala | 2 +- .../materializer/LocalMaterializer.scala | 2 +- .../materializer/LocalMaterializerImpl.scala | 2 +- .../materializer/RemoteMaterializerImpl.scala | 10 +- .../gearpump/module/GearpumpTaskModule.scala | 8 +- .../akka/stream/gearpump/scaladsl/Api.scala | 12 +- .../akka/stream/gearpump/task/BalanceTask.scala | 6 +- .../stream/gearpump/task/BroadcastTask.scala | 6 +- .../akka/stream/gearpump/task/GraphTask.scala | 8 +- .../stream/gearpump/task/SinkBridgeTask.scala | 14 +- .../stream/gearpump/task/SourceBridgeTask.scala | 12 +- .../akka/stream/gearpump/task/UnZip2Task.scala | 6 +- .../gearpump/cluster/cgroup/CGroupResource.java | 68 --- .../gearpump/cluster/cgroup/CgroupCenter.java | 212 ------- .../gearpump/cluster/cgroup/CgroupCommon.java | 220 ------- .../cluster/cgroup/CgroupCommonOperation.java | 46 -- .../cluster/cgroup/CgroupCoreFactory.java | 42 -- .../cluster/cgroup/CgroupOperation.java | 43 -- .../io/gearpump/cluster/cgroup/CgroupUtils.java | 160 ----- .../io/gearpump/cluster/cgroup/Constants.java | 29 - .../io/gearpump/cluster/cgroup/Hierarchy.java | 106 ---- .../gearpump/cluster/cgroup/ResourceType.java | 30 - .../cluster/cgroup/core/CgroupCore.java | 25 - .../gearpump/cluster/cgroup/core/CpuCore.java | 59 -- .../gearpump/cluster/utils/SystemOperation.java | 63 -- .../gearpump/cluster/cgroup/CGroupResource.java | 68 +++ .../gearpump/cluster/cgroup/CgroupCenter.java | 212 +++++++ .../gearpump/cluster/cgroup/CgroupCommon.java | 220 +++++++ .../cluster/cgroup/CgroupCommonOperation.java | 46 ++ .../cluster/cgroup/CgroupCoreFactory.java | 42 ++ .../cluster/cgroup/CgroupOperation.java | 43 ++ .../gearpump/cluster/cgroup/CgroupUtils.java | 160 +++++ .../gearpump/cluster/cgroup/Constants.java | 29 + .../gearpump/cluster/cgroup/Hierarchy.java | 106 ++++ .../gearpump/cluster/cgroup/ResourceType.java | 30 + .../cluster/cgroup/core/CgroupCore.java | 25 + .../gearpump/cluster/cgroup/core/CpuCore.java | 59 ++ .../gearpump/cluster/utils/SystemOperation.java | 63 ++ .../gearpump/cluster/worker/CGroupManager.scala | 125 ---- .../cluster/worker/CGroupProcessLauncher.scala | 63 -- .../gearpump/cluster/worker/CGroupManager.scala | 125 ++++ .../cluster/worker/CGroupProcessLauncher.scala | 63 ++ .../storm/util/TimeCacheMapWrapper.java | 42 -- .../storm/util/TimeCacheMapWrapper.java | 42 ++ .../storm/src/main/resources/geardefault.conf | 2 +- .../experiments/storm/StormRunner.scala | 55 -- .../experiments/storm/main/GearpumpNimbus.scala | 289 --------- .../storm/main/GearpumpStormClient.scala | 74 --- .../storm/partitioner/StormPartitioner.scala | 48 -- .../processor/StormBoltOutputCollector.scala | 76 --- .../storm/processor/StormProcessor.scala | 71 --- .../storm/producer/StormProducer.scala | 91 --- .../producer/StormSpoutOutputCollector.scala | 105 ---- .../storm/topology/GearpumpStormComponent.scala | 388 ------------- .../storm/topology/GearpumpStormTopology.scala | 321 ---------- .../storm/topology/GearpumpTuple.scala | 69 --- .../experiments/storm/util/GraphBuilder.scala | 46 -- .../experiments/storm/util/Grouper.scala | 142 ----- .../experiments/storm/util/StormConstants.scala | 33 -- .../storm/util/StormOutputCollector.scala | 221 ------- .../util/StormSerializationFramework.scala | 82 --- .../experiments/storm/util/StormUtil.scala | 136 ----- .../experiments/storm/StormRunner.scala | 54 ++ .../experiments/storm/main/GearpumpNimbus.scala | 289 +++++++++ .../storm/main/GearpumpStormClient.scala | 73 +++ .../storm/partitioner/StormPartitioner.scala | 49 ++ .../processor/StormBoltOutputCollector.scala | 75 +++ .../storm/processor/StormProcessor.scala | 71 +++ .../storm/producer/StormProducer.scala | 91 +++ .../producer/StormSpoutOutputCollector.scala | 104 ++++ .../storm/topology/GearpumpStormComponent.scala | 385 ++++++++++++ .../storm/topology/GearpumpStormTopology.scala | 320 ++++++++++ .../storm/topology/GearpumpTuple.scala | 69 +++ .../experiments/storm/util/GraphBuilder.scala | 46 ++ .../experiments/storm/util/Grouper.scala | 142 +++++ .../experiments/storm/util/StormConstants.scala | 33 ++ .../storm/util/StormOutputCollector.scala | 221 +++++++ .../util/StormSerializationFramework.scala | 82 +++ .../experiments/storm/util/StormUtil.scala | 136 +++++ .../partitioner/StormPartitionerSpec.scala | 67 --- .../StormBoltOutputCollectorSpec.scala | 56 -- .../storm/processor/StormProcessorSpec.scala | 66 --- .../storm/producer/StormProducerSpec.scala | 73 --- .../StormSpoutOutputCollectorSpec.scala | 50 -- .../topology/GearpumpStormComponentSpec.scala | 125 ---- .../topology/GearpumpStormTopologySpec.scala | 104 ---- .../storm/topology/GearpumpTupleSpec.scala | 63 -- .../storm/util/GraphBuilderSpec.scala | 55 -- .../experiments/storm/util/GrouperSpec.scala | 131 ----- .../storm/util/StormOutputCollectorSpec.scala | 109 ---- .../storm/util/StormSerializerPoolSpec.scala | 63 -- .../experiments/storm/util/StormUtilSpec.scala | 157 ----- .../experiments/storm/util/TopologyUtil.scala | 41 -- .../partitioner/StormPartitionerSpec.scala | 67 +++ .../StormBoltOutputCollectorSpec.scala | 56 ++ .../storm/processor/StormProcessorSpec.scala | 65 +++ .../storm/producer/StormProducerSpec.scala | 72 +++ .../StormSpoutOutputCollectorSpec.scala | 50 ++ .../topology/GearpumpStormComponentSpec.scala | 124 ++++ .../topology/GearpumpStormTopologySpec.scala | 104 ++++ .../storm/topology/GearpumpTupleSpec.scala | 63 ++ .../storm/util/GraphBuilderSpec.scala | 55 ++ .../experiments/storm/util/GrouperSpec.scala | 131 +++++ .../storm/util/StormOutputCollectorSpec.scala | 109 ++++ .../storm/util/StormSerializerPoolSpec.scala | 63 ++ .../experiments/storm/util/StormUtilSpec.scala | 157 +++++ .../experiments/storm/util/TopologyUtil.scala | 41 ++ .../gearpump/experiments/yarn/Constants.scala | 46 -- .../experiments/yarn/appmaster/Command.scala | 126 ---- .../experiments/yarn/appmaster/UIService.scala | 94 --- .../yarn/appmaster/YarnAppMaster.scala | 385 ------------ .../yarn/client/AppMasterResolver.scala | 77 --- .../experiments/yarn/client/Client.scala | 61 -- .../experiments/yarn/client/LaunchCluster.scala | 221 ------- .../experiments/yarn/client/ManageCluster.scala | 156 ----- .../yarn/glue/ContainerLaunchContext.scala | 99 ---- .../experiments/yarn/glue/FileSystem.scala | 73 --- .../experiments/yarn/glue/NMClient.scala | 100 ---- .../experiments/yarn/glue/RMClient.scala | 128 ---- .../experiments/yarn/glue/Records.scala | 219 ------- .../experiments/yarn/glue/YarnClient.scala | 105 ---- .../experiments/yarn/glue/YarnConfig.scala | 33 -- .../experiments/yarn/glue/package.scala | 26 - .../gearpump/experiments/yarn/Constants.scala | 46 ++ .../experiments/yarn/appmaster/Command.scala | 126 ++++ .../experiments/yarn/appmaster/UIService.scala | 93 +++ .../yarn/appmaster/YarnAppMaster.scala | 385 ++++++++++++ .../yarn/client/AppMasterResolver.scala | 77 +++ .../experiments/yarn/client/Client.scala | 61 ++ .../experiments/yarn/client/LaunchCluster.scala | 220 +++++++ .../experiments/yarn/client/ManageCluster.scala | 156 +++++ .../yarn/glue/ContainerLaunchContext.scala | 99 ++++ .../experiments/yarn/glue/FileSystem.scala | 73 +++ .../experiments/yarn/glue/NMClient.scala | 99 ++++ .../experiments/yarn/glue/RMClient.scala | 127 ++++ .../experiments/yarn/glue/Records.scala | 219 +++++++ .../experiments/yarn/glue/YarnClient.scala | 103 ++++ .../experiments/yarn/glue/YarnConfig.scala | 33 ++ .../experiments/yarn/glue/package.scala | 25 + .../yarn/appmaster/CommandSpec.scala | 95 --- .../yarn/appmaster/UIServiceSpec.scala | 85 --- .../yarn/appmaster/YarnAppMasterSpec.scala | 221 ------- .../yarn/client/LaunchClusterSpec.scala | 232 -------- .../yarn/client/ManageClusterSpec.scala | 142 ----- .../yarn/appmaster/CommandSpec.scala | 95 +++ .../yarn/appmaster/UIServiceSpec.scala | 84 +++ .../yarn/appmaster/YarnAppMasterSpec.scala | 220 +++++++ .../yarn/client/LaunchClusterSpec.scala | 232 ++++++++ .../yarn/client/ManageClusterSpec.scala | 141 +++++ .../hadoop/HadoopCheckpointStore.scala | 164 ------ .../hadoop/HadoopCheckpointStoreFactory.scala | 61 -- .../streaming/hadoop/SequenceFileSink.scala | 110 ---- .../lib/HadoopCheckpointStoreReader.scala | 72 --- .../lib/HadoopCheckpointStoreWriter.scala | 40 -- .../streaming/hadoop/lib/HadoopUtil.scala | 80 --- .../lib/format/DefaultSequenceFormatter.scala | 32 - .../hadoop/lib/format/OutputFormatter.scala | 32 - .../hadoop/lib/rotation/FileSizeRotation.scala | 37 -- .../hadoop/lib/rotation/Rotation.scala | 27 - .../hadoop/HadoopCheckpointStore.scala | 164 ++++++ .../hadoop/HadoopCheckpointStoreFactory.scala | 61 ++ .../streaming/hadoop/SequenceFileSink.scala | 110 ++++ .../lib/HadoopCheckpointStoreReader.scala | 72 +++ .../lib/HadoopCheckpointStoreWriter.scala | 40 ++ .../streaming/hadoop/lib/HadoopUtil.scala | 80 +++ .../lib/format/DefaultSequenceFormatter.scala | 32 + .../hadoop/lib/format/OutputFormatter.scala | 32 + .../hadoop/lib/rotation/FileSizeRotation.scala | 37 ++ .../hadoop/lib/rotation/Rotation.scala | 27 + .../HadoopCheckpointStoreIntegrationSpec.scala | 86 --- .../lib/rotation/FileSizeRotationSpec.scala | 44 -- .../HadoopCheckpointStoreIntegrationSpec.scala | 86 +++ .../lib/rotation/FileSizeRotationSpec.scala | 44 ++ .../io/gearpump/external/hbase/HBaseSink.scala | 139 ----- .../external/hbase/dsl/HBaseDSLSink.scala | 47 -- .../gearpump/external/hbase/HBaseSink.scala | 139 +++++ .../external/hbase/dsl/HBaseDSLSink.scala | 47 ++ .../gearpump/external/hbase/HBaseSinkSpec.scala | 41 -- .../gearpump/external/hbase/HBaseSinkSpec.scala | 41 ++ .../io/gearpump/streaming/kafka/KafkaSink.scala | 80 --- .../gearpump/streaming/kafka/KafkaSource.scala | 195 ------- .../gearpump/streaming/kafka/KafkaStorage.scala | 148 ----- .../streaming/kafka/dsl/KafkaDSLSink.scala | 53 -- .../streaming/kafka/dsl/KafkaDSLUtil.scala | 88 --- .../kafka/lib/DefaultMessageDecoder.scala | 41 -- .../kafka/lib/KafkaOffsetManager.scala | 66 --- .../streaming/kafka/lib/KafkaSourceConfig.scala | 175 ------ .../streaming/kafka/lib/KafkaUtil.scala | 167 ------ .../consumer/ExponentialBackoffSleeper.scala | 55 -- .../kafka/lib/consumer/FetchThread.scala | 139 ----- .../kafka/lib/consumer/KafkaConsumer.scala | 103 ---- .../kafka/lib/consumer/KafkaMessage.scala | 38 -- .../kafka/lib/grouper/KafkaDefaultGrouper.scala | 38 -- .../kafka/lib/grouper/KafkaGrouper.scala | 30 - .../gearpump/streaming/kafka/KafkaSink.scala | 80 +++ .../gearpump/streaming/kafka/KafkaSource.scala | 195 +++++++ .../gearpump/streaming/kafka/KafkaStorage.scala | 148 +++++ .../streaming/kafka/dsl/KafkaDSLSink.scala | 53 ++ .../streaming/kafka/dsl/KafkaDSLUtil.scala | 88 +++ .../kafka/lib/DefaultMessageDecoder.scala | 41 ++ .../kafka/lib/KafkaOffsetManager.scala | 66 +++ .../streaming/kafka/lib/KafkaSourceConfig.scala | 178 ++++++ .../streaming/kafka/lib/KafkaUtil.scala | 167 ++++++ .../consumer/ExponentialBackoffSleeper.scala | 55 ++ .../kafka/lib/consumer/FetchThread.scala | 139 +++++ .../kafka/lib/consumer/KafkaConsumer.scala | 103 ++++ .../kafka/lib/consumer/KafkaMessage.scala | 38 ++ .../kafka/lib/grouper/KafkaDefaultGrouper.scala | 38 ++ .../kafka/lib/grouper/KafkaGrouper.scala | 30 + .../streaming/kafka/KafkaSinkSpec.scala | 63 -- .../streaming/kafka/KafkaSourceSpec.scala | 167 ------ .../kafka/lib/DefaultMessageDecoderSpec.scala | 44 -- .../kafka/lib/KafkaOffsetManagerSpec.scala | 117 ---- .../streaming/kafka/lib/KafkaStorageSpec.scala | 187 ------ .../streaming/kafka/lib/KafkaUtilSpec.scala | 107 ---- .../ExponentialBackoffSleeperSpec.scala | 68 --- .../kafka/lib/consumer/FetchThreadSpec.scala | 113 ---- .../kafka/lib/consumer/KafkaConsumerSpec.scala | 88 --- .../lib/grouper/KafkaDefaultGrouperSpec.scala | 42 -- .../kafka/util/KafkaServerHarness.scala | 69 --- .../streaming/kafka/util/ZookeeperHarness.scala | 43 -- .../streaming/kafka/KafkaSinkSpec.scala | 63 ++ .../streaming/kafka/KafkaSourceSpec.scala | 167 ++++++ .../kafka/lib/DefaultMessageDecoderSpec.scala | 44 ++ .../kafka/lib/KafkaOffsetManagerSpec.scala | 117 ++++ .../streaming/kafka/lib/KafkaStorageSpec.scala | 187 ++++++ .../streaming/kafka/lib/KafkaUtilSpec.scala | 107 ++++ .../ExponentialBackoffSleeperSpec.scala | 68 +++ .../kafka/lib/consumer/FetchThreadSpec.scala | 113 ++++ .../kafka/lib/consumer/KafkaConsumerSpec.scala | 88 +++ .../lib/grouper/KafkaDefaultGrouperSpec.scala | 42 ++ .../kafka/util/KafkaServerHarness.scala | 69 +++ .../streaming/kafka/util/ZookeeperHarness.scala | 43 ++ .../streaming/monoid/AlgebirdMonoid.scala | 37 -- .../streaming/monoid/AlgebirdMonoid.scala | 36 ++ .../streaming/serializer/ChillSerializer.scala | 33 -- .../streaming/serializer/ChillSerializer.scala | 32 + integrationtest/README.md | 6 +- .../integrationtest/MiniClusterProvider.scala | 4 +- .../gearpump/integrationtest/TestSpecBase.scala | 8 +- .../checklist/CommandLineSpec.scala | 6 +- .../checklist/ConnectorKafkaSpec.scala | 10 +- .../checklist/DynamicDagSpec.scala | 18 +- .../integrationtest/checklist/ExampleSpec.scala | 12 +- .../checklist/MessageDeliverySpec.scala | 12 +- .../checklist/RestServiceSpec.scala | 10 +- .../checklist/StabilitySpec.scala | 10 +- .../checklist/StormCompatibilitySpec.scala | 12 +- .../suites/StandaloneModeSuite.scala | 8 +- .../io/gearpump/integrationtest/Docker.scala | 2 +- .../io/gearpump/integrationtest/ShellExec.scala | 2 +- .../io/gearpump/integrationtest/Util.scala | 2 +- .../integrationtest/hadoop/HadoopCluster.scala | 4 +- .../integrationtest/kafka/KafkaCluster.scala | 6 +- .../kafka/NumericalDataProducer.scala | 4 +- .../integrationtest/kafka/ResultVerifier.scala | 2 +- .../kafka/SimpleKafkaReader.scala | 4 +- .../minicluster/BaseContainer.scala | 4 +- .../minicluster/CommandLineClient.scala | 6 +- .../minicluster/MiniCluster.scala | 4 +- .../minicluster/RestClient.scala | 34 +- .../integrationtest/storm/StormClient.scala | 6 +- .../integrationtest/storm/Adaptor.scala | 2 +- .../storm/Storm010KafkaTopology.scala | 4 +- .../integrationtest/storm/Adaptor.scala | 2 +- .../storm/Storm09KafkaTopology.scala | 4 +- project/Build.scala | 7 +- project/BuildExample.scala | 24 +- project/Pack.scala | 14 +- services/README.md | 2 +- services/dashboard/dashboard.js | 2 +- services/dashboard/services/models/dag.js | 2 +- services/dashboard/services/models/metrics.js | 8 +- .../services/models/metrics_provider.js | 2 +- services/dashboard/services/models/models.js | 6 +- .../services/models/streamingapp_dag.js | 2 +- .../models/streamingapp_metrics_provider.js | 2 +- services/dashboard/services/restapi.js | 2 +- .../io/gearpump/dashboard/DashboardApp.scala | 28 - .../gearpump/dashboard/DashboardApp.scala | 28 + .../io/gearpump/services/AdminService.scala | 53 -- .../io/gearpump/services/AppMasterService.scala | 226 -------- .../io/gearpump/services/BasicService.scala | 71 --- .../io/gearpump/services/MasterService.scala | 350 ----------- .../io/gearpump/services/RestServices.scala | 105 ---- .../io/gearpump/services/SecurityService.scala | 290 --------- .../io/gearpump/services/StaticService.scala | 70 --- .../gearpump/services/SupervisorService.scala | 116 ---- .../io/gearpump/services/WorkerService.scala | 79 --- .../io/gearpump/services/main/Services.scala | 148 ----- .../scala/io/gearpump/services/package.scala | 22 - .../security/oauth2/OAuth2Authenticator.scala | 152 ----- .../oauth2/impl/BaseOAuth2Authenticator.scala | 231 -------- .../CloudFoundryUAAOAuth2Authenticator.scala | 244 -------- .../oauth2/impl/GoogleOAuth2Authenticator.scala | 106 ---- .../io/gearpump/services/util/UpickleUtil.scala | 49 -- .../apache/gearpump/services/AdminService.scala | 53 ++ .../gearpump/services/AppMasterService.scala | 226 ++++++++ .../apache/gearpump/services/BasicService.scala | 71 +++ .../gearpump/services/MasterService.scala | 350 +++++++++++ .../apache/gearpump/services/RestServices.scala | 105 ++++ .../gearpump/services/SecurityService.scala | 290 +++++++++ .../gearpump/services/StaticService.scala | 70 +++ .../gearpump/services/SupervisorService.scala | 116 ++++ .../gearpump/services/WorkerService.scala | 79 +++ .../gearpump/services/main/Services.scala | 148 +++++ .../org/apache/gearpump/services/package.scala | 22 + .../security/oauth2/OAuth2Authenticator.scala | 152 +++++ .../oauth2/impl/BaseOAuth2Authenticator.scala | 232 ++++++++ .../CloudFoundryUAAOAuth2Authenticator.scala | 244 ++++++++ .../oauth2/impl/GoogleOAuth2Authenticator.scala | 108 ++++ .../gearpump/services/util/UpickleUtil.scala | 49 ++ .../io/gearpump/services/AdminServiceSpec.scala | 53 -- .../services/AppMasterServiceSpec.scala | 167 ------ .../gearpump/services/MasterServiceSpec.scala | 217 ------- .../gearpump/services/SecurityServiceSpec.scala | 162 ------ .../gearpump/services/StaticServiceSpec.scala | 63 -- .../gearpump/services/WorkerServiceSpec.scala | 121 ---- ...CloudFoundryUAAOAuth2AuthenticatorSpec.scala | 139 ----- .../oauth2/GoogleOAuth2AuthenticatorSpec.scala | 163 ------ .../security/oauth2/MockOAuth2Server.scala | 65 --- .../io/gearpump/services/util/UpickleSpec.scala | 67 --- .../gearpump/services/AdminServiceSpec.scala | 53 ++ .../services/AppMasterServiceSpec.scala | 167 ++++++ .../gearpump/services/MasterServiceSpec.scala | 217 +++++++ .../gearpump/services/SecurityServiceSpec.scala | 162 ++++++ .../gearpump/services/StaticServiceSpec.scala | 63 ++ .../gearpump/services/WorkerServiceSpec.scala | 121 ++++ ...CloudFoundryUAAOAuth2AuthenticatorSpec.scala | 140 +++++ .../oauth2/GoogleOAuth2AuthenticatorSpec.scala | 163 ++++++ .../security/oauth2/MockOAuth2Server.scala | 65 +++ .../gearpump/services/util/UpickleSpec.scala | 67 +++ .../io/gearpump/streaming/javaapi/Graph.java | 35 -- .../gearpump/streaming/javaapi/Processor.java | 141 ----- .../streaming/javaapi/StreamApplication.java | 61 -- .../io/gearpump/streaming/javaapi/Task.java | 58 -- .../javaapi/dsl/functions/FilterFunction.java | 30 - .../javaapi/dsl/functions/FlatMapFunction.java | 32 - .../javaapi/dsl/functions/GroupByFunction.java | 31 - .../javaapi/dsl/functions/MapFunction.java | 31 - .../javaapi/dsl/functions/ReduceFunction.java | 30 - .../gearpump/streaming/javaapi/Graph.java | 35 ++ .../gearpump/streaming/javaapi/Processor.java | 141 +++++ .../streaming/javaapi/StreamApplication.java | 61 ++ .../apache/gearpump/streaming/javaapi/Task.java | 58 ++ .../javaapi/dsl/functions/FilterFunction.java | 30 + .../javaapi/dsl/functions/FlatMapFunction.java | 32 + .../javaapi/dsl/functions/GroupByFunction.java | 31 + .../javaapi/dsl/functions/MapFunction.java | 31 + .../javaapi/dsl/functions/ReduceFunction.java | 30 + streaming/src/main/resources/geardefault.conf | 6 +- .../io/gearpump/streaming/ClusterMessage.scala | 85 --- .../scala/io/gearpump/streaming/Constants.scala | 37 -- .../main/scala/io/gearpump/streaming/DAG.scala | 62 -- .../gearpump/streaming/MessageSerializer.scala | 108 ---- .../gearpump/streaming/StreamApplication.scala | 170 ------ .../streaming/appmaster/AppMaster.scala | 362 ------------ .../streaming/appmaster/ClockService.scala | 447 -------------- .../streaming/appmaster/DagManager.scala | 182 ------ .../streaming/appmaster/ExecutorManager.scala | 209 ------- .../streaming/appmaster/JarScheduler.scala | 176 ------ .../appmaster/StreamAppMasterSummary.scala | 61 -- .../streaming/appmaster/TaskLocator.scala | 99 ---- .../streaming/appmaster/TaskManager.scala | 497 ---------------- .../streaming/appmaster/TaskRegistry.scala | 143 ----- .../streaming/appmaster/TaskSchedulerImpl.scala | 178 ------ .../io/gearpump/streaming/dsl/Stream.scala | 230 -------- .../io/gearpump/streaming/dsl/StreamApp.scala | 124 ---- .../streaming/dsl/javaapi/JavaStream.scala | 77 --- .../streaming/dsl/javaapi/JavaStreamApp.scala | 47 -- .../scala/io/gearpump/streaming/dsl/op/OP.scala | 109 ---- .../dsl/partitioner/GroupbyPartitioner.scala | 46 -- .../streaming/dsl/plan/OpTranslator.scala | 296 ---------- .../gearpump/streaming/dsl/plan/Planner.scala | 96 --- .../gearpump/streaming/executor/Executor.scala | 476 --------------- .../executor/ExecutorRestartPolicy.scala | 64 -- .../streaming/executor/TaskLauncher.scala | 83 --- .../streaming/metrics/ProcessorAggregator.scala | 301 ---------- .../metrics/TaskFilterAggregator.scala | 157 ----- .../scala/io/gearpump/streaming/package.scala | 25 - .../io/gearpump/streaming/sink/DataSink.scala | 65 --- .../streaming/sink/DataSinkProcessor.scala | 49 -- .../gearpump/streaming/sink/DataSinkTask.scala | 50 -- .../gearpump/streaming/source/DataSource.scala | 70 --- .../streaming/source/DataSourceConfig.scala | 25 - .../streaming/source/DataSourceProcessor.scala | 49 -- .../streaming/source/DataSourceTask.scala | 65 --- .../source/DefaultTimeStampFilter.scala | 31 - .../gearpump/streaming/state/api/Monoid.scala | 28 - .../streaming/state/api/MonoidState.scala | 52 -- .../streaming/state/api/PersistentState.scala | 61 -- .../streaming/state/api/PersistentTask.scala | 114 ---- .../streaming/state/api/Serializer.scala | 26 - .../state/impl/CheckpointManager.scala | 64 -- .../state/impl/InMemoryCheckpointStore.scala | 50 -- .../streaming/state/impl/NonWindowState.scala | 53 -- .../state/impl/PersistentStateConfig.scala | 28 - .../gearpump/streaming/state/impl/Window.scala | 54 -- .../streaming/state/impl/WindowConfig.scala | 25 - .../streaming/state/impl/WindowState.scala | 165 ------ .../streaming/storage/AppDataStore.scala | 30 - .../storage/InMemoryAppStoreOnMaster.scala | 48 -- .../streaming/task/ExpressTransport.scala | 75 --- .../streaming/task/SerializedMessage.scala | 42 -- .../streaming/task/SerializerResolver.scala | 47 -- .../io/gearpump/streaming/task/StartTime.scala | 24 - .../task/StreamingTransportSerializer.scala | 68 --- .../io/gearpump/streaming/task/Subscriber.scala | 58 -- .../gearpump/streaming/task/Subscription.scala | 235 -------- .../scala/io/gearpump/streaming/task/Task.scala | 189 ------ .../io/gearpump/streaming/task/TaskActor.scala | 397 ------------- .../streaming/task/TaskContextData.scala | 32 - .../streaming/task/TaskControlMessage.scala | 72 --- .../io/gearpump/streaming/task/TaskId.scala | 28 - .../streaming/task/TaskMessageSerializer.scala | 28 - .../io/gearpump/streaming/task/TaskUtil.scala | 33 -- .../gearpump/streaming/task/TaskWrapper.scala | 133 ----- .../transaction/api/CheckpointStore.scala | 42 -- .../transaction/api/MessageDecoder.scala | 29 - .../transaction/api/OffsetManager.scala | 44 -- .../transaction/api/OffsetStorage.scala | 66 --- .../transaction/api/TimeReplayableSource.scala | 30 - .../transaction/api/TimeStampFilter.scala | 29 - .../gearpump/streaming/util/ActorPathUtil.scala | 41 -- .../gearpump/streaming/ClusterMessage.scala | 85 +++ .../apache/gearpump/streaming/Constants.scala | 36 ++ .../org/apache/gearpump/streaming/DAG.scala | 62 ++ .../gearpump/streaming/MessageSerializer.scala | 108 ++++ .../gearpump/streaming/StreamApplication.scala | 171 ++++++ .../streaming/appmaster/AppMaster.scala | 362 ++++++++++++ .../streaming/appmaster/ClockService.scala | 447 ++++++++++++++ .../streaming/appmaster/DagManager.scala | 181 ++++++ .../streaming/appmaster/ExecutorManager.scala | 209 +++++++ .../streaming/appmaster/JarScheduler.scala | 175 ++++++ .../appmaster/StreamAppMasterSummary.scala | 61 ++ .../streaming/appmaster/TaskLocator.scala | 97 ++++ .../streaming/appmaster/TaskManager.scala | 496 ++++++++++++++++ .../streaming/appmaster/TaskRegistry.scala | 142 +++++ .../streaming/appmaster/TaskSchedulerImpl.scala | 177 ++++++ .../apache/gearpump/streaming/dsl/Stream.scala | 229 ++++++++ .../gearpump/streaming/dsl/StreamApp.scala | 123 ++++ .../streaming/dsl/javaapi/JavaStream.scala | 77 +++ .../streaming/dsl/javaapi/JavaStreamApp.scala | 47 ++ .../apache/gearpump/streaming/dsl/op/OP.scala | 109 ++++ .../dsl/partitioner/GroupbyPartitioner.scala | 46 ++ .../streaming/dsl/plan/OpTranslator.scala | 296 ++++++++++ .../gearpump/streaming/dsl/plan/Planner.scala | 96 +++ .../gearpump/streaming/executor/Executor.scala | 476 +++++++++++++++ .../executor/ExecutorRestartPolicy.scala | 64 ++ .../streaming/executor/TaskLauncher.scala | 83 +++ .../streaming/metrics/ProcessorAggregator.scala | 301 ++++++++++ .../metrics/TaskFilterAggregator.scala | 157 +++++ .../org/apache/gearpump/streaming/package.scala | 25 + .../gearpump/streaming/sink/DataSink.scala | 65 +++ .../streaming/sink/DataSinkProcessor.scala | 49 ++ .../gearpump/streaming/sink/DataSinkTask.scala | 50 ++ .../gearpump/streaming/source/DataSource.scala | 70 +++ .../streaming/source/DataSourceConfig.scala | 25 + .../streaming/source/DataSourceProcessor.scala | 49 ++ .../streaming/source/DataSourceTask.scala | 66 +++ .../source/DefaultTimeStampFilter.scala | 31 + .../gearpump/streaming/state/api/Monoid.scala | 28 + .../streaming/state/api/MonoidState.scala | 52 ++ .../streaming/state/api/PersistentState.scala | 61 ++ .../streaming/state/api/PersistentTask.scala | 114 ++++ .../streaming/state/api/Serializer.scala | 26 + .../state/impl/CheckpointManager.scala | 64 ++ .../state/impl/InMemoryCheckpointStore.scala | 50 ++ .../streaming/state/impl/NonWindowState.scala | 53 ++ .../state/impl/PersistentStateConfig.scala | 28 + .../gearpump/streaming/state/impl/Window.scala | 54 ++ .../streaming/state/impl/WindowConfig.scala | 25 + .../streaming/state/impl/WindowState.scala | 165 ++++++ .../streaming/storage/AppDataStore.scala | 30 + .../storage/InMemoryAppStoreOnMaster.scala | 48 ++ .../streaming/task/ExpressTransport.scala | 75 +++ .../streaming/task/SerializedMessage.scala | 42 ++ .../streaming/task/SerializerResolver.scala | 47 ++ .../gearpump/streaming/task/StartTime.scala | 24 + .../task/StreamingTransportSerializer.scala | 68 +++ .../gearpump/streaming/task/Subscriber.scala | 58 ++ .../gearpump/streaming/task/Subscription.scala | 235 ++++++++ .../apache/gearpump/streaming/task/Task.scala | 189 ++++++ .../gearpump/streaming/task/TaskActor.scala | 397 +++++++++++++ .../streaming/task/TaskContextData.scala | 32 + .../streaming/task/TaskControlMessage.scala | 72 +++ .../apache/gearpump/streaming/task/TaskId.scala | 28 + .../streaming/task/TaskMessageSerializer.scala | 28 + .../gearpump/streaming/task/TaskUtil.scala | 33 ++ .../gearpump/streaming/task/TaskWrapper.scala | 134 +++++ .../transaction/api/CheckpointStore.scala | 42 ++ .../transaction/api/MessageDecoder.scala | 29 + .../transaction/api/OffsetManager.scala | 44 ++ .../transaction/api/OffsetStorage.scala | 66 +++ .../transaction/api/TimeReplayableSource.scala | 30 + .../transaction/api/TimeStampFilter.scala | 29 + .../gearpump/streaming/util/ActorPathUtil.scala | 41 ++ .../scala/io/gearpump/streaming/DAGSpec.scala | 45 -- .../streaming/MessageSerializerSpec.scala | 78 --- .../scala/io/gearpump/streaming/MockUtil.scala | 48 -- .../gearpump/streaming/StreamingTestUtil.scala | 49 -- .../streaming/appmaster/AppMasterSpec.scala | 238 -------- .../streaming/appmaster/ClockServiceSpec.scala | 241 -------- .../streaming/appmaster/DagManagerSpec.scala | 108 ---- .../appmaster/ExecutorManagerSpec.scala | 151 ----- .../appmaster/ExecutorRestartPolicySpec.scala | 47 -- .../appmaster/HistoryMetricsServiceSpec.scala | 197 ------- .../streaming/appmaster/JarSchedulerSpec.scala | 96 --- .../streaming/appmaster/TaskLocatorSpec.scala | 35 -- .../streaming/appmaster/TaskManagerSpec.scala | 276 --------- .../streaming/appmaster/TaskRegistrySpec.scala | 64 -- .../streaming/appmaster/TaskSchedulerSpec.scala | 130 ----- .../gearpump/streaming/dsl/StreamAppSpec.scala | 94 --- .../io/gearpump/streaming/dsl/StreamSpec.scala | 126 ---- .../partitioner/GroupByPartitionerSpec.scala | 44 -- .../streaming/dsl/plan/OpTranslatorSpec.scala | 143 ----- .../streaming/executor/ExecutorSpec.scala | 110 ---- .../executor/TaskArgumentStoreSpec.scala | 45 -- .../streaming/executor/TaskLauncherSpec.scala | 78 --- .../metrics/ProcessorAggregatorSpec.scala | 261 --------- .../metrics/TaskFilterAggregatorSpec.scala | 80 --- .../source/DefaultTimeStampFilterSpec.scala | 48 -- .../state/impl/CheckpointManagerSpec.scala | 89 --- .../impl/InMemoryCheckpointStoreSpec.scala | 37 -- .../state/impl/NonWindowStateSpec.scala | 132 ----- .../streaming/state/impl/WindowSpec.scala | 61 -- .../streaming/state/impl/WindowStateSpec.scala | 239 -------- .../storage/InMemoryAppStoreOnMasterSpec.scala | 62 -- .../streaming/task/SubscriberSpec.scala | 51 -- .../streaming/task/SubscriptionSpec.scala | 142 ----- .../gearpump/streaming/task/TaskActorSpec.scala | 123 ---- .../org/apache/gearpump/streaming/DAGSpec.scala | 44 ++ .../streaming/MessageSerializerSpec.scala | 78 +++ .../apache/gearpump/streaming/MockUtil.scala | 47 ++ .../gearpump/streaming/StreamingTestUtil.scala | 48 ++ .../streaming/appmaster/AppMasterSpec.scala | 237 ++++++++ .../streaming/appmaster/ClockServiceSpec.scala | 240 ++++++++ .../streaming/appmaster/DagManagerSpec.scala | 107 ++++ .../appmaster/ExecutorManagerSpec.scala | 150 +++++ .../appmaster/ExecutorRestartPolicySpec.scala | 46 ++ .../appmaster/HistoryMetricsServiceSpec.scala | 196 +++++++ .../streaming/appmaster/JarSchedulerSpec.scala | 95 +++ .../streaming/appmaster/TaskLocatorSpec.scala | 34 ++ .../streaming/appmaster/TaskManagerSpec.scala | 275 +++++++++ .../streaming/appmaster/TaskRegistrySpec.scala | 63 ++ .../streaming/appmaster/TaskSchedulerSpec.scala | 129 ++++ .../gearpump/streaming/dsl/StreamAppSpec.scala | 93 +++ .../gearpump/streaming/dsl/StreamSpec.scala | 125 ++++ .../partitioner/GroupByPartitionerSpec.scala | 44 ++ .../streaming/dsl/plan/OpTranslatorSpec.scala | 143 +++++ .../streaming/executor/ExecutorSpec.scala | 110 ++++ .../executor/TaskArgumentStoreSpec.scala | 45 ++ .../streaming/executor/TaskLauncherSpec.scala | 78 +++ .../metrics/ProcessorAggregatorSpec.scala | 261 +++++++++ .../metrics/TaskFilterAggregatorSpec.scala | 80 +++ .../source/DefaultTimeStampFilterSpec.scala | 48 ++ .../state/impl/CheckpointManagerSpec.scala | 89 +++ .../impl/InMemoryCheckpointStoreSpec.scala | 37 ++ .../state/impl/NonWindowStateSpec.scala | 132 +++++ .../streaming/state/impl/WindowSpec.scala | 61 ++ .../streaming/state/impl/WindowStateSpec.scala | 239 ++++++++ .../storage/InMemoryAppStoreOnMasterSpec.scala | 62 ++ .../streaming/task/SubscriberSpec.scala | 51 ++ .../streaming/task/SubscriptionSpec.scala | 142 +++++ .../gearpump/streaming/task/TaskActorSpec.scala | 123 ++++ 1049 files changed, 46925 insertions(+), 46957 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/83b36ef7/conf/gear.conf ---------------------------------------------------------------------- diff --git a/conf/gear.conf b/conf/gear.conf index b21f59a..30eacd2 100644 --- a/conf/gear.conf +++ b/conf/gear.conf @@ -31,11 +31,11 @@ gearpump { worker.slots = 1000 ## The class responsable for launching the executor process. - ## User can switch to "io.gearpump.cluster.worker.CGroupProcessLauncher" to enable CGroup support. - worker.executor-process-launcher = "io.gearpump.cluster.worker.DefaultExecutorProcessLauncher" + ## User can switch to "org.apache.gearpump.cluster.worker.CGroupProcessLauncher" to enable CGroup support. + worker.executor-process-launcher = "org.apache.gearpump.cluster.worker.DefaultExecutorProcessLauncher" ## To enable worker use cgroup to make resource isolation, - ## set gearpump.worker.executor-process-launcher = "io.gearpump.cluster.worker.CGroupProcessLauncher" + ## set gearpump.worker.executor-process-launcher = "org.apache.gearpump.cluster.worker.CGroupProcessLauncher" ## ## Before enable it, you should also make sure: ## 1. Linux version (>= 2.6.18) @@ -144,7 +144,7 @@ gearpump { ### Whitelist for Metrics Aggregator class. ### See class [[MetricsAggregator]] for more information. metrics-aggregator-class { - ## Format io.gearpump.KeyFullClassName = "" + ## Format org.apache.gearpump.KeyFullClassName = "" } } } @@ -171,9 +171,9 @@ gearpump { ### Gearpump has built-in serialization framework using Kryo. ### User are allowed to use a different serialization framework, like Protobuf - ### See [io.gearpump.serializer.FastKryoSerializationFramework] to find how + ### See [org.apache.gearpump.serializer.FastKryoSerializationFramework] to find how ### a custom serialization framework can be defined. - serialization-framework = "io.gearpump.serializer.FastKryoSerializationFramework" + serialization-framework = "org.apache.gearpump.serializer.FastKryoSerializationFramework" ### Define where the submitted jar file will be stored at @@ -190,7 +190,7 @@ gearpump { ### If you don't know what is this about, don't change it ######################### scheduling { - scheduler-class = "io.gearpump.cluster.scheduler.PriorityScheduler" + scheduler-class = "org.apache.gearpump.cluster.scheduler.PriorityScheduler" } ############################################# @@ -388,15 +388,15 @@ gearpump-ui { ## authentication channel like OAuth2. ## ## User can replace this with a custom User-Password based authenticator, - ## which implements interface io.gearpump.security.Authenticator + ## which implements interface org.apache.gearpump.security.Authenticator ## - authenticator = "io.gearpump.security.ConfigFileBasedAuthenticator" + authenticator = "org.apache.gearpump.security.ConfigFileBasedAuthenticator" - ## Configuration options for authenticator io.gearpump.security.ConfigFileBasedAuthenticator + ## Configuration options for authenticator org.apache.gearpump.security.ConfigFileBasedAuthenticator config-file-based-authenticator = { ## Format: username = "password_hash_value" ## password_hash_value can be generated by running shell tool: - ## bin/gear io.gearpump.security.PasswordUtil -password <your raw password> + ## bin/gear org.apache.gearpump.security.PasswordUtil -password <your raw password> ## Admin users have super permission to do everything admins = { @@ -447,7 +447,7 @@ gearpump-ui { ## For steps to enable OAuth2 Authentication on Google, please view docs/deployment-ui-authentication.md ## "google" { - "class" = "io.gearpump.services.security.oauth2.impl.GoogleOAuth2Authenticator" + "class" = "org.apache.gearpump.services.security.oauth2.impl.GoogleOAuth2Authenticator" ## Please replace "127.0.0.1:8090" with your address of UI service. "callback" = "http://127.0.0.1:8090/login/oauth2/google/callback"; @@ -478,7 +478,7 @@ gearpump-ui { ## For steps to enable OAuth2 Authentication for UAA, please view docs/deployment-ui-authentication.md ## "cloudfoundryuaa" { - "class" = "io.gearpump.services.security.oauth2.impl.CloudFoundryUAAOAuth2Authenticator" + "class" = "org.apache.gearpump.services.security.oauth2.impl.CloudFoundryUAAOAuth2Authenticator" ## Please replace "127.0.0.1:8090" with your address of UI service. "callback" = "http://127.0.0.1:8090/login/oauth2/cloudfoundryuaa/callback"; @@ -516,7 +516,7 @@ gearpump-ui { ## Define how to do additional authorization check. The class should implement ## interface CloudFoundryUAAOAuth2Authenticator.AdditionalAuthenticator additional-authenticator = { - "class" = "io.gearpump.services.security.oauth2.impl.CloudFoundryUAAOAuth2Authenticator$OrganizationAccessChecker" + "class" = "org.apache.gearpump.services.security.oauth2.impl.CloudFoundryUAAOAuth2Authenticator$OrganizationAccessChecker" ## Please fill the Cloud Foundry API endpoint and organization GUID "organization-url" = "http://<cloud foundry api endpoint>/v2/organizations/<organization-guid>" http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/83b36ef7/conf/log4j.properties ---------------------------------------------------------------------- diff --git a/conf/log4j.properties b/conf/log4j.properties index 7175778..32638e4 100644 --- a/conf/log4j.properties +++ b/conf/log4j.properties @@ -77,7 +77,7 @@ log4j.threshhold=ALL # # Rolling File Appender # -log4j.appender.RollingFileAppender=io.gearpump.util.RecreateRollingFileAppender +log4j.appender.RollingFileAppender=org.apache.gearpump.util.RecreateRollingFileAppender log4j.appender.RollingFileAppender.File=${gearpump.log.dir}/${gearpump.log.file} log4j.appender.RollingFileAppender.checkFileInterval=60 log4j.appender.RollingFileAppender.layout=org.apache.log4j.PatternLayout @@ -98,7 +98,7 @@ log4j.appender.console.layout.ConversionPattern=[%c{1}] %m%n # # Application Log Appender # -log4j.appender.ApplicationLogAppender=io.gearpump.util.RecreateRollingFileAppender +log4j.appender.ApplicationLogAppender=org.apache.gearpump.util.RecreateRollingFileAppender log4j.appender.ApplicationLogAppender.File=${gearpump.application.log.dir}/${gearpump.application.log.file} log4j.appender.ApplicationLogAppender.checkFileInterval=60 http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/83b36ef7/core/src/main/java/io/gearpump/transport/netty/ITransportMessageSerializer.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/io/gearpump/transport/netty/ITransportMessageSerializer.java b/core/src/main/java/io/gearpump/transport/netty/ITransportMessageSerializer.java deleted file mode 100644 index c01135e..0000000 --- a/core/src/main/java/io/gearpump/transport/netty/ITransportMessageSerializer.java +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.gearpump.transport.netty; - -import java.io.DataInput; -import java.io.DataOutput; - -public interface ITransportMessageSerializer { - - int getLength(Object obj); - - void serialize(DataOutput dataOutput, Object transportMessage); - - Object deserialize(DataInput dataInput, int length); -} http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/83b36ef7/core/src/main/java/io/gearpump/transport/netty/MessageBatch.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/io/gearpump/transport/netty/MessageBatch.java b/core/src/main/java/io/gearpump/transport/netty/MessageBatch.java deleted file mode 100644 index f556161..0000000 --- a/core/src/main/java/io/gearpump/transport/netty/MessageBatch.java +++ /dev/null @@ -1,152 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.gearpump.transport.netty; - -import io.gearpump.google.common.io.Closeables; -import org.jboss.netty.buffer.ChannelBuffer; -import org.jboss.netty.buffer.ChannelBufferOutputStream; -import org.jboss.netty.buffer.ChannelBuffers; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; - -/** - * Netty message on the wire is wrapped as MessageBatch - */ -public class MessageBatch { - private static final Logger log = LoggerFactory.getLogger(MessageBatch.class); - - private int buffer_size; - private List<TaskMessage> messages; - private int encoded_length; - private ITransportMessageSerializer serializer; - - MessageBatch(int buffer_size, ITransportMessageSerializer serializer) { - this.buffer_size = buffer_size; - messages = new ArrayList<TaskMessage>(); - encoded_length = 0; - this.serializer = serializer; - } - - void add(TaskMessage taskMessage) { - if (taskMessage == null) { - throw new RuntimeException("null object forbidden in a message batch"); - } - - messages.add(taskMessage); - encoded_length += msgEncodeLength(taskMessage); - } - - TaskMessage get(int index) { - return messages.get(index); - } - - /** - * try to add a TaskMessage to a batch - * - * @param taskMsg - {@link io.gearpump.transport.netty.TaskMessage} - * @return false if the msg could not be added due to buffer size limit; true otherwise - */ - boolean tryAdd(TaskMessage taskMsg) { - if ((encoded_length + msgEncodeLength(taskMsg)) <= buffer_size) { - add(taskMsg); - return true; - } - return false; - } - - private int msgEncodeLength(TaskMessage taskMsg) { - int size = 0; - if (taskMsg != null) { - size = 24; //sessionId(INT) + sourceTask(LONG) + targetTask(LONG) + messageLength(INT) - if (taskMsg.message() != null) { - size += serializer.getLength(taskMsg.message()); - } - } - return size; - } - - /** - * @return true, if allowed buffer is Full - */ - boolean isFull() { - return encoded_length >= buffer_size; - } - - /** - * @return true, if no messages in this batch - */ - boolean isEmpty() { - return messages.isEmpty(); - } - - /** - * @return number of messages available in this batch - */ - int size() { - return messages.size(); - } - - /** - * create a buffer containing the encoding of this batch - */ - ChannelBuffer buffer() throws IOException { - ChannelBufferOutputStream bout = - new ChannelBufferOutputStream(ChannelBuffers.directBuffer(encoded_length)); - - try { - for (TaskMessage msg : messages) { - writeTaskMessage(bout, msg); - } - return bout.buffer(); - } catch (IOException e) { - log.error("Error while writing Tasks to Channel Buffer - {}", e.getMessage()); - } finally { - Closeables.close(bout, false); - } - return null; - } - - /** - * write a TaskMessage into a stream - * <p> - * Each TaskMessage is encoded as: - * sessionId ... int(4) - * source task ... Long(8) - * target task ... long(8) - * len ... int(4) - * payload ... byte[] * - */ - private void writeTaskMessage(ChannelBufferOutputStream bout, - TaskMessage message) throws IOException { - long target_id = message.targetTask(); - long source_id = message.sourceTask(); - int sessionId = message.sessionId(); - int msgLength = serializer.getLength(message.message()); - - bout.writeInt(sessionId); - bout.writeLong(target_id); - bout.writeLong(source_id); - bout.writeInt(msgLength); - serializer.serialize(bout, message.message()); - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/83b36ef7/core/src/main/java/io/gearpump/transport/netty/MessageDecoder.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/io/gearpump/transport/netty/MessageDecoder.java b/core/src/main/java/io/gearpump/transport/netty/MessageDecoder.java deleted file mode 100644 index 7566014..0000000 --- a/core/src/main/java/io/gearpump/transport/netty/MessageDecoder.java +++ /dev/null @@ -1,104 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.gearpump.transport.netty; - -import org.jboss.netty.buffer.ChannelBuffer; -import org.jboss.netty.channel.Channel; -import org.jboss.netty.channel.ChannelHandlerContext; -import org.jboss.netty.handler.codec.frame.FrameDecoder; - -import java.util.ArrayList; -import java.util.List; - -public class MessageDecoder extends FrameDecoder { - private ITransportMessageSerializer serializer; - private WrappedChannelBuffer dataInput = new WrappedChannelBuffer(); - - public MessageDecoder(ITransportMessageSerializer serializer) { - this.serializer = serializer; - } - - /* - * Each TaskMessage is encoded as: - * sessionId ... int(4) - * source task ... long(8) - * target task ... long(8) - * len ... int(4) - * payload ... byte[] * - */ - protected List<TaskMessage> decode(ChannelHandlerContext ctx, Channel channel, - ChannelBuffer buf) { - this.dataInput.setChannelBuffer(buf); - - final int SESION_LENGTH = 4; //int - final int SOURCE_TASK_LENGTH = 8; //long - final int TARGET_TASK_LENGTH = 8; //long - final int MESSAGE_LENGTH = 4; //int - final int HEADER_LENGTH = SESION_LENGTH + SOURCE_TASK_LENGTH + TARGET_TASK_LENGTH + MESSAGE_LENGTH; - - // Make sure that we have received at least a short message - long available = buf.readableBytes(); - if (available < HEADER_LENGTH) { - //need more data - return null; - } - - List<TaskMessage> taskMessageList = new ArrayList<TaskMessage>(); - - // Use while loop, try to decode as more messages as possible in single call - while (available >= HEADER_LENGTH) { - - // Mark the current buffer position before reading task/len field - // because the whole frame might not be in the buffer yet. - // We will reset the buffer position to the marked position if - // there's not enough bytes in the buffer. - buf.markReaderIndex(); - - int sessionId = buf.readInt(); - long targetTask = buf.readLong(); - long sourceTask = buf.readLong(); - // Read the length field. - int length = buf.readInt(); - - available -= HEADER_LENGTH; - - if (length <= 0) { - taskMessageList.add(new TaskMessage(sessionId, targetTask, sourceTask, null)); - break; - } - - // Make sure if there's enough bytes in the buffer. - if (available < length) { - // The whole bytes were not received yet - return null. - buf.resetReaderIndex(); - break; - } - available -= length; - - // There's enough bytes in the buffer. Read it. - Object message = serializer.deserialize(dataInput, length); - - // Successfully decoded a frame. - // Return a TaskMessage object - taskMessageList.add(new TaskMessage(sessionId, targetTask, sourceTask, message)); - } - - return taskMessageList.size() == 0 ? null : taskMessageList; - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/83b36ef7/core/src/main/java/io/gearpump/transport/netty/MessageEncoder.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/io/gearpump/transport/netty/MessageEncoder.java b/core/src/main/java/io/gearpump/transport/netty/MessageEncoder.java deleted file mode 100644 index 674e96f..0000000 --- a/core/src/main/java/io/gearpump/transport/netty/MessageEncoder.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.gearpump.transport.netty; - -import org.jboss.netty.channel.Channel; -import org.jboss.netty.channel.ChannelHandlerContext; -import org.jboss.netty.handler.codec.oneone.OneToOneEncoder; - -public class MessageEncoder extends OneToOneEncoder { - @Override - protected Object encode(ChannelHandlerContext ctx, Channel channel, Object obj) throws Exception { - if (obj instanceof MessageBatch) { - return ((MessageBatch) obj).buffer(); - } - - throw new RuntimeException("Unsupported encoding of object of class " + obj.getClass().getName()); - } -} http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/83b36ef7/core/src/main/java/io/gearpump/transport/netty/NettyRenameThreadFactory.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/io/gearpump/transport/netty/NettyRenameThreadFactory.java b/core/src/main/java/io/gearpump/transport/netty/NettyRenameThreadFactory.java deleted file mode 100644 index 026684e..0000000 --- a/core/src/main/java/io/gearpump/transport/netty/NettyRenameThreadFactory.java +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.gearpump.transport.netty; - -import org.jboss.netty.util.ThreadNameDeterminer; -import org.jboss.netty.util.ThreadRenamingRunnable; - -import java.util.concurrent.ThreadFactory; -import java.util.concurrent.atomic.AtomicInteger; - -public class NettyRenameThreadFactory implements ThreadFactory { - - static { - //Rename Netty threads - ThreadRenamingRunnable.setThreadNameDeterminer(ThreadNameDeterminer.CURRENT); - } - - final ThreadGroup group; - final AtomicInteger index = new AtomicInteger(1); - final String name; - - NettyRenameThreadFactory(String name) { - SecurityManager s = System.getSecurityManager(); - group = (s != null) ? s.getThreadGroup() : - Thread.currentThread().getThreadGroup(); - this.name = name; - } - - public Thread newThread(Runnable r) { - Thread t = new Thread(group, r, name + "-" + index.getAndIncrement(), 0); - if (t.isDaemon()) - t.setDaemon(false); - if (t.getPriority() != Thread.NORM_PRIORITY) - t.setPriority(Thread.NORM_PRIORITY); - return t; - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/83b36ef7/core/src/main/java/io/gearpump/transport/netty/TaskMessage.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/io/gearpump/transport/netty/TaskMessage.java b/core/src/main/java/io/gearpump/transport/netty/TaskMessage.java deleted file mode 100644 index 243d3f0..0000000 --- a/core/src/main/java/io/gearpump/transport/netty/TaskMessage.java +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.gearpump.transport.netty; - -public class TaskMessage { - - // When network partition happen, there may be several task instances of - // same taskId co-existing for a short period of time. When they send messages - // to same target task, it may cause confusion. - // With sessionId, we can know which messages are from an old session, and which - // are from new session. Messages of old sesson will be dropped. - - private int _sessionId; - private long _targetTask; - private long _sourceTask; - private Object _message; - - public TaskMessage(int sessionId, long targetTask, long sourceTask, Object message) { - _sessionId = sessionId; - _targetTask = targetTask; - _sourceTask = sourceTask; - _message = message; - } - - public int sessionId() { - return _sessionId; - } - - public long targetTask() { - return _targetTask; - } - - public long sourceTask() { - return _sourceTask; - } - - public Object message() { - return _message; - } -}
