Merge branch 'master' into port-scheduler-to-java
Project: http://git-wip-us.apache.org/repos/asf/storm/repo Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/672c3c00 Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/672c3c00 Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/672c3c00 Branch: refs/heads/master Commit: 672c3c002d0fc88de846ec4bdb98898e4104825d Parents: a199e9e 7f52aec Author: vesense <[email protected]> Authored: Wed Mar 16 13:52:30 2016 +0800 Committer: vesense <[email protected]> Committed: Wed Mar 16 13:52:30 2016 +0800 ---------------------------------------------------------------------- .gitignore | 2 + CHANGELOG.md | 70 +- README.markdown | 8 +- bin/flight.bash | 4 +- bin/storm | 19 +- bin/storm-config.cmd | 4 + bin/storm.cmd | 26 +- bin/storm.py | 22 +- conf/cgconfig.conf.example | 41 + conf/defaults.yaml | 21 +- conf/log4j2.xml | 2 +- .../travis/print-errors-from-test-reports.py | 4 + examples/storm-starter/pom.xml | 10 + .../org/apache/storm/starter/clj/word_count.clj | 3 +- .../org/apache/storm/starter/ManualDRPC.java | 53 +- .../starter/ResourceAwareExampleTopology.java | 2 +- .../storm/starter/ThroughputVsLatency.java | 2 +- .../storm/starter/spout/RandomIntegerSpout.java | 15 +- .../spout/RandomNumberGeneratorSpout.java | 95 + .../trident/TridentMinMaxOfDevicesTopology.java | 201 ++ .../TridentMinMaxOfVehiclesTopology.java | 180 ++ .../src/jvm/storm/starter/StatefulTopology.java | 1 + .../apache/storm/sql/compiler/CompilerUtil.java | 7 +- .../apache/storm/sql/compiler/ExprCompiler.java | 32 +- .../backends/standalone/RelNodeCompiler.java | 6 +- .../apache/storm/sql/parser/StormParser.java | 5 + .../test/org/apache/storm/sql/TestStormSql.java | 64 +- .../storm/sql/compiler/TestCompilerUtils.java | 62 +- .../storm/sql/compiler/TestExprSemantic.java | 18 + .../backends/standalone/TestPlanCompiler.java | 20 + .../backends/trident/TestPlanCompiler.java | 4 +- .../test/org/apache/storm/sql/TestUtils.java | 32 +- external/storm-hdfs/pom.xml | 23 +- .../storm/hdfs/bolt/AbstractHdfsBolt.java | 10 +- .../storm/hdfs/bolt/AvroGenericRecordBolt.java | 8 +- .../org/apache/storm/hdfs/bolt/HdfsBolt.java | 8 +- .../storm/hdfs/bolt/SequenceFileBolt.java | 8 +- external/storm-kafka/README.md | 1 - .../org/apache/storm/kafka/IntSerializer.java | 10 +- .../apache/storm/kafka/PartitionManager.java | 17 +- .../kafka/trident/TridentKafkaEmitter.java | 5 +- external/storm-mongodb/README.md | 195 ++ external/storm-mongodb/pom.xml | 74 + .../storm/mongodb/bolt/AbstractMongoBolt.java | 56 + .../storm/mongodb/bolt/MongoInsertBolt.java | 62 + .../storm/mongodb/bolt/MongoUpdateBolt.java | 75 + .../storm/mongodb/common/MongoDBClient.java | 91 + .../mongodb/common/QueryFilterCreator.java | 38 + .../common/SimpleQueryFilterCreator.java | 39 + .../mongodb/common/mapper/MongoMapper.java | 38 + .../common/mapper/SimpleMongoMapper.java | 40 + .../common/mapper/SimpleMongoUpdateMapper.java | 41 + .../storm/mongodb/trident/state/MongoState.java | 97 + .../trident/state/MongoStateFactory.java | 42 + .../trident/state/MongoStateUpdater.java | 34 + .../storm/mongodb/topology/InsertWordCount.java | 81 + .../storm/mongodb/topology/UpdateWordCount.java | 91 + .../storm/mongodb/topology/WordCounter.java | 67 + .../storm/mongodb/topology/WordSpout.java | 88 + .../storm/mongodb/trident/WordCountTrident.java | 85 + pom.xml | 42 +- storm-clojure/pom.xml | 74 + .../src/clj/org/apache/storm/clojure.clj | 207 ++ .../src/clj/org/apache/storm/thrift.clj | 286 ++ storm-clojure/src/test/clj/clojure_test.clj | 158 ++ storm-core/pom.xml | 9 + .../src/clj/org/apache/storm/LocalDRPC.clj | 56 - .../src/clj/org/apache/storm/MockAutoCred.clj | 58 - storm-core/src/clj/org/apache/storm/clojure.clj | 207 -- storm-core/src/clj/org/apache/storm/cluster.clj | 700 ----- .../cluster_state/zookeeper_state_factory.clj | 165 -- .../clj/org/apache/storm/command/activate.clj | 24 - .../clj/org/apache/storm/command/deactivate.clj | 24 - .../org/apache/storm/command/dev_zookeeper.clj | 28 - .../clj/org/apache/storm/command/get_errors.clj | 3 +- .../org/apache/storm/command/healthcheck.clj | 90 - .../clj/org/apache/storm/command/heartbeats.clj | 9 +- .../org/apache/storm/command/kill_topology.clj | 29 - .../src/clj/org/apache/storm/command/list.clj | 38 - .../clj/org/apache/storm/command/monitor.clj | 37 - .../clj/org/apache/storm/command/rebalance.clj | 46 - .../org/apache/storm/command/set_log_level.clj | 75 - .../apache/storm/command/shell_submission.clj | 2 +- .../apache/storm/command/upload_credentials.clj | 35 - .../src/clj/org/apache/storm/converter.clj | 68 +- .../src/clj/org/apache/storm/daemon/acker.clj | 108 - .../org/apache/storm/daemon/builtin_metrics.clj | 33 +- .../src/clj/org/apache/storm/daemon/common.clj | 351 +-- .../src/clj/org/apache/storm/daemon/drpc.clj | 221 +- .../clj/org/apache/storm/daemon/executor.clj | 167 +- .../clj/org/apache/storm/daemon/logviewer.clj | 111 +- .../src/clj/org/apache/storm/daemon/nimbus.clj | 542 ++-- .../clj/org/apache/storm/daemon/supervisor.clj | 323 ++- .../src/clj/org/apache/storm/daemon/task.clj | 17 +- .../src/clj/org/apache/storm/daemon/worker.clj | 268 +- .../src/clj/org/apache/storm/disruptor.clj | 89 - storm-core/src/clj/org/apache/storm/event.clj | 71 - .../clj/org/apache/storm/internal/clojure.clj | 201 ++ .../clj/org/apache/storm/internal/thrift.clj | 96 + .../src/clj/org/apache/storm/local_state.clj | 134 - .../org/apache/storm/local_state_converter.clj | 24 + .../storm/pacemaker/pacemaker_state_factory.clj | 141 - .../clj/org/apache/storm/process_simulator.clj | 49 - storm-core/src/clj/org/apache/storm/stats.clj | 1567 ----------- storm-core/src/clj/org/apache/storm/testing.clj | 161 +- storm-core/src/clj/org/apache/storm/thrift.clj | 286 -- storm-core/src/clj/org/apache/storm/timer.clj | 128 - .../clj/org/apache/storm/trident/testing.clj | 2 - storm-core/src/clj/org/apache/storm/ui/core.clj | 295 +-- .../src/clj/org/apache/storm/ui/helpers.clj | 207 +- storm-core/src/clj/org/apache/storm/util.clj | 22 +- .../src/clj/org/apache/storm/zookeeper.clj | 74 - storm-core/src/jvm/org/apache/storm/Config.java | 97 + .../src/jvm/org/apache/storm/LocalDRPC.java | 72 + .../jvm/org/apache/storm/ProcessSimulator.java | 82 + .../src/jvm/org/apache/storm/StormTimer.java | 241 ++ storm-core/src/jvm/org/apache/storm/Thrift.java | 351 +++ .../storm/blobstore/LocalFsBlobStore.java | 2 +- .../jvm/org/apache/storm/callback/Callback.java | 23 - .../storm/callback/ZKStateChangedCallback.java | 25 + .../org/apache/storm/cluster/ClusterState.java | 217 -- .../storm/cluster/ClusterStateContext.java | 2 +- .../storm/cluster/ClusterStateFactory.java | 28 - .../org/apache/storm/cluster/ClusterUtils.java | 244 ++ .../org/apache/storm/cluster/ExecutorBeat.java | 44 + .../org/apache/storm/cluster/IStateStorage.java | 222 ++ .../storm/cluster/IStormClusterState.java | 124 + .../storm/cluster/PaceMakerStateStorage.java | 216 ++ .../cluster/PaceMakerStateStorageFactory.java | 64 + .../storm/cluster/StateStorageFactory.java | 28 + .../storm/cluster/StormClusterStateImpl.java | 697 +++++ .../apache/storm/cluster/ZKStateStorage.java | 244 ++ .../storm/cluster/ZKStateStorageFactory.java | 36 + .../jvm/org/apache/storm/command/Activate.java | 40 + .../src/jvm/org/apache/storm/command/CLI.java | 369 +++ .../org/apache/storm/command/Deactivate.java | 40 + .../org/apache/storm/command/DevZookeeper.java | 35 + .../org/apache/storm/command/HealthCheck.java | 125 + .../org/apache/storm/command/KillTopology.java | 51 + .../apache/storm/command/ListTopologies.java | 52 + .../jvm/org/apache/storm/command/Monitor.java | 65 + .../jvm/org/apache/storm/command/Rebalance.java | 86 + .../org/apache/storm/command/SetLogLevel.java | 116 + .../apache/storm/command/UploadCredentials.java | 61 + .../container/ResourceIsolationInterface.java | 51 + .../storm/container/cgroup/CgroupCenter.java | 216 ++ .../storm/container/cgroup/CgroupCommon.java | 270 ++ .../container/cgroup/CgroupCommonOperation.java | 81 + .../container/cgroup/CgroupCoreFactory.java | 74 + .../storm/container/cgroup/CgroupManager.java | 210 ++ .../storm/container/cgroup/CgroupOperation.java | 79 + .../storm/container/cgroup/CgroupUtils.java | 118 + .../apache/storm/container/cgroup/Device.java | 75 + .../storm/container/cgroup/Hierarchy.java | 130 + .../storm/container/cgroup/SubSystem.java | 81 + .../storm/container/cgroup/SubSystemType.java | 36 + .../storm/container/cgroup/SystemOperation.java | 75 + .../storm/container/cgroup/core/BlkioCore.java | 213 ++ .../storm/container/cgroup/core/CgroupCore.java | 26 + .../storm/container/cgroup/core/CpuCore.java | 135 + .../container/cgroup/core/CpuacctCore.java | 71 + .../storm/container/cgroup/core/CpusetCore.java | 209 ++ .../container/cgroup/core/DevicesCore.java | 189 ++ .../container/cgroup/core/FreezerCore.java | 66 + .../storm/container/cgroup/core/MemoryCore.java | 188 ++ .../storm/container/cgroup/core/NetClsCore.java | 69 + .../container/cgroup/core/NetPrioCore.java | 65 + .../src/jvm/org/apache/storm/daemon/Acker.java | 128 + .../org/apache/storm/daemon/DaemonCommon.java | 22 + .../jvm/org/apache/storm/daemon/DrpcServer.java | 357 +++ .../org/apache/storm/daemon/StormCommon.java | 537 ++++ .../storm/daemon/metrics/MetricsUtils.java | 2 +- .../jvm/org/apache/storm/drpc/DRPCSpout.java | 2 + .../org/apache/storm/event/EventManager.java | 24 + .../org/apache/storm/event/EventManagerImp.java | 97 + .../apache/storm/messaging/netty/Client.java | 34 + .../storm/metric/FileBasedEventLogger.java | 18 +- .../storm/metric/StormMetricsRegistry.java | 84 + .../apache/storm/pacemaker/PacemakerClient.java | 6 +- .../auth/AbstractSaslClientCallbackHandler.java | 76 + .../auth/AbstractSaslServerCallbackHandler.java | 94 + .../apache/storm/security/auth/AuthUtils.java | 40 + .../security/auth/ThriftConnectionType.java | 2 +- .../auth/digest/ClientCallbackHandler.java | 60 +- .../auth/digest/ServerCallbackHandler.java | 61 +- .../storm/security/auth/kerberos/AutoTGT.java | 64 +- .../auth/kerberos/AutoTGTKrb5LoginModule.java | 8 +- .../auth/plain/PlainClientCallbackHandler.java | 31 + .../auth/plain/PlainSaslTransportPlugin.java | 71 + .../auth/plain/PlainServerCallbackHandler.java | 55 + .../security/auth/plain/SaslPlainServer.java | 158 ++ .../serialization/SerializationFactory.java | 2 + .../apache/storm/stats/BoltExecutorStats.java | 105 + .../jvm/org/apache/storm/stats/CommonStats.java | 112 + .../apache/storm/stats/SpoutExecutorStats.java | 79 + .../jvm/org/apache/storm/stats/StatsUtil.java | 2441 ++++++++++++++++++ .../jvm/org/apache/storm/testing/NGrouping.java | 4 +- .../storm/testing/PythonShellMetricsBolt.java | 14 +- .../storm/testing/PythonShellMetricsSpout.java | 8 +- .../testing/staticmocking/MockedCluster.java | 31 + .../MockedPaceMakerStateStorageFactory.java | 32 + .../topology/CheckpointTupleForwarder.java | 22 +- .../apache/storm/topology/IStatefulBolt.java | 7 +- .../storm/topology/StatefulBoltExecutor.java | 46 +- .../apache/storm/topology/TopologyBuilder.java | 18 +- .../jvm/org/apache/storm/trident/Stream.java | 121 +- .../operation/builtin/ComparisonAggregator.java | 91 + .../storm/trident/operation/builtin/Max.java | 37 + .../operation/builtin/MaxWithComparator.java | 51 + .../storm/trident/operation/builtin/Min.java | 36 + .../operation/builtin/MinWithComparator.java | 51 + .../apache/storm/trident/tuple/ConsList.java | 20 +- .../apache/storm/ui/FilterConfiguration.java | 63 + .../jvm/org/apache/storm/ui/IConfigurator.java | 24 + .../src/jvm/org/apache/storm/ui/UIHelpers.java | 267 ++ .../jvm/org/apache/storm/utils/ConfigUtils.java | 28 +- .../org/apache/storm/utils/DisruptorQueue.java | 15 +- .../jvm/org/apache/storm/utils/LocalState.java | 112 +- .../org/apache/storm/utils/NimbusClient.java | 19 +- .../storm/utils/StormCommonInstaller.java | 43 + .../src/jvm/org/apache/storm/utils/Time.java | 1 + .../src/jvm/org/apache/storm/utils/Utils.java | 161 +- .../storm/utils/WorkerBackpressureCallback.java | 2 +- .../storm/utils/WorkerBackpressureThread.java | 38 +- .../org/apache/storm/zookeeper/Zookeeper.java | 77 +- storm-core/src/ui/public/component.html | 2 +- .../templates/topology-page-template.html | 6 +- storm-core/src/ui/public/topology.html | 2 +- .../org/apache/storm/integration_test.clj | 274 +- .../org/apache/storm/testing4j_test.clj | 72 +- .../test/clj/org/apache/storm/clojure_test.clj | 64 +- .../test/clj/org/apache/storm/cluster_test.clj | 203 +- .../test/clj/org/apache/storm/drpc_test.clj | 50 +- .../test/clj/org/apache/storm/grouping_test.clj | 56 +- .../storm/messaging/netty_integration_test.clj | 19 +- .../apache/storm/messaging/netty_unit_test.clj | 14 +- .../clj/org/apache/storm/messaging_test.clj | 14 +- .../test/clj/org/apache/storm/metrics_test.clj | 85 +- .../test/clj/org/apache/storm/nimbus_test.clj | 548 ++-- .../storm/pacemaker_state_factory_test.clj | 121 +- .../scheduler/resource_aware_scheduler_test.clj | 3 +- .../apache/storm/security/auth/auth_test.clj | 18 +- .../security/auth/auto_login_module_test.clj | 24 +- .../storm/security/auth/drpc_auth_test.clj | 20 +- .../storm/security/auth/nimbus_auth_test.clj | 18 +- .../clj/org/apache/storm/serialization_test.clj | 85 +- .../clj/org/apache/storm/supervisor_test.clj | 108 +- .../clj/org/apache/storm/tick_tuple_test.clj | 15 +- .../clj/org/apache/storm/transactional_test.clj | 3 +- .../test/clj/org/apache/storm/utils_test.clj | 111 - .../test/jvm/org/apache/storm/MockAutoCred.java | 75 + .../test/jvm/org/apache/storm/TestCgroups.java | 130 + .../org/apache/storm/TestConfigValidate.java | 20 + .../org/apache/storm/command/RebalanceTest.java | 41 + .../apache/storm/command/SetLogLevelTest.java | 54 + .../jvm/org/apache/storm/command/TestCLI.java | 81 + .../resource/TestResourceAwareScheduler.java | 3 + .../storm/serialization/SerializationTest.java | 115 + .../topology/StatefulBoltExecutorTest.java | 1 + .../storm/topology/TopologyBuilderTest.java | 65 + .../jvm/org/apache/storm/utils/TimeTest.java | 112 + .../jvm/org/apache/storm/utils/UtilsTest.java | 219 ++ .../utils/WorkerBackpressureThreadTest.java | 50 + .../storm/utils/staticmocking/package-info.java | 2 +- storm-dist/binary/src/main/assembly/binary.xml | 14 + 265 files changed, 18933 insertions(+), 7636 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/storm/blob/672c3c00/storm-core/src/clj/org/apache/storm/daemon/nimbus.clj ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/storm/blob/672c3c00/storm-core/src/jvm/org/apache/storm/utils/Utils.java ---------------------------------------------------------------------- diff --cc storm-core/src/jvm/org/apache/storm/utils/Utils.java index ae3d387,b8a6c1a..93f4aa6 --- a/storm-core/src/jvm/org/apache/storm/utils/Utils.java +++ b/storm-core/src/jvm/org/apache/storm/utils/Utils.java @@@ -2249,22 -2316,23 +2316,42 @@@ public class Utils return process; } + public static <T> List<T> interleaveAll(List<List<T>> nodeList) { + if (nodeList != null && nodeList.size() > 0) { + List<T> first = new ArrayList<T>(); + List<List<T>> rest = new ArrayList<List<T>>(); + for (List<T> node : nodeList) { + if (node != null && node.size() > 0) { + first.add(node.get(0)); + rest.add(node.subList(1, node.size())); + } + } + List<T> interleaveRest = interleaveAll(rest); + if (interleaveRest != null) { + first.addAll(interleaveRest); + } + return first; + } + return null; + } ++ + public static long bitXor(Long a, Long b) { + return a ^ b; + } + + public static List<String> getRepeat(List<String> list) { + List<String> rtn = new ArrayList<String>(); + Set<String> idSet = new HashSet<String>(); + + for (String id : list) { + if (idSet.contains(id)) { + rtn.add(id); + } else { + idSet.add(id); + } + } + + return rtn; + } -} + +}
