Merge branch 'master' of https://github.com/apache/storm into STORM-1837-2
Project: http://git-wip-us.apache.org/repos/asf/storm/repo Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/591618a7 Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/591618a7 Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/591618a7 Branch: refs/heads/master Commit: 591618a73690c8fa727a385d107ad0168d62de00 Parents: e7b1284 8115ef5 Author: Stig Rohde Døssing <s...@it-minds.dk> Authored: Mon Jul 25 09:26:25 2016 +0200 Committer: Stig Rohde Døssing <s...@it-minds.dk> Committed: Mon Jul 25 09:26:25 2016 +0200 ---------------------------------------------------------------------- CHANGELOG.md | 817 +++---------------- DEVELOPER.md | 14 + README.markdown | 13 +- SECURITY.md | 25 +- bin/storm-kafka-monitor | 43 + bin/storm.py | 44 +- conf/defaults.yaml | 9 +- conf/storm.yaml.example | 28 + docs/Clojure-DSL.md | 4 +- docs/Command-line-client.md | 165 ++++ docs/Common-patterns.md | 4 +- docs/Distributed-RPC.md | 2 +- docs/Metrics.md | 97 ++- docs/Powered-By.md | 10 + docs/Resource_Aware_Scheduler_overview.md | 54 +- docs/SECURITY.md | 2 +- docs/STORM-UI-REST-API.md | 276 +++++++ docs/State-checkpointing.md | 3 + docs/Transactional-topologies.md | 8 +- docs/Trident-state.md | 4 +- docs/Tutorial.md | 2 +- docs/Windowing.md | 37 +- docs/distcache-blobstore.md | 10 +- ...ware_scheduler_default_eviction_strategy.png | Bin 0 -> 331891 bytes ...ware_scheduler_default_eviction_strategy.svg | 3 + docs/images/transactional-batches.png | Bin 23293 -> 21797 bytes docs/images/transactional-commit-flow.png | Bin 17725 -> 17179 bytes docs/images/transactional-design-2.png | Bin 13537 -> 13352 bytes examples/storm-starter/README.markdown | 12 +- .../storm/starter/SlidingWindowTopology.java | 4 +- .../storm/starter/spout/TwitterSampleSpout.java | 12 +- .../starter/trident/TridentKafkaWordCount.java | 69 +- .../storm/starter/trident/TridentWordCount.java | 13 +- .../jvm/storm/starter/HdfsSpoutTopology.java | 22 +- .../java/org/apache/storm/flux/FluxBuilder.java | 8 + external/flux/flux-examples/README.md | 18 + .../flux/examples/StatefulWordCounter.java | 64 ++ .../resources/simple_stateful_wordcount.yaml | 60 ++ .../flux/wrappers/bolts/FluxShellBolt.java | 79 +- .../flux/wrappers/spouts/FluxShellSpout.java | 77 +- .../jvm/org/apache/storm/sql/StormSqlImpl.java | 25 +- .../apache/storm/sql/compiler/ExprCompiler.java | 2 +- .../sql/compiler/PostOrderRelNodeVisitor.java | 6 + .../standalone/BuiltinAggregateFunctions.java | 178 ++++ .../backends/standalone/PlanCompiler.java | 36 +- .../backends/standalone/RelNodeCompiler.java | 349 +++++++- .../test/org/apache/storm/sql/TestStormSql.java | 231 ++++++ .../sql/runtime/AbstractChannelHandler.java | 10 + .../storm/sql/runtime/ChannelContext.java | 2 + .../storm/sql/runtime/ChannelHandler.java | 4 + .../org/apache/storm/sql/runtime/Channels.java | 31 + .../test/org/apache/storm/sql/TestUtils.java | 77 ++ .../storm/hbase/bolt/AbstractHBaseBolt.java | 11 +- .../org/apache/storm/hbase/bolt/HBaseBolt.java | 38 +- .../apache/storm/hbase/common/HBaseClient.java | 8 +- .../hbase/topology/PersistentWordCount.java | 14 +- external/storm-hdfs/README.md | 108 +-- external/storm-hdfs/pom.xml | 8 + .../hdfs/bolt/format/SimpleFileNameFormat.java | 102 +++ .../org/apache/storm/hdfs/spout/HdfsSpout.java | 194 +++-- .../apache/storm/hdfs/spout/TextFileReader.java | 2 +- .../trident/format/SimpleFileNameFormat.java | 97 +++ .../storm/hdfs/bolt/HdfsFileTopology.java | 4 +- .../storm/hdfs/bolt/SequenceFileTopology.java | 21 +- .../bolt/format/TestSimpleFileNameFormat.java | 79 ++ .../apache/storm/hdfs/spout/TestHdfsSpout.java | 142 ++-- .../storm/hdfs/trident/TridentFileTopology.java | 27 +- .../hdfs/trident/TridentSequenceTopology.java | 24 +- .../format/TestSimpleFileNameFormat.java | 70 ++ external/storm-hive/pom.xml | 2 +- .../org/apache/storm/hive/bolt/HiveBolt.java | 34 +- .../storm/hive/bolt/mapper/HiveMapper.java | 4 +- .../apache/storm/hive/common/HiveWriter.java | 2 +- .../apache/storm/hive/trident/HiveState.java | 8 +- .../apache/storm/hive/bolt/HiveTopology.java | 1 - .../storm/hive/trident/TridentHiveTopology.java | 29 +- external/storm-kafka-client/README.md | 141 +++- .../apache/storm/kafka/spout/KafkaSpout.java | 58 +- .../storm/kafka/spout/KafkaSpoutConfig.java | 27 +- .../KafkaSpoutRetryExponentialBackoff.java | 6 +- .../storm/kafka/spout/KafkaSpoutStream.java | 61 +- .../storm/kafka/spout/KafkaSpoutStreams.java | 133 +-- .../spout/KafkaSpoutStreamsNamedTopics.java | 154 ++++ .../spout/KafkaSpoutStreamsWildcardTopics.java | 61 ++ .../kafka/spout/KafkaSpoutTuplesBuilder.java | 54 +- .../KafkaSpoutTuplesBuilderNamedTopics.java | 78 ++ .../KafkaSpoutTuplesBuilderWildcardTopics.java | 36 + .../spout/test/KafkaSpoutTopologyMain.java | 133 --- .../test/KafkaSpoutTopologyMainNamedTopics.java | 140 ++++ .../KafkaSpoutTopologyMainWildcardTopics.java | 62 ++ external/storm-kafka-monitor/README.md | 41 + external/storm-kafka-monitor/pom.xml | 127 +++ .../kafka/monitor/KafkaOffsetLagResult.java | 102 +++ .../storm/kafka/monitor/KafkaOffsetLagUtil.java | 403 +++++++++ .../kafka/monitor/KafkaPartitionOffsetLag.java | 71 ++ .../kafka/monitor/NewKafkaSpoutOffsetQuery.java | 76 ++ .../kafka/monitor/OldKafkaSpoutOffsetQuery.java | 124 +++ external/storm-kafka/README.md | 59 +- external/storm-kafka/pom.xml | 5 + .../storm/kafka/DynamicBrokersReader.java | 2 +- .../kafka/DynamicPartitionConnections.java | 2 +- .../ExponentialBackoffMsgRetryManager.java | 38 +- .../storm/kafka/FailedMsgRetryManager.java | 60 +- .../jvm/org/apache/storm/kafka/KafkaConfig.java | 2 +- .../jvm/org/apache/storm/kafka/KafkaSpout.java | 44 +- .../jvm/org/apache/storm/kafka/KafkaUtils.java | 2 +- .../jvm/org/apache/storm/kafka/Partition.java | 3 +- .../apache/storm/kafka/PartitionManager.java | 57 +- .../jvm/org/apache/storm/kafka/SpoutConfig.java | 8 +- .../storm/kafka/StringMultiSchemeWithTopic.java | 3 +- .../org/apache/storm/kafka/ZkCoordinator.java | 2 +- .../src/jvm/org/apache/storm/kafka/ZkState.java | 2 +- .../bolt/selector/FieldIndexTopicSelector.java | 48 ++ .../bolt/selector/FieldNameTopicSelector.java | 49 ++ .../trident/GlobalPartitionInformation.java | 4 + .../kafka/trident/TridentKafkaEmitter.java | 2 +- .../storm/kafka/trident/ZkBrokerReader.java | 2 +- .../ExponentialBackoffMsgRetryManagerTest.java | 99 ++- .../org/apache/storm/kafka/KafkaTestBroker.java | 15 +- .../storm/mongodb/bolt/MongoInsertBolt.java | 61 +- external/storm-opentsdb/README.md | 76 ++ external/storm-opentsdb/pom.xml | 97 +++ .../storm/opentsdb/OpenTsdbMetricDatapoint.java | 119 +++ .../bolt/ITupleOpenTsdbDatapointMapper.java | 39 + .../storm/opentsdb/bolt/OpenTsdbBolt.java | 179 ++++ .../bolt/TupleOpenTsdbDatapointMapper.java | 120 +++ .../storm/opentsdb/client/ClientResponse.java | 178 ++++ .../storm/opentsdb/client/OpenTsdbClient.java | 153 ++++ .../storm/opentsdb/trident/OpenTsdbState.java | 89 ++ .../opentsdb/trident/OpenTsdbStateFactory.java | 50 ++ .../opentsdb/trident/OpenTsdbStateUpdater.java | 36 + .../storm/opentsdb/MetricGenBatchSpout.java | 94 +++ .../apache/storm/opentsdb/MetricGenSpout.java | 72 ++ .../opentsdb/SampleOpenTsdbBoltTopology.java | 70 ++ .../opentsdb/SampleOpenTsdbTridentTopology.java | 87 ++ external/storm-redis/README.md | 57 +- external/storm-redis/pom.xml | 2 +- .../storm/redis/bolt/RedisFilterBolt.java | 148 ++++ .../storm/redis/bolt/RedisLookupBolt.java | 6 +- .../apache/storm/redis/bolt/RedisStoreBolt.java | 17 +- .../redis/common/config/JedisPoolConfig.java | 3 +- .../common/container/JedisClusterContainer.java | 7 +- .../common/mapper/RedisDataTypeDescription.java | 7 +- .../redis/common/mapper/RedisFilterMapper.java | 32 + .../redis/common/mapper/RedisLookupMapper.java | 2 +- .../storm/redis/state/RedisKeyValueState.java | 6 +- .../redis/topology/WhitelistWordCount.java | 155 ++++ .../storm/solr/topology/SolrTopology.java | 6 +- log4j2/cluster.xml | 14 + pom.xml | 26 +- storm-core/pom.xml | 17 +- .../apache/storm/command/shell_submission.clj | 3 +- .../clj/org/apache/storm/daemon/executor.clj | 11 +- .../src/clj/org/apache/storm/daemon/nimbus.clj | 347 ++++---- .../src/clj/org/apache/storm/daemon/worker.clj | 2 +- .../src/clj/org/apache/storm/metric/testing.clj | 68 -- storm-core/src/clj/org/apache/storm/testing.clj | 2 +- storm-core/src/clj/org/apache/storm/ui/core.clj | 123 ++- .../src/clj/org/apache/storm/ui/helpers.clj | 21 +- storm-core/src/jvm/org/apache/storm/Config.java | 50 +- .../jvm/org/apache/storm/StormSubmitter.java | 48 +- .../storm/blobstore/BlobStoreAclHandler.java | 10 + .../storm/cluster/StormClusterStateImpl.java | 2 + .../apache/storm/cluster/ZKStateStorage.java | 10 +- .../src/jvm/org/apache/storm/daemon/Acker.java | 12 +- .../org/apache/storm/daemon/StormCommon.java | 44 +- .../daemon/supervisor/SupervisorUtils.java | 39 + .../daemon/supervisor/SyncProcessEvent.java | 29 +- .../daemon/supervisor/SyncSupervisorEvent.java | 108 +-- .../workermanager/DefaultWorkerManager.java | 15 +- .../apache/storm/generated/HBMessageData.java | 6 +- .../storm/hooks/SubmitterHookException.java | 41 + .../metric/ClusterMetricsConsumerExecutor.java | 70 ++ .../metric/LoggingClusterMetricsConsumer.java | 86 ++ .../storm/metric/MetricsConsumerBolt.java | 89 +- .../org/apache/storm/metric/api/DataPoint.java | 43 + .../metric/api/IClusterMetricsConsumer.java | 63 ++ .../storm/metric/api/IMetricsConsumer.java | 21 + .../storm/metric/filter/FilterByMetricName.java | 110 +++ .../storm/metric/filter/MetricsFilter.java | 26 + .../storm/metric/util/DataPointExpander.java | 79 ++ .../apache/storm/pacemaker/PacemakerServer.java | 9 +- .../storm/pacemaker/codec/ThriftDecoder.java | 21 +- .../jvm/org/apache/storm/scheduler/Cluster.java | 7 + .../storm/scheduler/resource/RAS_Node.java | 11 +- .../DefaultResourceAwareStrategy.java | 228 ++++-- .../apache/storm/security/auth/AuthUtils.java | 139 ++-- .../jvm/org/apache/storm/spout/ShellSpout.java | 56 +- .../storm/spout/SpoutOutputCollector.java | 20 +- .../jvm/org/apache/storm/stats/StatsUtil.java | 62 +- .../jvm/org/apache/storm/task/ShellBolt.java | 20 +- .../topology/BaseConfigurationDeclarer.java | 14 +- .../storm/topology/WindowedBoltExecutor.java | 28 +- .../topology/base/BaseStatefulWindowedBolt.java | 8 + .../storm/topology/base/BaseWindowedBolt.java | 101 +++ .../apache/storm/trident/TridentTopology.java | 82 +- .../org/apache/storm/trident/graph/Group.java | 74 +- .../operation/DefaultResourceDeclarer.java | 14 +- .../org/apache/storm/trident/planner/Node.java | 9 +- .../storm/trident/planner/ProcessorNode.java | 5 + .../spout/PartitionedTridentSpoutExecutor.java | 33 +- .../topology/TridentTopologyBuilder.java | 29 +- .../windowing/WindowTridentProcessor.java | 8 +- .../src/jvm/org/apache/storm/tuple/Fields.java | 9 + .../jvm/org/apache/storm/utils/BatchHelper.java | 92 +++ .../org/apache/storm/utils/DisruptorQueue.java | 16 +- .../apache/storm/utils/TopologySpoutLag.java | 185 +++++ .../src/jvm/org/apache/storm/utils/Utils.java | 31 +- .../storm/validation/ConfigValidation.java | 16 + .../storm/windowing/CountEvictionPolicy.java | 12 +- .../storm/windowing/CountTriggerPolicy.java | 2 +- .../storm/windowing/DefaultEvictionContext.java | 53 ++ .../apache/storm/windowing/EvictionContext.java | 47 ++ .../apache/storm/windowing/EvictionPolicy.java | 3 +- .../storm/windowing/TimeEvictionPolicy.java | 4 +- .../storm/windowing/TimeTriggerPolicy.java | 2 +- .../windowing/WatermarkCountEvictionPolicy.java | 27 +- .../windowing/WatermarkCountTriggerPolicy.java | 2 +- .../windowing/WatermarkTimeTriggerPolicy.java | 3 +- .../storm/zookeeper/LeaderElectorImp.java | 21 +- .../org/apache/storm/zookeeper/Zookeeper.java | 48 +- storm-core/src/py/storm/ttypes.py | 422 +++++----- storm-core/src/storm.thrift | 26 +- storm-core/src/ui/public/component.html | 30 +- .../src/ui/public/deep_search_result.html | 10 +- storm-core/src/ui/public/index.html | 18 +- storm-core/src/ui/public/js/script.js | 12 + storm-core/src/ui/public/js/visualization.js | 4 +- storm-core/src/ui/public/logviewer_search.html | 6 +- storm-core/src/ui/public/search_result.html | 6 +- .../public/templates/index-page-template.html | 10 +- .../templates/topology-page-template.html | 87 ++ storm-core/src/ui/public/topology.html | 66 +- .../apache/storm/trident/integration_test.clj | 3 + .../test/clj/org/apache/storm/fields_test.clj | 59 -- .../apache/storm/messaging/netty_unit_test.clj | 2 +- .../test/clj/org/apache/storm/metrics_test.clj | 32 +- .../test/clj/org/apache/storm/nimbus_test.clj | 39 +- .../storm/security/auth/AuthUtils_test.clj | 75 -- .../storm/security/auth/ReqContext_test.clj | 73 -- .../security/auth/SaslTransportPlugin_test.clj | 43 - .../storm/security/auth/ThriftClient_test.clj | 61 -- .../storm/security/auth/ThriftServer_test.clj | 32 - .../BlowfishTupleSerializer_test.clj | 77 -- .../clj/org/apache/storm/supervisor_test.clj | 21 +- .../apache/storm/metric/FakeMetricConsumer.java | 88 ++ .../metric/filter/FilterByMetricNameTest.java | 95 +++ .../metric/util/DataPointExpanderTest.java | 109 +++ .../apache/storm/nimbus/LocalNimbusTest.java | 132 +++ .../resource/TestResourceAwareScheduler.java | 788 +----------------- .../TestUtilsForResourceAwareScheduler.java | 68 +- .../TestDefaultResourceAwareStrategy.java | 333 ++++++++ .../storm/security/auth/AuthUtilsTest.java | 240 ++++++ .../storm/security/auth/AuthUtilsTestMock.java | 82 ++ .../storm/security/auth/ReqContextTest.java | 87 ++ .../security/auth/SaslTransportPluginTest.java | 49 ++ .../storm/security/auth/ThriftClientTest.java | 68 ++ .../storm/security/auth/ThriftServerTest.java | 38 + .../BlowfishTupleSerializerTest.java | 86 ++ .../topology/WindowedBoltExecutorTest.java | 87 +- .../jvm/org/apache/storm/tuple/FieldsTest.java | 126 +++ .../apache/storm/utils/DisruptorQueueTest.java | 2 +- .../utils/ThrowableNestedCauseMatcher.java | 44 + .../storm/windowing/WindowManagerTest.java | 69 +- storm-dist/binary/pom.xml | 10 + storm-dist/binary/src/main/assembly/binary.xml | 41 +- .../src/main/resources/resources/storm.js | 17 +- .../src/main/resources/resources/storm.py | 12 +- 268 files changed, 11866 insertions(+), 3861 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/storm/blob/591618a7/storm-core/src/clj/org/apache/storm/testing.clj ----------------------------------------------------------------------
