METRON-1699 Create Batch Profiler (nickwallen)
Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/fbabfcad Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/fbabfcad Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/fbabfcad Branch: refs/heads/master Commit: fbabfcad49581b531a7b7eb755a561a030a2bc06 Parents: 4f0b608 fa3be8d Author: Nick Allen <[email protected]> Authored: Fri Sep 28 08:22:36 2018 -0400 Committer: Nick Allen <[email protected]> Committed: Fri Sep 28 08:22:36 2018 -0400 ---------------------------------------------------------------------- README.md | 2 +- dependencies_with_url.csv | 66 +- .../client/stellar/ProfilerFunctions.java | 283 ------ .../client/stellar/ProfilerFunctionsTest.java | 334 ------- .../metron-profiler-common/README.md | 386 ++++++++ .../profiler/DefaultMessageDistributor.java | 33 +- .../metron/profiler/DefaultMessageRouter.java | 37 +- .../metron/profiler/DefaultProfileBuilder.java | 4 +- .../metron/profiler/MessageDistributor.java | 4 +- .../apache/metron/profiler/MessageRoute.java | 76 +- .../metron/profiler/ProfileMeasurement.java | 2 +- .../apache/metron/profiler/ProfilePeriod.java | 51 +- .../metron/profiler/StandAloneProfiler.java | 186 ---- .../profiler/clock/DefaultClockFactory.java | 4 +- .../metron/profiler/clock/EventTimeClock.java | 3 +- .../clock/EventTimeOnlyClockFactory.java | 58 ++ .../profiler/DefaultMessageDistributorTest.java | 32 +- .../profiler/DefaultMessageRouterTest.java | 55 ++ .../profiler/DefaultProfileBuilderTest.java | 6 +- .../metron/profiler/ProfilePeriodTest.java | 51 +- .../metron/profiler/StandAloneProfilerTest.java | 255 ------ .../clock/EventTimeOnlyClockFactoryTest.java | 61 ++ metron-analytics/metron-profiler-repl/README.md | 179 ++++ metron-analytics/metron-profiler-repl/pom.xml | 150 ++++ .../src/main/assembly/assembly.xml | 36 + .../metron/profiler/repl/ProfilerFunctions.java | 278 ++++++ .../profiler/repl/StandAloneProfiler.java | 176 ++++ .../profiler/repl/ProfilerFunctionsTest.java | 332 +++++++ .../profiler/repl/StandAloneProfilerTest.java | 255 ++++++ .../metron-profiler-spark/README.md | 317 +++++++ metron-analytics/metron-profiler-spark/pom.xml | 222 +++++ .../src/main/assembly/assembly.xml | 60 ++ .../src/main/config/batch-profiler.properties | 26 + .../metron/profiler/spark/BatchProfiler.java | 139 +++ .../profiler/spark/BatchProfilerConfig.java | 194 ++++ .../spark/ProfileMeasurementAdapter.java | 132 +++ .../metron/profiler/spark/TimestampParser.java | 55 ++ .../profiler/spark/cli/BatchProfilerCLI.java | 200 +++++ .../spark/cli/BatchProfilerCLIOptions.java | 150 ++++ .../spark/function/GroupByPeriodFunction.java | 60 ++ .../spark/function/HBaseWriterFunction.java | 171 ++++ .../spark/function/MessageRouterFunction.java | 203 +++++ .../spark/function/ProfileBuilderFunction.java | 107 +++ .../profiler/spark/function/TaskUtils.java | 41 + .../src/main/scripts/start_batch_profiler.sh | 32 + .../spark/BatchProfilerIntegrationTest.java | 324 +++++++ .../profiler/spark/TimestampParserTest.java | 67 ++ .../spark/cli/BatchProfilerCLITest.java | 61 ++ .../spark/function/HBaseWriterFunctionTest.java | 176 ++++ .../function/MessageRouterFunctionTest.java | 229 +++++ .../function/ProfileBuilderFunctionTest.java | 98 ++ .../src/test/resources/globals.json | 3 + .../src/test/resources/log4j.properties | 31 + .../src/test/resources/profiles-empty.json | 5 + .../resources/profiles-no-timestamp-field.json | 12 + .../src/test/resources/profiles.json | 13 + .../src/test/resources/telemetry.json | 100 +++ .../metron-profiler-storm/.gitignore | 1 + .../metron-profiler-storm/README.md | 400 +++++++++ metron-analytics/metron-profiler-storm/pom.xml | 407 +++++++++ .../src/main/assembly/assembly.xml | 72 ++ .../src/main/config/profiler.properties | 71 ++ .../src/main/flux/profiler/remote.yaml | 218 +++++ .../storm/FixedFrequencyFlushSignal.java | 135 +++ .../metron/profiler/storm/FlushSignal.java | 51 ++ .../metron/profiler/storm/HBaseEmitter.java | 73 ++ .../metron/profiler/storm/KafkaEmitter.java | 164 ++++ .../profiler/storm/ManualFlushSignal.java | 54 ++ .../profiler/storm/ProfileBuilderBolt.java | 552 ++++++++++++ .../profiler/storm/ProfileHBaseMapper.java | 117 +++ .../storm/ProfileMeasurementEmitter.java | 59 ++ .../profiler/storm/ProfileSplitterBolt.java | 228 +++++ .../src/main/resources/META-INF/LICENSE | 604 +++++++++++++ .../src/main/resources/META-INF/NOTICE | 92 ++ .../src/main/scripts/start_profiler_topology.sh | 22 + .../zookeeper/event-time-test/profiler.json | 19 + .../processing-time-test/profiler.json | 11 + .../zookeeper/profile-with-stats/profiler.json | 12 + .../storm/FixedFrequencyFlushSignalTest.java | 71 ++ .../metron/profiler/storm/HBaseEmitterTest.java | 118 +++ .../metron/profiler/storm/KafkaEmitterTest.java | 291 ++++++ .../profiler/storm/ProfileBuilderBoltTest.java | 378 ++++++++ .../profiler/storm/ProfileHBaseMapperTest.java | 93 ++ .../profiler/storm/ProfileSplitterBoltTest.java | 455 ++++++++++ .../integration/ConfigUploadComponent.java | 124 +++ .../storm/integration/MessageBuilder.java | 75 ++ .../integration/ProfilerIntegrationTest.java | 478 ++++++++++ .../src/test/resources/log4j.properties | 32 + .../src/test/resources/telemetry.json | 100 +++ metron-analytics/metron-profiler/.gitignore | 1 - metron-analytics/metron-profiler/README.md | 898 ------------------- metron-analytics/metron-profiler/pom.xml | 407 --------- .../src/main/assembly/assembly.xml | 72 -- .../src/main/config/profiler.properties | 71 -- .../src/main/flux/profiler/remote.yaml | 218 ----- .../bolt/FixedFrequencyFlushSignal.java | 135 --- .../metron/profiler/bolt/FlushSignal.java | 51 -- .../metron/profiler/bolt/HBaseEmitter.java | 73 -- .../metron/profiler/bolt/KafkaEmitter.java | 164 ---- .../metron/profiler/bolt/ManualFlushSignal.java | 54 -- .../profiler/bolt/ProfileBuilderBolt.java | 552 ------------ .../profiler/bolt/ProfileHBaseMapper.java | 117 --- .../bolt/ProfileMeasurementEmitter.java | 59 -- .../profiler/bolt/ProfileSplitterBolt.java | 246 ----- .../src/main/resources/META-INF/LICENSE | 604 ------------- .../src/main/resources/META-INF/NOTICE | 92 -- .../src/main/scripts/start_profiler_topology.sh | 22 - .../zookeeper/event-time-test/profiler.json | 19 - .../processing-time-test/profiler.json | 11 - .../zookeeper/profile-with-stats/profiler.json | 12 - .../bolt/FixedFrequencyFlushSignalTest.java | 71 -- .../metron/profiler/bolt/HBaseEmitterTest.java | 120 --- .../metron/profiler/bolt/KafkaEmitterTest.java | 291 ------ .../profiler/bolt/ProfileBuilderBoltTest.java | 378 -------- .../profiler/bolt/ProfileHBaseMapperTest.java | 93 -- .../profiler/bolt/ProfileSplitterBoltTest.java | 452 ---------- .../integration/ConfigUploadComponent.java | 124 --- .../profiler/integration/MessageBuilder.java | 75 -- .../integration/ProfilerIntegrationTest.java | 478 ---------- .../src/test/resources/log4j.properties | 32 - .../src/test/resources/telemetry.json | 100 --- metron-analytics/pom.xml | 4 +- .../common-services/METRON/CURRENT/metainfo.xml | 8 +- .../packaging/docker/deb-docker/pom.xml | 14 +- .../docker/rpm-docker/SPECS/metron.spec | 64 +- .../packaging/docker/rpm-docker/pom.xml | 14 +- .../metron-common/src/main/scripts/stellar | 3 +- .../ZKConfigurationsCacheIntegrationTest.java | 2 +- .../apache/metron/hbase/client/HBaseClient.java | 8 +- .../metron/hbase/client/HBaseClientTest.java | 30 +- pom.xml | 1 + 131 files changed, 11195 insertions(+), 7245 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/metron/blob/fbabfcad/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/metainfo.xml ----------------------------------------------------------------------
