Merge remote-tracking branch 'apache/master' into metron-1699-create-batch-profiler
Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/f5eaef3c Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/f5eaef3c Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/f5eaef3c Branch: refs/heads/feature/METRON-1699-create-batch-profiler Commit: f5eaef3c24e939ef14757eb2db4b758e71d33fca Parents: 113ac0d d32bd50 Author: Nick Allen <n...@nickallen.org> Authored: Fri Aug 31 15:11:49 2018 -0400 Committer: Nick Allen <n...@nickallen.org> Committed: Fri Aug 31 15:11:49 2018 -0400 ---------------------------------------------------------------------- dependencies_with_url.csv | 8 + metron-analytics/metron-profiler-common/pom.xml | 26 +- .../metron/profiler/ProfileMeasurement.java | 35 + .../apache/metron/profiler/ProfilePeriod.java | 11 + .../metron/profiler/ProfileMeasurementTest.java | 28 +- .../metron/profiler/ProfilePeriodTest.java | 22 +- .../profiler/hbase/SaltyRowKeyBuilderTest.java | 11 - metron-analytics/metron-profiler/README.md | 18 +- .../profiler/bolt/ProfileSplitterBolt.java | 22 +- .../profiler/bolt/ProfileSplitterBoltTest.java | 16 + .../integration/ProfilerIntegrationTest.java | 30 +- metron-deployment/Kerberos-manual-setup.md | 4 +- metron-deployment/amazon-ec2/conf/defaults.yml | 1 - metron-deployment/amazon-ec2/playbook.yml | 2 +- .../ansible/playbooks/metron_install.yml | 7 - .../roles/ambari_config/vars/single_node_vm.yml | 1 + .../roles/metron_pcapservice/defaults/main.yml | 28 - .../roles/metron_pcapservice/meta/main.yml | 19 - .../metron_pcapservice/tasks/config-hbase.yml | 26 - .../roles/metron_pcapservice/tasks/main.yml | 25 - .../metron_pcapservice/tasks/pcapservice.yml | 30 - .../templates/metron-pcapservice-logrotate.yml | 27 - .../metron_pcapservice/templates/pcapservice | 84 - .../ubuntu14/ansible/inventory/hosts | 4 - .../CURRENT/configuration/metron-rest-env.xml | 48 + .../package/scripts/params/params_linux.py | 16 + .../package/scripts/params/status_params.py | 7 + .../CURRENT/package/scripts/rest_commands.py | 62 + .../CURRENT/package/scripts/rest_master.py | 9 + .../METRON/CURRENT/package/templates/metron.j2 | 6 + .../METRON/CURRENT/themes/metron_theme.json | 30 + .../docker/rpm-docker/SPECS/metron.spec | 1 + .../metron-alerts/package-lock.json | 3719 ++-- metron-interface/metron-alerts/pom.xml | 6 +- .../metron-alerts/src/app/app-routing.module.ts | 3 +- .../metron-alerts/src/app/app.component.html | 10 +- .../metron-alerts/src/app/app.component.scss | 18 + .../metron-alerts/src/app/app.component.ts | 3 + .../metron-alerts/src/app/app.module.ts | 6 +- .../src/app/pcap/model/pcap-pagination.ts | 21 + .../src/app/pcap/model/pcap-status-response.ts | 24 + .../src/app/pcap/model/pcap.mock.ts | 39 + .../src/app/pcap/model/pcap.request.ts | 29 + .../src/app/pcap/model/pdml.mock.ts | 93 + .../metron-alerts/src/app/pcap/model/pdml.ts | 50 + .../pcap-filters/pcap-filters.component.html | 73 + .../pcap-filters/pcap-filters.component.scss | 70 + .../pcap-filters/pcap-filters.component.spec.ts | 464 + .../pcap/pcap-filters/pcap-filters.component.ts | 167 + .../app/pcap/pcap-list/pcap-list.component.html | 38 + .../app/pcap/pcap-list/pcap-list.component.scss | 21 + .../pcap/pcap-list/pcap-list.component.spec.ts | 98 + .../app/pcap/pcap-list/pcap-list.component.ts | 43 + .../pcap-packet-line.component.html | 19 + .../pcap-packet-line.component.scss | 17 + .../pcap-packet-line.component.spec.ts | 179 + .../pcap-packet-line.component.ts | 56 + .../pcap/pcap-packet/pcap-packet.component.html | 22 + .../pcap/pcap-packet/pcap-packet.component.scss | 36 + .../pcap-packet/pcap-packet.component.spec.ts | 79 + .../pcap/pcap-packet/pcap-packet.component.ts | 34 + .../pcap-pagination.component.html | 19 + .../pcap-pagination.component.scss | 40 + .../pcap-pagination.component.spec.ts | 78 + .../pcap-pagination.component.ts | 41 + .../pcap/pcap-panel/pcap-panel.component.html | 31 + .../pcap/pcap-panel/pcap-panel.component.scss | 67 + .../pcap-panel/pcap-panel.component.spec.ts | 557 + .../app/pcap/pcap-panel/pcap-panel.component.ts | 164 + .../metron-alerts/src/app/pcap/pcap.module.ts | 55 + .../metron-alerts/src/app/pcap/pcap.routing.ts | 27 + .../src/app/pcap/service/pcap.service.spec.ts | 197 + .../src/app/pcap/service/pcap.service.ts | 88 + .../date-picker/date-picker.component.scss | 14 +- .../shared/date-picker/date-picker.component.ts | 34 +- .../shared/date-picker/date-picker.module.ts | 3 +- .../shared/directives/alert-search.directive.ts | 2 +- .../metron-alerts/src/app/utils/constants.ts | 3 + .../metron-alerts/src/app/utils/utils.ts | 12 +- .../src/environments/environment.e2e.ts | 3 +- .../src/environments/environment.prod.ts | 3 +- .../src/environments/environment.ts | 3 +- metron-interface/metron-alerts/src/index.html | 2 +- .../metron-config/package-lock.json | 17734 +++++++++++++++++ metron-interface/metron-config/pom.xml | 30 +- metron-interface/metron-rest-client/pom.xml | 10 + .../apache/metron/rest/model/PcapResponse.java | 38 + .../apache/metron/rest/model/pcap/Field.java | 154 + .../rest/model/pcap/FixedPcapOptions.java | 42 + .../rest/model/pcap/FixedPcapRequest.java | 116 + .../apache/metron/rest/model/pcap/Packet.java | 53 + .../metron/rest/model/pcap/PcapRequest.java | 83 + .../metron/rest/model/pcap/PcapStatus.java | 87 + .../org/apache/metron/rest/model/pcap/Pdml.java | 98 + .../apache/metron/rest/model/pcap/Proto.java | 108 + .../rest/model/pcap/QueryPcapOptions.java | 35 + .../rest/model/pcap/QueryPcapRequest.java | 41 + metron-interface/metron-rest/README.md | 105 + metron-interface/metron-rest/pom.xml | 44 +- .../src/main/config/rest_application.yml | 11 +- .../apache/metron/rest/MetronRestConstants.java | 9 + .../apache/metron/rest/config/PcapConfig.java | 49 + .../metron/rest/config/PcapJobSupplier.java | 54 + .../metron/rest/controller/PcapController.java | 180 + .../apache/metron/rest/service/PcapService.java | 116 + .../rest/service/impl/PcapServiceImpl.java | 299 + .../service/impl/PcapToPdmlScriptWrapper.java | 59 + .../src/main/resources/application.yml | 9 +- .../metron-rest/src/main/scripts/metron-rest.sh | 2 +- .../src/main/scripts/pcap_to_pdml.sh | 19 + .../apache/metron/rest/config/TestConfig.java | 47 +- .../PcapControllerIntegrationTest.java | 474 + .../apache/metron/rest/mock/MockPcapJob.java | 161 + .../metron/rest/mock/MockPcapJobSupplier.java | 36 + .../rest/mock/MockPcapToPdmlScriptWrapper.java | 55 + .../rest/service/impl/PcapServiceImplTest.java | 746 + metron-interface/pom.xml | 3 + metron-platform/metron-api/README.md | 65 - metron-platform/metron-api/pom.xml | 269 - .../apache/metron/api/ConfigurationManager.java | 135 - .../api/helper/service/PcapServiceCli.java | 169 - .../metron/pcapservice/ConfigurationUtil.java | 64 - .../pcapservice/PcapReceiverImplRestEasy.java | 299 - .../metron/pcapservice/PcapsResponse.java | 118 - .../metron/pcapservice/RestTestingUtil.java | 329 - .../pcapservice/rest/JettyServiceRunner.java | 44 - .../metron/pcapservice/rest/PcapService.java | 56 - .../src/main/resources/META-INF/LICENSE | 605 - .../src/main/resources/META-INF/NOTICE | 39 - .../pcapservice/ConfigurationUtilTest.java | 43 - .../PcapReceiverImplRestEasyTest.java | 255 - .../src/test/resources/log4j.properties | 24 - .../src/test/resources/test-tcp-packet.pcap | Bin 144 -> 0 bytes .../common/configuration/ConfigOption.java | 71 + .../configuration/profiler/ProfileResult.java | 8 +- .../profiler/ProfileResultExpressions.java | 8 +- .../apache/metron/common/utils/HDFSUtils.java | 53 +- .../common/configuration/ConfigOptionTest.java | 112 + .../profiler/ProfilerConfigTest.java | 35 +- .../metron/common/utils/HDFSUtilsTest.java | 59 + metron-platform/metron-job/README.md | 26 + .../metron-job_state_statechart_diagram.svg | 14 + .../metron-job_state_statechart_diagram.xml | 14 + metron-platform/metron-job/pom.xml | 39 + .../java/org/apache/metron/job/Finalizer.java | 38 + .../org/apache/metron/job/JobException.java | 31 + .../apache/metron/job/JobNotFoundException.java | 30 + .../java/org/apache/metron/job/JobStatus.java | 124 + .../java/org/apache/metron/job/Pageable.java | 38 + .../apache/metron/job/RuntimeJobException.java | 30 + .../java/org/apache/metron/job/Statusable.java | 82 + .../metron/job/manager/InMemoryJobManager.java | 94 + .../apache/metron/job/manager/JobManager.java | 41 + .../org/apache/metron/job/JobStatusTest.java | 55 + .../job/manager/InMemoryJobManagerTest.java | 208 + .../metron/parsers/cef/CEFParserTest.java | 4 +- metron-platform/metron-pcap-backend/README.md | 4 + metron-platform/metron-pcap-backend/pom.xml | 6 + .../src/main/config/pcap.properties | 2 +- .../org/apache/metron/pcap/query/CliConfig.java | 128 - .../org/apache/metron/pcap/query/CliParser.java | 55 +- .../metron/pcap/query/FixedCliConfig.java | 50 - .../metron/pcap/query/FixedCliParser.java | 10 +- .../org/apache/metron/pcap/query/PcapCli.java | 112 +- .../metron/pcap/query/QueryCliConfig.java | 34 - .../metron/pcap/query/QueryCliParser.java | 10 +- .../apache/metron/pcap/query/ResultsWriter.java | 48 - .../apache/metron/pcap/FixedPcapFilterTest.java | 286 - .../org/apache/metron/pcap/PcapJobTest.java | 49 - .../apache/metron/pcap/QueryPcapFilterTest.java | 228 - .../PcapTopologyIntegrationTest.java | 847 +- .../apache/metron/pcap/query/PcapCliTest.java | 229 +- metron-platform/metron-pcap/pom.xml | 7 +- .../java/org/apache/metron/pcap/PcapHelper.java | 18 +- .../java/org/apache/metron/pcap/PcapPages.java | 86 + .../metron/pcap/config/FixedPcapConfig.java | 46 + .../apache/metron/pcap/config/PcapConfig.java | 156 + .../metron/pcap/config/PcapGlobalDefaults.java | 29 + .../apache/metron/pcap/config/PcapOptions.java | 70 + .../metron/pcap/config/QueryPcapConfig.java | 33 + .../pcap/filter/fixed/FixedPcapFilter.java | 14 +- .../pcap/filter/query/QueryPcapFilter.java | 17 +- .../metron/pcap/finalizer/PcapCliFinalizer.java | 55 + .../metron/pcap/finalizer/PcapFinalizer.java | 190 + .../pcap/finalizer/PcapFinalizerStrategies.java | 47 + .../pcap/finalizer/PcapRestFinalizer.java | 57 + .../metron/pcap/mr/OutputDirFormatter.java | 37 + .../java/org/apache/metron/pcap/mr/PcapJob.java | 426 +- .../metron/pcap/writer/PcapResultsWriter.java | 75 + .../org/apache/metron/pcap/PcapHelperTest.java | 76 + .../org/apache/metron/pcap/PcapPagesTest.java | 76 + .../pcap/filter/fixed/FixedPcapFilterTest.java | 271 +- .../pcap/filter/query/QueryPcapFilterTest.java | 207 +- .../metron/pcap/mr/OutputDirFormatterTest.java | 62 + .../org/apache/metron/pcap/mr/PcapJobTest.java | 323 + metron-platform/pom.xml | 2 +- metron-sensors/pycapa/README.md | 139 +- metron-sensors/pycapa/requirements.txt | 6 +- site-book/bin/generate-md.sh | 2 + 199 files changed, 30846 insertions(+), 6158 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/metron/blob/f5eaef3c/dependencies_with_url.csv ---------------------------------------------------------------------- diff --cc dependencies_with_url.csv index 6b4385b,bf3e382..ffd0fbc --- a/dependencies_with_url.csv +++ b/dependencies_with_url.csv @@@ -174,8 -148,8 +175,10 @@@ com.fasterxml.jackson.datatype:jackson- com.fasterxml.jackson.datatype:jackson-datatype-jdk8:jar:2.9.5:compile,ASLv2,https://github.com/FasterXML/jackson-modules-java8 com.fasterxml.jackson.datatype:jackson-datatype-jsr310:jar:2.9.5:compile,ASLv2,https://github.com/FasterXML/jackson-modules-java8 com.fasterxml.jackson.module:jackson-module-parameter-names:jar:2.9.5:compile,ASLv2,https://github.com/FasterXML/jackson-modules-java8 +com.fasterxml.jackson.module:jackson-module-paranamer:jar:2.7.9:compile,ASLv2,https://github.com/FasterXML/jackson-modules-base +com.fasterxml.jackson.module:jackson-module-scala_2.11:jar:2.6.7.1:compile,ASLv2,https://github.com/FasterXML/jackson-module-scala + com.fasterxml.jackson.module:jackson-module-jaxb-annotations:jar:2.9.5:compile,ASLv2,https://github.com/FasterXML/jackson-modules-java8 + com.fasterxml.woodstox:woodstox-core:jar:5.0.3:compile,ASLv2,https://github.com/FasterXML/woodstox com.fasterxml:classmate:jar:1.3.1:compile,ASLv2,http://github.com/cowtowncoder/java-classmate com.fasterxml:classmate:jar:1.3.4:compile,ASLv2,http://github.com/cowtowncoder/java-classmate com.google.code.gson:gson:jar:2.2.4:compile,The Apache Software License, Version 2.0,http://code.google.com/p/google-gson/ @@@ -243,21 -212,14 +246,26 @@@ commons-text:commons-text:jar:1.1:compi commons-validator:commons-validator:jar:1.4.0:compile,ASLv2,http://commons.apache.org/validator/ commons-validator:commons-validator:jar:1.5.1:compile,ASLv2,http://commons.apache.org/proper/commons-validator/ commons-validator:commons-validator:jar:1.6:compile,ASLv2,http://commons.apache.org/proper/commons-validator/ +et.razorvine:pyrolite:jar:4.13:compile,MIT,https://github.com/irmen/Pyrolite +io.airlift:aircompressor:jar:0.8:compile,ASLv2,https://github.com/airlift/aircompressor io.confluent:kafka-avro-serializer:jar:1.0:compile,ASLv2,https://github.com/confluentinc/schema-registry/ io.confluent:kafka-schema-registry-client:jar:1.0:compile,ASLv2,https://github.com/confluentinc/schema-registry/ +io.dropwizard.metrics:metrics-core:jar:3.1.5:compile,ASLv2,https://github.com/dropwizard/metrics +io.dropwizard.metrics:metrics-graphite:jar:3.1.5:compile,ASLv2,https://github.com/dropwizard/metrics +io.dropwizard.metrics:metrics-json:jar:3.1.5:compile,ASLv2,https://github.com/dropwizard/metrics +io.dropwizard.metrics:metrics-jvm:jar:3.1.5:compile,ASLv2,https://github.com/dropwizard/metrics io.netty:netty-all:jar:4.0.23.Final:compile,ASLv2, io.netty:netty-all:jar:4.0.23.Final:provided,ASLv2, ++<<<<<<< HEAD +io.netty:netty-all:jar:4.1.17.Final:compile,ASLv2, ++======= + io.netty:netty-all:jar:4.1.23.Final:compile,ASLv2, + io.netty:netty:jar:3.10.5.Final:compile,Apache License, Version 2.0,http://netty.io/ ++>>>>>>> apache/master io.netty:netty:jar:3.6.2.Final:compile,Apache License, Version 2.0,http://netty.io/ io.netty:netty:jar:3.7.0.Final:compile,Apache License, Version 2.0,http://netty.io/ +io.netty:netty:jar:3.9.9.Final:compile,Apache License, Version 2.0,http://netty.io/ +io.netty:netty:jar:3.10.5.Final:compile,Apache License, Version 2.0,http://netty.io/ io.thekraken:grok:jar:0.1.0:compile,Apache License, Version 2.0,http://maven.apache.org javax.inject:javax.inject:jar:1:compile,The Apache Software License, Version 2.0,http://code.google.com/p/atinject/ joda-time:joda-time:jar:2.3:compile,Apache 2,http://www.joda.org/joda-time/ http://git-wip-us.apache.org/repos/asf/metron/blob/f5eaef3c/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/ProfileMeasurement.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/metron/blob/f5eaef3c/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/ProfilePeriod.java ---------------------------------------------------------------------- diff --cc metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/ProfilePeriod.java index 78aa796,6c971a7..02a4932 --- a/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/ProfilePeriod.java +++ b/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/ProfilePeriod.java @@@ -47,20 -47,11 +47,31 @@@ public class ProfilePeriod implements S */ private long durationMillis; + /** ++ * A no-arg constructor is required for using Kryo serialization in Storm. Using this ++ * constructor elsewhere is discouraged. ++ * ++ * <p>Use either {@link #fromPeriodId(long, long, TimeUnit)} or {@link #fromTimestamp(long, long, TimeUnit)} ++ * to create a {@link ProfilePeriod}. ++ */ + public ProfilePeriod() { + // no-arg constructor required for kryo serialization in storm + } + + /** + * @param periodId A monotonically increasing number identifying the period. + * @param duration The duration of each profile period. + * @param units The units of the duration; hours, minutes, etc. + */ + private ProfilePeriod(long periodId, long duration, TimeUnit units) { + this.durationMillis = units.toMillis(duration); + this.period = periodId; + } + + /** + * Creates a {@link ProfilePeriod} given a timestamp defined in milliseconds + * from the epoch. + * * @param epochMillis A timestamp contained somewhere within the profile period. * @param duration The duration of each profile period. * @param units The units of the duration; hours, minutes, etc. http://git-wip-us.apache.org/repos/asf/metron/blob/f5eaef3c/metron-analytics/metron-profiler-common/src/test/java/org/apache/metron/profiler/ProfilePeriodTest.java ---------------------------------------------------------------------- diff --cc metron-analytics/metron-profiler-common/src/test/java/org/apache/metron/profiler/ProfilePeriodTest.java index 295117a,17bb8fa..afb6299 --- a/metron-analytics/metron-profiler-common/src/test/java/org/apache/metron/profiler/ProfilePeriodTest.java +++ b/metron-analytics/metron-profiler-common/src/test/java/org/apache/metron/profiler/ProfilePeriodTest.java @@@ -137,12 -141,24 +141,24 @@@ public class ProfilePeriodTest */ @Test public void testKryoSerialization() throws Exception { - ProfilePeriod expected = new ProfilePeriod(AUG2016, 1, TimeUnit.HOURS); + ProfilePeriod expected = ProfilePeriod.fromTimestamp(AUG2016, 1, TimeUnit.HOURS); + Kryo kryo = new Kryo(); - // round-trip java serialization - byte[] raw = SerDeUtils.toBytes(expected); - Object actual = SerDeUtils.fromBytes(raw, Object.class); + // serialize + ByteArrayOutputStream byteStream = new ByteArrayOutputStream(); + Output output = new Output(byteStream); + kryo.writeObject(output, expected); + // validate serialization + byte[] bits = output.toBytes(); + assertNotNull(bits); + + // deserialize + Input input = new Input(new ByteArrayInputStream(bits)); + ProfilePeriod actual = kryo.readObject(input, ProfilePeriod.class); + + // validate deserialization + assertNotNull(actual); assertEquals(expected, actual); } http://git-wip-us.apache.org/repos/asf/metron/blob/f5eaef3c/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/ProfileSplitterBolt.java ---------------------------------------------------------------------- diff --cc metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/ProfileSplitterBolt.java index f57deb7,87f1ba9..f73a9f1 --- a/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/ProfileSplitterBolt.java +++ b/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/ProfileSplitterBolt.java @@@ -151,16 -157,25 +151,20 @@@ public class ProfileSplitterBolt extend // retrieve the input message byte[] data = input.getBinary(0); - JSONObject message = (JSONObject) parser.parse(new String(data, "UTF8")); + if(data == null) { + LOG.debug("Received null message. Nothing to do."); + return; + } - JSONObject message = (JSONObject) parser.parse(new String(data, "UTF8")); // ensure there is a valid profiler configuration ProfilerConfig config = getProfilerConfig(); - if(config != null && config.getProfiles().size() > 0) { - routeMessage(input, message, config); - - } else if(LOG.isDebugEnabled()) { - LOG.debug("No Profiler configuration found. Nothing to do."); + if(config == null || getProfilerConfig().getProfiles().size() == 0) { + LOG.debug("No Profiler configuration found. Nothing to do."); + return; } + - // what time is it? - Clock clock = clockFactory.createClock(config); - Optional<Long> timestamp = clock.currentTimeMillis(message); - - // route the message. if a message does not contain the timestamp field, it cannot be routed. - timestamp.ifPresent(ts -> routeMessage(input, message, config, ts)); ++ JSONObject message = (JSONObject) parser.parse(new String(data, "UTF8")); ++ routeMessage(input, message, config); } /** http://git-wip-us.apache.org/repos/asf/metron/blob/f5eaef3c/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/bolt/ProfileSplitterBoltTest.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/metron/blob/f5eaef3c/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec ----------------------------------------------------------------------