METRON-1136 Track Master in Feature Branch (ottobackwards) closes apache/metron#752
Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/24b668b0 Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/24b668b0 Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/24b668b0 Branch: refs/heads/feature/METRON-1136-extensions-parsers Commit: 24b668b0b4041c7fe251a27bc9c322cc4b201d44 Parents: 1c63c1e Author: ottobackwards <ottobackwa...@gmail.com> Authored: Tue Sep 12 21:47:20 2017 -0400 Committer: otto <o...@apache.org> Committed: Tue Sep 12 21:47:20 2017 -0400 ---------------------------------------------------------------------- .travis.yml | 3 +- dependencies_with_url.csv | 2 + metron-analytics/metron-maas-service/pom.xml | 10 + metron-analytics/metron-profiler-client/pom.xml | 12 +- .../client/stellar/ProfilerFunctions.java | 113 +++-- .../client/stellar/ProfilerFunctionsTest.java | 53 ++- metron-analytics/metron-profiler/pom.xml | 10 + metron-analytics/metron-statistics/pom.xml | 10 + .../CURRENT/configuration/metron-rest-env.xml | 10 +- .../package/scripts/management_ui_master.py | 1 - .../package/scripts/params/params_linux.py | 6 +- .../package/scripts/profiler_commands.py | 4 + .../CURRENT/package/scripts/profiler_master.py | 16 + .../CURRENT/package/scripts/rest_commands.py | 7 +- .../CURRENT/package/scripts/rest_master.py | 1 + .../METRON/CURRENT/package/templates/metron.j2 | 1 + .../METRON/CURRENT/themes/metron_theme.json | 10 + .../packaging/docker/rpm-docker/Dockerfile | 4 + .../docker/rpm-docker/SPECS/metron.spec | 13 +- .../packaging/docker/rpm-docker/build.sh | 2 +- .../packaging/docker/rpm-docker/pom.xml | 6 + .../roles/ambari_config/vars/small_cluster.yml | 1 + .../metron-alerts/alerts-server-e2e.js | 20 +- .../metron-alerts/scripts/alerts-server.js | 14 +- .../alert-details/alert-details.component.ts | 16 +- .../alerts/alerts-list/alerts-list.component.ts | 26 +- .../alerts/alerts-list/alerts-list.module.ts | 4 +- .../configure-table.component.ts | 10 +- .../metron-alerts/src/app/app.module.ts | 4 +- .../src/app/model/alerts-search-response.ts | 23 - .../src/app/model/search-request.ts | 3 +- .../src/app/model/search-response.ts | 23 + .../src/app/service/alert.service.ts | 72 ---- .../src/app/service/cluster-metadata.service.ts | 7 +- .../src/app/service/data-source.ts | 4 +- .../service/elasticsearch-localstorage-impl.ts | 4 +- .../src/app/service/rest-api-impl.ts | 46 -- .../src/app/service/search.service.ts | 112 +++++ .../metron-alerts/src/app/utils/constants.ts | 2 + .../src/app/utils/elasticsearch-utils.ts | 6 +- metron-interface/metron-config/README.md | 10 +- metron-interface/metron-rest/README.md | 45 +- .../src/main/config/rest_application.yml | 2 + .../apache/metron/rest/MetronRestConstants.java | 2 + .../apache/metron/rest/config/IndexConfig.java | 12 +- .../apache/metron/rest/config/KafkaConfig.java | 16 + .../rest/controller/AlertsController.java | 55 +++ .../metron/rest/controller/HdfsController.java | 11 +- .../metron/rest/controller/KafkaController.java | 14 +- .../rest/controller/SearchController.java | 13 +- .../metron/rest/service/AlertService.java | 30 ++ .../apache/metron/rest/service/GrokService.java | 4 +- .../apache/metron/rest/service/HdfsService.java | 6 +- .../metron/rest/service/KafkaService.java | 5 +- .../metron/rest/service/SearchService.java | 5 +- .../rest/service/impl/AlertServiceImpl.java | 62 +++ .../rest/service/impl/ExtensionServiceImpl.java | 4 +- .../rest/service/impl/GrokServiceImpl.java | 35 +- .../rest/service/impl/HdfsServiceImpl.java | 44 +- .../rest/service/impl/KafkaServiceImpl.java | 25 +- .../rest/service/impl/SearchServiceImpl.java | 14 +- .../impl/SensorParserConfigServiceImpl.java | 20 +- .../src/main/resources/application-docker.yml | 1 - .../src/main/resources/application-vagrant.yml | 3 +- .../src/main/resources/application.yml | 11 +- .../apache/metron/rest/config/TestConfig.java | 16 + .../AlertControllerIntegrationTest.java | 86 ++++ .../KafkaControllerIntegrationTest.java | 123 +++--- .../SearchControllerIntegrationTest.java | 18 +- ...orParserConfigControllerIntegrationTest.java | 23 +- .../rest/service/impl/AlertServiceImplTest.java | 69 +++ .../rest/service/impl/GrokServiceImplTest.java | 43 +- .../impl/HdfsServiceImplExceptionTest.java | 12 +- .../rest/service/impl/HdfsServiceImplTest.java | 16 +- .../rest/service/impl/KafkaServiceImplTest.java | 55 ++- .../impl/SensorParserConfigServiceImplTest.java | 9 +- metron-platform/elasticsearch-shaded/pom.xml | 10 + metron-platform/metron-api/pom.xml | 10 + metron-platform/metron-common/pom.xml | 10 + .../configuration/SensorParserConfig.java | 208 ++++++++- metron-platform/metron-data-management/pom.xml | 12 +- metron-platform/metron-elasticsearch/pom.xml | 10 + .../elasticsearch/dao/ElasticsearchDao.java | 252 +++++++---- .../ElasticsearchSearchIntegrationTest.java | 5 +- metron-platform/metron-enrichment/pom.xml | 27 ++ .../adapters/geo/GeoLiteDatabase.java | 77 +++- .../adapters/geo/hash/DistanceStrategies.java | 46 ++ .../adapters/geo/hash/DistanceStrategy.java | 24 ++ .../adapters/geo/hash/GeoHashUtil.java | 189 +++++++++ .../enrichment/stellar/GeoHashFunctions.java | 299 +++++++++++++ .../stellar/GeoHashFunctionsTest.java | 337 +++++++++++++++ metron-platform/metron-hbase-client/pom.xml | 10 + metron-platform/metron-indexing/pom.xml | 10 + .../metron/indexing/dao/AccessConfig.java | 14 +- .../apache/metron/indexing/dao/HBaseDao.java | 7 + .../apache/metron/indexing/dao/IndexDao.java | 5 +- .../metron/indexing/dao/MultiIndexDao.java | 13 + .../metron/indexing/dao/search/Group.java | 43 ++ .../metron/indexing/dao/search/GroupOrder.java | 37 ++ .../indexing/dao/search/GroupOrderType.java | 39 ++ .../indexing/dao/search/GroupRequest.java | 58 +++ .../indexing/dao/search/GroupResponse.java | 39 ++ .../metron/indexing/dao/search/GroupResult.java | 73 ++++ .../apache/metron/indexing/dao/InMemoryDao.java | 22 +- .../indexing/dao/SearchIntegrationTest.java | 247 ++++++++++- metron-platform/metron-management/pom.xml | 10 + .../management/ConfigurationFunctionsTest.java | 10 +- metron-platform/metron-parsers/README.md | 11 + metron-platform/metron-parsers/pom.xml | 10 + .../parsers/topology/ParserTopologyBuilder.java | 87 ++-- .../parsers/topology/ParserTopologyCLI.java | 218 +++++++--- .../parsers/topology/config/ValueSupplier.java | 30 ++ .../components/ParserTopologyComponent.java | 30 +- .../parsers/topology/ParserTopologyCLITest.java | 425 ++++++++++++++++++- metron-platform/metron-pcap-backend/pom.xml | 10 + metron-platform/metron-solr/pom.xml | 10 + metron-platform/metron-writer/pom.xml | 10 + .../metron/writer/hdfs/SourceHandler.java | 21 +- .../metron/writer/hdfs/HdfsWriterTest.java | 91 +++- metron-stellar/stellar-common/README.md | 262 ++++++++++-- metron-stellar/stellar-common/pom.xml | 10 + .../metron/stellar/common/StellarCompiler.java | 8 +- .../stellar/common/shell/StellarExecutor.java | 33 +- .../stellar/common/shell/StellarShell.java | 146 ++++++- .../dsl/functions/DataStructureFunctions.java | 5 + .../stellar/dsl/functions/SetFunctions.java | 287 +++++++++++++ .../stellar/dsl/functions/StringFunctions.java | 167 ++++++++ .../stellar/dsl/functions/BasicStellarTest.java | 12 + .../stellar/dsl/functions/SetFunctionsTest.java | 326 ++++++++++++++ .../dsl/functions/StringFunctionsTest.java | 296 +++++++++++++ pom.xml | 6 +- use-cases/README.md | 4 + use-cases/geographic_login_outliers/README.md | 267 ++++++++++++ 133 files changed, 5740 insertions(+), 805 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/.travis.yml ---------------------------------------------------------------------- diff --git a/.travis.yml b/.travis.yml index d85a7ca..8023830 100644 --- a/.travis.yml +++ b/.travis.yml @@ -31,14 +31,13 @@ before_install: - export PATH=$M2_HOME/bin:$PATH - npm config set cache $HOME/.npm-cache --global - npm config set prefix $HOME/.npm-prefix --global - - time build_utils/verify_licenses.sh install: - cd bundles-maven-plugin && mvn -q install && cd .. - time mvn -q -T 2C -DskipTests clean install script: - - time mvn -q -T 2C surefire:test@unit-tests && time mvn -q surefire:test@integration-tests && time mvn -q test --projects metron-interface/metron-config + - time mvn -q -T 2C surefire:test@unit-tests && time mvn -q surefire:test@integration-tests && time mvn -q test --projects metron-interface/metron-config && time build_utils/verify_licenses.sh before_cache: - rm -rf $HOME/.m2/repository/org/apache/metron http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/dependencies_with_url.csv ---------------------------------------------------------------------- diff --git a/dependencies_with_url.csv b/dependencies_with_url.csv index d09fed6..9ebb9e4 100644 --- a/dependencies_with_url.csv +++ b/dependencies_with_url.csv @@ -307,3 +307,5 @@ net.byteseek:byteseek:jar:2.0.3:compile,BSD,https://github.com/nishihatapalmer/b org.springframework.security.kerberos:spring-security-kerberos-client:jar:1.0.1.RELEASE:compile,ASLv2,https://github.com/spring-projects/spring-security-kerberos org.springframework.security.kerberos:spring-security-kerberos-core:jar:1.0.1.RELEASE:compile,ASLv2,https://github.com/spring-projects/spring-security-kerberos org.springframework.kafka:spring-kafka:jar:1.1.1.RELEASE:compile,ASLv2,https://github.com/spring-projects/spring-kafka +ch.hsr:geohash:jar:1.3.0:compile,ASLv2,https://github.com/kungfoo/geohash-java +org.locationtech.spatial4j:spatial4j:jar:0.6:compile,ASLv2,https://github.com/locationtech/spatial4j http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/metron-analytics/metron-maas-service/pom.xml ---------------------------------------------------------------------- diff --git a/metron-analytics/metron-maas-service/pom.xml b/metron-analytics/metron-maas-service/pom.xml index 4eeceae..555e73d 100644 --- a/metron-analytics/metron-maas-service/pom.xml +++ b/metron-analytics/metron-maas-service/pom.xml @@ -252,6 +252,16 @@ <configuration> <shadedArtifactAttached>true</shadedArtifactAttached> <shadedClassifierName>uber</shadedClassifierName> + <filters> + <filter> + <artifact>*:*</artifact> + <excludes> + <exclude>META-INF/*.SF</exclude> + <exclude>META-INF/*.DSA</exclude> + <exclude>META-INF/*.RSA</exclude> + </excludes> + </filter> + </filters> <transformers> <transformer implementation="org.apache.maven.plugins.shade.resource.DontIncludeResourceTransformer"> http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/metron-analytics/metron-profiler-client/pom.xml ---------------------------------------------------------------------- diff --git a/metron-analytics/metron-profiler-client/pom.xml b/metron-analytics/metron-profiler-client/pom.xml index bba881d..69b8c29 100644 --- a/metron-analytics/metron-profiler-client/pom.xml +++ b/metron-analytics/metron-profiler-client/pom.xml @@ -304,7 +304,17 @@ <pattern>com.google.common</pattern> <shadedPattern>org.apache.metron.guava</shadedPattern> </relocation> - </relocations> + </relocations> + <filters> + <filter> + <artifact>*:*</artifact> + <excludes> + <exclude>META-INF/*.SF</exclude> + <exclude>META-INF/*.DSA</exclude> + <exclude>META-INF/*.RSA</exclude> + </excludes> + </filter> + </filters> <transformers> <transformer implementation="org.apache.maven.plugins.shade.resource.DontIncludeResourceTransformer"> http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/stellar/ProfilerFunctions.java ---------------------------------------------------------------------- diff --git a/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/stellar/ProfilerFunctions.java b/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/stellar/ProfilerFunctions.java index 8df5ca8..d6f2c6a 100644 --- a/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/stellar/ProfilerFunctions.java +++ b/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/stellar/ProfilerFunctions.java @@ -20,6 +20,7 @@ package org.apache.metron.profiler.client.stellar; +import org.apache.commons.collections4.ListUtils; import org.apache.metron.common.configuration.profiler.ProfilerConfig; import org.apache.metron.common.utils.JSONUtils; import org.apache.metron.profiler.ProfileMeasurement; @@ -41,6 +42,7 @@ import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; @@ -109,7 +111,7 @@ public class ProfilerFunctions { name="APPLY", description="Apply a message to a local profile runner.", params={ - "message(s)", "The message to apply. A JSON list can be used to apply multiple messages.", + "message(s)", "The message to apply; a JSON string or list of JSON strings.", "profiler", "A local profile runner returned by PROFILER_INIT." }, returns="The local profile runner." @@ -131,50 +133,99 @@ public class ProfilerFunctions { @Override public Object apply(List<Object> args, Context context) throws ParseException { - // user must provide the message as a string - String arg0 = Util.getArg(0, String.class, args); - if(arg0 == null) { - throw new IllegalArgumentException(format("expected string, found null")); + // the use can pass in one or more messages in a few different forms + Object arg0 = Util.getArg(0, Object.class, args); + List<JSONObject> messages = getMessages(arg0); + + // user must provide the stand alone profiler + StandAloneProfiler profiler = Util.getArg(1, StandAloneProfiler.class, args); + try { + for (JSONObject message : messages) { + profiler.apply(message); + } + + } catch (ExecutionException e) { + throw new IllegalArgumentException(format("Failed to apply message; error=%s", e.getMessage()), e); + } + + return profiler; + } + + /** + * Gets a message or messages from the function arguments. + * + * @param arg The function argument containing the message(s). + * @return A list of messages + */ + private List<JSONObject> getMessages(Object arg) { + List<JSONObject> messages; + + if (arg instanceof String) { + messages = getMessagesFromString((String) arg); + + } else if (arg instanceof Iterable) { + messages = getMessagesFromIterable((Iterable<String>) arg); + + } else if (arg instanceof JSONObject) { + messages = Collections.singletonList((JSONObject) arg); + + } else { + throw new IllegalArgumentException(format("invalid message: found '%s', expected String, List, or JSONObject", + ClassUtils.getShortClassName(arg, "null"))); } - // there could be one or more messages + return messages; + } + + /** + * Gets a message or messages from a List + * + * @param strings The function argument that is a bunch of strings. + * @return A list of messages. + */ + private List<JSONObject> getMessagesFromIterable(Iterable<String> strings) { List<JSONObject> messages = new ArrayList<>(); + + // the user pass in a list of strings + for (String str : strings) { + messages.addAll(getMessagesFromString(str)); + } + + return messages; + } + + /** + * Gets a message or messages from a String argument. + * + * @param arg0 The function argument is just a List. + * @return A list of messages. + */ + private List<JSONObject> getMessagesFromString(String arg0) { + List<JSONObject> messages = new ArrayList<>(); + try { Object parsedArg0 = parser.parse(arg0); - if(parsedArg0 instanceof JSONObject) { - // if there is only one message + if (parsedArg0 instanceof JSONObject) { + // if the string only contains one message messages.add((JSONObject) parsedArg0); - } else if(parsedArg0 instanceof JSONArray) { - // there are multiple messages + } else if (parsedArg0 instanceof JSONArray) { + // if the string contains multiple messages JSONArray jsonArray = (JSONArray) parsedArg0; - for(Object json: jsonArray) { - if(json instanceof JSONObject) { - messages.add((JSONObject) json); - - } else { - throw new IllegalArgumentException(format("invalid message: found '%s', expected JSONObject", - ClassUtils.getShortClassName(json, "null"))); - } + for (Object item : jsonArray) { + messages.addAll(getMessages(item)); } - } - } catch(org.json.simple.parser.ParseException e) { - throw new IllegalArgumentException("invalid message", e); - } - - // user must provide the stand alone profiler - StandAloneProfiler profiler = Util.getArg(1, StandAloneProfiler.class, args); - try { - for(JSONObject message : messages) { - profiler.apply(message); + } else { + throw new IllegalArgumentException(format("invalid message: found '%s', expected JSONObject or JSONArray", + ClassUtils.getShortClassName(parsedArg0, "null"))); } - } catch(ExecutionException e) { - throw new IllegalArgumentException(format("Failed to apply message; error=%s", e.getMessage()), e); + } catch (org.json.simple.parser.ParseException e) { + throw new IllegalArgumentException(format("invalid message: '%s'", e.getMessage()), e); } - return profiler; + return messages; } } http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/metron-analytics/metron-profiler-client/src/test/java/org/apache/metron/profiler/client/stellar/ProfilerFunctionsTest.java ---------------------------------------------------------------------- diff --git a/metron-analytics/metron-profiler-client/src/test/java/org/apache/metron/profiler/client/stellar/ProfilerFunctionsTest.java b/metron-analytics/metron-profiler-client/src/test/java/org/apache/metron/profiler/client/stellar/ProfilerFunctionsTest.java index bad3efe..e1c6aa8 100644 --- a/metron-analytics/metron-profiler-client/src/test/java/org/apache/metron/profiler/client/stellar/ProfilerFunctionsTest.java +++ b/metron-analytics/metron-profiler-client/src/test/java/org/apache/metron/profiler/client/stellar/ProfilerFunctionsTest.java @@ -21,13 +21,13 @@ package org.apache.metron.profiler.client.stellar; import org.adrianwalker.multilinestring.Multiline; -import org.apache.metron.profiler.ProfileMeasurement; import org.apache.metron.profiler.StandAloneProfiler; import org.apache.metron.stellar.common.DefaultStellarStatefulExecutor; import org.apache.metron.stellar.common.StellarStatefulExecutor; -import org.apache.metron.stellar.common.shell.StellarExecutor; import org.apache.metron.stellar.dsl.Context; import org.apache.metron.stellar.dsl.functions.resolver.SimpleFunctionResolver; +import org.json.simple.JSONObject; +import org.json.simple.parser.JSONParser; import org.junit.Before; import org.junit.Test; @@ -176,7 +176,7 @@ public class ProfilerFunctionsTest { } @Test - public void testProfilerApply() { + public void testProfilerApplyWithString() { // initialize the profiler state.put("config", helloWorldProfilerDef); @@ -195,7 +195,28 @@ public class ProfilerFunctionsTest { } @Test - public void testProfilerApplyWithMultipleMessages() { + public void testProfilerApplyWithJSONObject() throws Exception { + + // initialize the profiler + state.put("config", helloWorldProfilerDef); + StandAloneProfiler profiler = run("PROFILER_INIT(config)", StandAloneProfiler.class); + state.put("profiler", profiler); + + // apply a message to the profiler + JSONParser parser = new JSONParser(); + JSONObject jsonObject = (JSONObject) parser.parse(message); + state.put("jsonObj", jsonObject); + StandAloneProfiler result = run("PROFILER_APPLY(jsonObj, profiler)", StandAloneProfiler.class); + + // validate + assertSame(profiler, result); + assertEquals(1, profiler.getProfileCount()); + assertEquals(1, profiler.getMessageCount()); + assertEquals(1, profiler.getRouteCount()); + } + + @Test + public void testProfilerApplyWithMultipleMessagesInJSONString() { // initialize the profiler state.put("config", helloWorldProfilerDef); @@ -214,6 +235,26 @@ public class ProfilerFunctionsTest { } @Test + public void testProfilerApplyWithListOfMessages() { + + // initialize the profiler + state.put("config", helloWorldProfilerDef); + StandAloneProfiler profiler = run("PROFILER_INIT(config)", StandAloneProfiler.class); + state.put("profiler", profiler); + + // apply a message to the profiler + state.put("msg", message); + StandAloneProfiler result = run("PROFILER_APPLY([msg, msg, msg], profiler)", StandAloneProfiler.class); + + // validate + assertSame(profiler, result); + assertEquals(1, profiler.getProfileCount()); + assertEquals(3, profiler.getMessageCount()); + assertEquals(3, profiler.getRouteCount()); + } + + + @Test public void testProfilerApplyWithEmptyList() { // initialize the profiler @@ -250,8 +291,8 @@ public class ProfilerFunctionsTest { StandAloneProfiler profiler = run("PROFILER_INIT(config)", StandAloneProfiler.class); state.put("profiler", profiler); - // there is no 'messages' variable - StandAloneProfiler result = run("PROFILER_APPLY(messages, profiler)", StandAloneProfiler.class); + // there is no 'messages' variable - should throw exception + run("PROFILER_APPLY(messages, profiler)", StandAloneProfiler.class); } @Test http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/metron-analytics/metron-profiler/pom.xml ---------------------------------------------------------------------- diff --git a/metron-analytics/metron-profiler/pom.xml b/metron-analytics/metron-profiler/pom.xml index 41888a1..e1ee806 100644 --- a/metron-analytics/metron-profiler/pom.xml +++ b/metron-analytics/metron-profiler/pom.xml @@ -305,6 +305,16 @@ <configuration> <shadedArtifactAttached>true</shadedArtifactAttached> <shadedClassifierName>uber</shadedClassifierName> + <filters> + <filter> + <artifact>*:*</artifact> + <excludes> + <exclude>META-INF/*.SF</exclude> + <exclude>META-INF/*.DSA</exclude> + <exclude>META-INF/*.RSA</exclude> + </excludes> + </filter> + </filters> <relocations> <relocation> <pattern>com.google.common</pattern> http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/metron-analytics/metron-statistics/pom.xml ---------------------------------------------------------------------- diff --git a/metron-analytics/metron-statistics/pom.xml b/metron-analytics/metron-statistics/pom.xml index 5fab63e..b4d2ed6 100644 --- a/metron-analytics/metron-statistics/pom.xml +++ b/metron-analytics/metron-statistics/pom.xml @@ -74,6 +74,16 @@ <goal>shade</goal> </goals> <configuration> + <filters> + <filter> + <artifact>*:*</artifact> + <excludes> + <exclude>META-INF/*.SF</exclude> + <exclude>META-INF/*.DSA</exclude> + <exclude>META-INF/*.RSA</exclude> + </excludes> + </filter> + </filters> <relocations> <relocation> <pattern>com.tdunning</pattern> http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-rest-env.xml ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-rest-env.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-rest-env.xml index c68f5b2..c25208d 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-rest-env.xml +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-rest-env.xml @@ -90,8 +90,8 @@ </property> <property> <name>metron_temp_grok_path</name> - <description>Temporary local file path where grok patterns are written during testing</description> - <value>./patterns/temp</value> + <description>Temporary HDFS file path where grok patterns are written during testing</description> + <value>{{metron_apps_hdfs_dir}}/patterns/tmp</value> <display-name>Metron temp grok path</display-name> </property> <property> @@ -109,4 +109,10 @@ <empty-value-valid>true</empty-value-valid> </value-attributes> </property> + <property> + <name>metron_escalation_topic</name> + <description>Escalated alerts will be produced to this topic</description> + <value>escalation</value> + <display-name>Metron escalation topic</display-name> + </property> </configuration> http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/management_ui_master.py ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/management_ui_master.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/management_ui_master.py index e858ed3..54e91aa 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/management_ui_master.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/management_ui_master.py @@ -38,7 +38,6 @@ class ManagementUIMaster(Script): from params import params env.set_params(params) self.install_packages(env) - Execute('npm --prefix ' + params.metron_home + '/web/expressjs/ install') def configure(self, env, upgrade_type=None, config_dir=None): print 'configure managment_ui' http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py index abbddc5..2796674 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py @@ -63,9 +63,9 @@ metron_jdbc_username = config['configurations']['metron-rest-env']['metron_jdbc_ metron_jdbc_password = config['configurations']['metron-rest-env']['metron_jdbc_password'] metron_jdbc_platform = config['configurations']['metron-rest-env']['metron_jdbc_platform'] metron_jdbc_client_path = config['configurations']['metron-rest-env']['metron_jdbc_client_path'] -metron_temp_grok_path = config['configurations']['metron-rest-env']['metron_temp_grok_path'] metron_default_grok_path = config['configurations']['metron-rest-env']['metron_default_grok_path'] metron_spring_options = config['configurations']['metron-rest-env']['metron_spring_options'] +metron_escalation_topic = config['configurations']['metron-rest-env']['metron_escalation_topic'] metron_config_path = metron_home + '/config' metron_zookeeper_config_dir = status_params.metron_zookeeper_config_dir metron_zookeeper_config_path = status_params.metron_zookeeper_config_path @@ -139,6 +139,10 @@ if has_kafka_host: metron_apps_hdfs_dir = config['configurations']['metron-env']['metron_apps_hdfs_dir'] +# the double "format" is not an error - we are pulling in a jinja-templated param. This is a bit of a hack, but works +# well enough until we find a better way via Ambari +metron_temp_grok_path = format(format(config['configurations']['metron-rest-env']['metron_temp_grok_path'])) + metron_topic_retention = config['configurations']['metron-env']['metron_topic_retention'] local_grok_patterns_dir = format("{metron_home}/patterns") http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/profiler_commands.py ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/profiler_commands.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/profiler_commands.py index ddd66cb..4dfa371 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/profiler_commands.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/profiler_commands.py @@ -98,6 +98,10 @@ class ProfilerCommands: Logger.info("Done creating HBase Tables") self.set_hbase_configured() + def init_kafka_acls(self): + Logger.info('Creating Kafka ACls for profiler') + metron_service.init_kafka_acls(self.__params, [self.__profiler_topic], ['profiler']) + def set_hbase_acls(self): Logger.info("Setting HBase ACLs") if self.__params.security_enabled: http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/profiler_master.py ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/profiler_master.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/profiler_master.py index 4946ab0..77c32f0 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/profiler_master.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/profiler_master.py @@ -27,6 +27,7 @@ from resource_management.libraries.script import Script from metron_security import storm_security_setup import metron_service +import metron_security from profiler_commands import ProfilerCommands @@ -59,6 +60,9 @@ class Profiler(Script): commands.create_hbase_tables() if params.security_enabled and not commands.is_hbase_acl_configured(): commands.set_hbase_acls() + if params.security_enabled and not commands.is_acl_configured(): + commands.init_kafka_acls() + commands.set_acl_configured() Logger.info("Calling security setup") storm_security_setup(params) @@ -68,6 +72,18 @@ class Profiler(Script): env.set_params(params) self.configure(env) commands = ProfilerCommands(params) + if params.security_enabled: + metron_security.kinit(params.kinit_path_local, + params.metron_keytab_path, + params.metron_principal_name, + execute_user=params.metron_user) + + if params.security_enabled and not commands.is_hbase_acl_configured(): + commands.set_hbase_acls() + if params.security_enabled and not commands.is_acl_configured(): + commands.init_kafka_acls() + commands.set_acl_configured() + commands.start_profiler_topology(env) def stop(self, env, upgrade_type=None): http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_commands.py ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_commands.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_commands.py index fe5fa6e..cf29a28 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_commands.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_commands.py @@ -45,10 +45,15 @@ class RestCommands: owner=self.__params.metron_user, mode=0755) + def init_kafka_topics(self): + Logger.info('Creating Kafka topics for rest') + topics = [self.__params.metron_escalation_topic] + metron_service.init_kafka_topics(self.__params, topics) + def init_kafka_acls(self): Logger.info('Creating Kafka ACLs for rest') # The following topics must be permissioned for the rest application list operation - topics = [self.__params.ambari_kafka_service_check_topic, self.__params.consumer_offsets_topic] + topics = [self.__params.ambari_kafka_service_check_topic, self.__params.consumer_offsets_topic, self.__params.metron_escalation_topic] metron_service.init_kafka_acls(self.__params, topics, ['metron-rest']) def start_rest_application(self): http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_master.py ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_master.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_master.py index 9331759..2f419df 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_master.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_master.py @@ -44,6 +44,7 @@ class RestMaster(Script): ) commands = RestCommands(params) + commands.init_kafka_topics() if params.security_enabled and not commands.is_acl_configured(): commands.init_kafka_acls() commands.set_acl_configured() http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/metron.j2 ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/metron.j2 b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/metron.j2 index 7bef3fa..26daa05 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/metron.j2 +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/metron.j2 @@ -41,3 +41,4 @@ SECURITY_ENABLED={{security_enabled|lower}} {% endif %} KAFKA_SECURITY_PROTOCOL="{{kafka_security_protocol}}" PARSER_TOPOLOGY_OPTIONS="/home/{{metron_user}}/.storm/storm.config" +METRON_ESCALATION_TOPIC="{{metron_escalation_topic}}" http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/themes/metron_theme.json ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/themes/metron_theme.json b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/themes/metron_theme.json index 748feb8..207051d 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/themes/metron_theme.json +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/themes/metron_theme.json @@ -617,6 +617,10 @@ "subsection-name": "subsection-rest" }, { + "config": "metron-rest-env/metron_escalation_topic", + "subsection-name": "subsection-rest" + }, + { "config": "metron-management-ui-env/metron_management_ui_port", "subsection-name": "subsection-management-ui" } @@ -1041,6 +1045,12 @@ } }, { + "config": "metron-rest-env/metron_escalation_topic", + "widget": { + "type": "text-field" + } + }, + { "config": "metron-management-ui-env/metron_management_ui_port", "widget": { "type": "text-field" http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/metron-deployment/packaging/docker/rpm-docker/Dockerfile ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/docker/rpm-docker/Dockerfile b/metron-deployment/packaging/docker/rpm-docker/Dockerfile index 2fd2565..a2dae8e 100644 --- a/metron-deployment/packaging/docker/rpm-docker/Dockerfile +++ b/metron-deployment/packaging/docker/rpm-docker/Dockerfile @@ -27,3 +27,7 @@ RUN mv apache-maven-3.2.5 /opt/maven RUN ln -s /opt/maven/bin/mvn /usr/bin/mvn RUN yum -y install asciidoc rpm-build rpm2cpio tar unzip xmlto zip rpmlint && yum clean all WORKDIR /root + +# install node so that the node dependencies can be packaged into the RPMs +RUN curl --silent --location https://rpm.nodesource.com/setup_6.x | bash - +RUN yum -y install gcc-c++ make nodejs http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec b/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec index c617842..3dd9b9b 100644 --- a/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec +++ b/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec @@ -17,7 +17,7 @@ %define timestamp %(date +%Y%m%d%H%M) %define version %{?_version}%{!?_version:UNKNOWN} %define full_version %{version}%{?_prerelease} -%define prerelease_fmt %{?_prerelease:.%{_prerelease}} +%define prerelease_fmt %{?_prerelease:.%{_prerelease}} %define vendor_version %{?_vendor_version}%{!?_vendor_version: UNKNOWN} %define url http://metron.apache.org/ %define base_name metron @@ -36,6 +36,8 @@ %define metron_extensions_alt_etc %{metron_home}/extension_alt_etc %define metron_extensions_alt_etc_parsers %{metron_extensions_alt_etc}/parsers +%define _binaries_in_noarch_packages_terminate_build 0 + # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Name: %{base_name} @@ -158,6 +160,9 @@ mv %{buildroot}%{metron_extensions_etc_parsers}/websphere/lib/*.bundle %{buildro install %{buildroot}%{metron_home}/bin/metron-rest %{buildroot}/etc/init.d/ install %{buildroot}%{metron_home}/bin/metron-management-ui %{buildroot}/etc/init.d/ +# allows node dependencies to be packaged in the RPMs +npm install --prefix="%{buildroot}%{metron_home}/web/expressjs" --only=production + # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ %package common @@ -833,6 +838,8 @@ This package installs the Metron Management UI %{metron_home} %dir %{metron_home}/bin %dir %{metron_home}/web %dir %{metron_home}/web/expressjs +%dir %{metron_home}/web/expressjs/node_modules +%dir %{metron_home}/web/expressjs/node_modules/.bin %dir %{metron_home}/web/management-ui %dir %{metron_home}/web/management-ui/assets %dir %{metron_home}/web/management-ui/assets/ace @@ -843,6 +850,8 @@ This package installs the Metron Management UI %{metron_home} %dir %{metron_home}/web/management-ui/license %{metron_home}/bin/metron-management-ui /etc/init.d/metron-management-ui +%attr(0755,root,root) %{metron_home}/web/expressjs/node_modules/* +%attr(0755,root,root) %{metron_home}/web/expressjs/node_modules/.bin/* %attr(0755,root,root) %{metron_home}/web/expressjs/server.js %attr(0644,root,root) %{metron_home}/web/expressjs/package.json %attr(0644,root,root) %{metron_home}/web/management-ui/favicon.ico @@ -909,7 +918,7 @@ This package install the Metron MaaS Service files %{metron_home} * Thu Jan 19 2017 Justin Leet <justinjl...@gmail.com> - 0.3.1 - Replace GeoIP files with new implementation * Thu Nov 03 2016 David Lyle <dlyle65...@gmail.com> - 0.2.1 -- Add ASA parser/enrichment configuration files +- Add ASA parser/enrichment configuration files * Thu Jul 21 2016 Michael Miklavcic <michael.miklav...@gmail.com> - 0.2.1 - Remove parser flux files - Add new enrichment files http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/metron-deployment/packaging/docker/rpm-docker/build.sh ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/docker/rpm-docker/build.sh b/metron-deployment/packaging/docker/rpm-docker/build.sh index 23ed90d..dd5149d 100755 --- a/metron-deployment/packaging/docker/rpm-docker/build.sh +++ b/metron-deployment/packaging/docker/rpm-docker/build.sh @@ -36,7 +36,7 @@ if [ $? -ne 0 ] && [ $OWNER_UID -ne 0 ]; then fi rm -rf SRPMS/ RPMS/ && \ -rpmbuild -v -ba --define "_topdir $(pwd)" --define "_version ${VERSION}" --define "_prerelease ${PRERELEASE}" SPECS/metron.spec && \ +QA_SKIP_BUILD_ROOT=1 rpmbuild -v -ba --define "_topdir $(pwd)" --define "_version ${VERSION}" --define "_prerelease ${PRERELEASE}" SPECS/metron.spec && \ rpmlint -i SPECS/metron.spec RPMS/*/metron* SRPMS/metron # Ensure original user permissions are maintained after build http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/metron-deployment/packaging/docker/rpm-docker/pom.xml ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/docker/rpm-docker/pom.xml b/metron-deployment/packaging/docker/rpm-docker/pom.xml index db5f288..b111b94 100644 --- a/metron-deployment/packaging/docker/rpm-docker/pom.xml +++ b/metron-deployment/packaging/docker/rpm-docker/pom.xml @@ -258,6 +258,12 @@ <include>*.tar.gz</include> </includes> </resource> + <resource> + <directory>${metron_dir}/metron-analytics/metron-maas-service/target/</directory> + <includes> + <include>*.tar.gz</include> + </includes> + </resource> </resources> </configuration> </execution> http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/metron-deployment/roles/ambari_config/vars/small_cluster.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/roles/ambari_config/vars/small_cluster.yml b/metron-deployment/roles/ambari_config/vars/small_cluster.yml index 45b15f3..51e0455 100644 --- a/metron-deployment/roles/ambari_config/vars/small_cluster.yml +++ b/metron-deployment/roles/ambari_config/vars/small_cluster.yml @@ -100,6 +100,7 @@ required_configurations: storm_rest_addr: "http://{{ groups.ambari_slave[1] }}:8744" es_hosts: "{{ groups.web[0] }},{{ groups.search | join(',') }}" zeppelin_server_url: "{{ groups.zeppelin[0] }}" + - metron-rest-env: metron_jdbc_driver: "org.h2.Driver" metron_jdbc_url: "jdbc:h2:file:~/metrondb" metron_jdbc_username: "root" http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/metron-interface/metron-alerts/alerts-server-e2e.js ---------------------------------------------------------------------- diff --git a/metron-interface/metron-alerts/alerts-server-e2e.js b/metron-interface/metron-alerts/alerts-server-e2e.js index 2a5f80b..f1b6410 100644 --- a/metron-interface/metron-alerts/alerts-server-e2e.js +++ b/metron-interface/metron-alerts/alerts-server-e2e.js @@ -30,7 +30,7 @@ var favicon = require('serve-favicon'); var proxy = require('http-proxy-middleware'); var argv = require('optimist') .demand(['p', 'r']) - .usage('Usage: server.js -p [port]') + .usage('Usage: alerts-server-e2e.js -p [port]') .describe('p', 'Port to run metron alerts ui') .describe('r', 'Url where metron rest application is available') .argv; @@ -40,7 +40,7 @@ var metronUIAddress = ''; var ifaces = os.networkInterfaces(); var restUrl = argv.r || argv.resturl; var conf = { - "elastic": { + "restapi": { "target": restUrl, "secure": false } @@ -134,24 +134,28 @@ var clusterState = function(req, res){ app.use(compression()); -app.use(bodyParser.json()); + app.use(favicon(path.join(__dirname, 'dist/favicon.ico'))); app.use(serveStatic(path.join(__dirname, 'dist'), { maxAge: '1d', setHeaders: setCustomCacheControl })); -app.use('/api/v1/user', proxy(conf.elastic)); -app.use('/logout', proxy(conf.elastic)); -app.post('/api/v1/search/search', searchResult); -app.use('/_cluster', clusterState); +app.use('/logout', proxy(conf.restapi)); +app.use('/api/v1/user', proxy(conf.restapi)); +app.use('/api/v1/search/findOne', proxy(conf.restapi)); +app.use('/api/v1/search/column/metadata', proxy(conf.restapi)); + app.get('/alerts-list', indexHTML); app.get('', indexHTML); + +app.use(bodyParser.json()); +app.post('/api/v1/search/search', searchResult); + app.use(function(req, res, next){ res.status(404).sendStatus(304); }); - app.listen(port, function(){ console.log("Metron alerts ui is listening on " + metronUIAddress); }); http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/metron-interface/metron-alerts/scripts/alerts-server.js ---------------------------------------------------------------------- diff --git a/metron-interface/metron-alerts/scripts/alerts-server.js b/metron-interface/metron-alerts/scripts/alerts-server.js index 6fb35c9..6a999f2 100644 --- a/metron-interface/metron-alerts/scripts/alerts-server.js +++ b/metron-interface/metron-alerts/scripts/alerts-server.js @@ -22,15 +22,13 @@ var os = require('os'); var app = require('express')(); var path = require('path'); -var compression = require('compression') +var compression = require('compression'); var serveStatic = require('serve-static'); var favicon = require('serve-favicon'); var proxy = require('http-proxy-middleware'); var argv = require('optimist') - .demand(['p', 'r', 'e']) - .alias('r', 'resturl') - .alias('e', 'elasticurl') - .usage('Usage: server.js -p [port] -r [restUrl] -e [elasticURL]') + .demand(['p', 'r']) + .usage('Usage: alert-server.js -p [port] -r [restUrl]') .describe('p', 'Port to run metron management ui') .describe('r', 'Url where metron rest application is available') .argv; @@ -39,12 +37,7 @@ var port = argv.p; var metronUIAddress = ''; var ifaces = os.networkInterfaces(); var restUrl = argv.r || argv.resturl; -var elasticUrl = argv.e || argv.elasticurl; var conf = { - "elastic": { - "target": elasticUrl, - "secure": false - }, "rest": { "target": restUrl, "secure": false @@ -78,7 +71,6 @@ var rewriteSearchProxy = proxy({ app.use(compression()); app.use('/api', proxy(conf.rest)); -app.use('/_cluster', proxy(conf.elastic)); app.use(favicon(path.join(__dirname, '../alerts-ui/favicon.ico'))); http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/metron-interface/metron-alerts/src/app/alerts/alert-details/alert-details.component.ts ---------------------------------------------------------------------- diff --git a/metron-interface/metron-alerts/src/app/alerts/alert-details/alert-details.component.ts b/metron-interface/metron-alerts/src/app/alerts/alert-details/alert-details.component.ts index 915c0c1..2e7884e 100644 --- a/metron-interface/metron-alerts/src/app/alerts/alert-details/alert-details.component.ts +++ b/metron-interface/metron-alerts/src/app/alerts/alert-details/alert-details.component.ts @@ -17,7 +17,7 @@ */ import { Component, OnInit } from '@angular/core'; import {Router, ActivatedRoute} from '@angular/router'; -import {AlertService} from '../../service/alert.service'; +import {SearchService} from '../../service/search.service'; import {Alert} from '../../model/alert'; import {WorkflowService} from '../../service/workflow.service'; import {AlertSource} from '../../model/alert-source'; @@ -42,7 +42,7 @@ export class AlertDetailsComponent implements OnInit { constructor(private router: Router, private activatedRoute: ActivatedRoute, - private alertsService: AlertService, + private searchService: SearchService, private workflowService: WorkflowService) { } goBack() { @@ -51,7 +51,7 @@ export class AlertDetailsComponent implements OnInit { } getData() { - this.alertsService.getAlert(this.alertSourceType, this.alertId).subscribe(alert => { + this.searchService.getAlert(this.alertSourceType, this.alertId).subscribe(alert => { this.alertSource = alert; this.alertFields = Object.keys(alert).filter(field => !field.includes(':ts') && field !== 'original_string').sort(); }); @@ -70,7 +70,7 @@ export class AlertDetailsComponent implements OnInit { tAlert.source = this.alertSource; this.selectedAlertState = AlertState.OPEN; - this.alertsService.updateAlertState([tAlert], 'OPEN', '').subscribe(results => { + this.searchService.updateAlertState([tAlert], 'OPEN', '').subscribe(results => { this.getData(); }); } @@ -80,7 +80,7 @@ export class AlertDetailsComponent implements OnInit { tAlert.source = this.alertSource; this.selectedAlertState = AlertState.NEW; - this.alertsService.updateAlertState([tAlert], 'NEW', '').subscribe(results => { + this.searchService.updateAlertState([tAlert], 'NEW', '').subscribe(results => { this.getData(); }); } @@ -91,7 +91,7 @@ export class AlertDetailsComponent implements OnInit { this.selectedAlertState = AlertState.ESCALATE; this.workflowService.start([tAlert]).subscribe(workflowId => { - this.alertsService.updateAlertState([tAlert], 'ESCALATE', workflowId).subscribe(results => { + this.searchService.updateAlertState([tAlert], 'ESCALATE', workflowId).subscribe(results => { this.getData(); }); }); @@ -102,7 +102,7 @@ export class AlertDetailsComponent implements OnInit { tAlert.source = this.alertSource; this.selectedAlertState = AlertState.DISMISS; - this.alertsService.updateAlertState([tAlert], 'DISMISS', '').subscribe(results => { + this.searchService.updateAlertState([tAlert], 'DISMISS', '').subscribe(results => { this.getData(); }); } @@ -112,7 +112,7 @@ export class AlertDetailsComponent implements OnInit { tAlert.source = this.alertSource; this.selectedAlertState = AlertState.RESOLVE; - this.alertsService.updateAlertState([tAlert], 'RESOLVE', '').subscribe(results => { + this.searchService.updateAlertState([tAlert], 'RESOLVE', '').subscribe(results => { this.getData(); }); } http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/metron-interface/metron-alerts/src/app/alerts/alerts-list/alerts-list.component.ts ---------------------------------------------------------------------- diff --git a/metron-interface/metron-alerts/src/app/alerts/alerts-list/alerts-list.component.ts b/metron-interface/metron-alerts/src/app/alerts/alerts-list/alerts-list.component.ts index 35cbeff..72046fc 100644 --- a/metron-interface/metron-alerts/src/app/alerts/alerts-list/alerts-list.component.ts +++ b/metron-interface/metron-alerts/src/app/alerts/alerts-list/alerts-list.component.ts @@ -20,7 +20,7 @@ import {Router, NavigationStart} from '@angular/router'; import {Observable, Subscription} from 'rxjs/Rx'; import {Alert} from '../../model/alert'; -import {AlertService} from '../../service/alert.service'; +import {SearchService} from '../../service/search.service'; import {QueryBuilder} from './query-builder'; import {ConfigureTableService} from '../../service/configure-table.service'; import {WorkflowService} from '../../service/workflow.service'; @@ -35,7 +35,7 @@ import {SaveSearch} from '../../model/save-search'; import {TableMetadata} from '../../model/table-metadata'; import {MetronDialogBox, DialogType} from '../../shared/metron-dialog-box'; import {AlertSearchDirective} from '../../shared/directives/alert-search.directive'; -import {AlertsSearchResponse} from '../../model/alerts-search-response'; +import {SearchResponse} from '../../model/search-response'; import {ElasticsearchUtils} from '../../utils/elasticsearch-utils'; @Component({ @@ -65,7 +65,7 @@ export class AlertsListComponent implements OnInit, OnDestroy { queryBuilder: QueryBuilder = new QueryBuilder(); constructor(private router: Router, - private alertsService: AlertService, + private searchService: SearchService, private configureTableService: ConfigureTableService, private workflowService: WorkflowService, private clusterMetaDataService: ClusterMetaDataService, @@ -204,7 +204,7 @@ export class AlertsListComponent implements OnInit, OnDestroy { } onConfigRowsChange() { - this.alertsService.interval = this.refreshInterval; + this.searchService.interval = this.refreshInterval; this.search(); } @@ -253,26 +253,26 @@ export class AlertsListComponent implements OnInit, OnDestroy { processEscalate() { this.workflowService.start(this.selectedAlerts).subscribe(workflowId => { - this.alertsService.updateAlertState(this.selectedAlerts, 'ESCALATE', workflowId).subscribe(results => { + this.searchService.updateAlertState(this.selectedAlerts, 'ESCALATE', workflowId).subscribe(results => { this.updateSelectedAlertStatus('ESCALATE'); }); }); } processDismiss() { - this.alertsService.updateAlertState(this.selectedAlerts, 'DISMISS', '').subscribe(results => { + this.searchService.updateAlertState(this.selectedAlerts, 'DISMISS', '').subscribe(results => { this.updateSelectedAlertStatus('DISMISS'); }); } processOpen() { - this.alertsService.updateAlertState(this.selectedAlerts, 'OPEN', '').subscribe(results => { + this.searchService.updateAlertState(this.selectedAlerts, 'OPEN', '').subscribe(results => { this.updateSelectedAlertStatus('OPEN'); }); } processResolve() { - this.alertsService.updateAlertState(this.selectedAlerts, 'RESOLVE', '').subscribe(results => { + this.searchService.updateAlertState(this.selectedAlerts, 'RESOLVE', '').subscribe(results => { this.updateSelectedAlertStatus('RESOLVE'); }); } @@ -313,10 +313,10 @@ export class AlertsListComponent implements OnInit, OnDestroy { this.saveSearchService.saveAsRecentSearches(savedSearch).subscribe(() => {}); } - this.alertsService.search(this.queryBuilder.searchRequest).subscribe(results => { + this.searchService.search(this.queryBuilder.searchRequest).subscribe(results => { this.setData(results); }, error => { - this.setData(new AlertsSearchResponse()); + this.setData(new SearchResponse()); this.metronDialogBox.showConfirmationMessage(ElasticsearchUtils.extractESErrorMessage(error), DialogType.Error); }); @@ -331,7 +331,7 @@ export class AlertsListComponent implements OnInit, OnDestroy { } } - setData(results: AlertsSearchResponse) { + setData(results: SearchResponse) { this.alerts = results.results; this.pagingData.total = results.total; } @@ -370,7 +370,7 @@ export class AlertsListComponent implements OnInit, OnDestroy { tryStartPolling() { if (!this.pauseRefresh) { this.tryStopPolling(); - this.refreshTimer = this.alertsService.pollSearch(this.queryBuilder.searchRequest).subscribe(results => { + this.refreshTimer = this.searchService.pollSearch(this.queryBuilder.searchRequest).subscribe(results => { this.setData(results); }); } @@ -383,7 +383,7 @@ export class AlertsListComponent implements OnInit, OnDestroy { } updateConfigRowsSettings() { - this.alertsService.interval = this.refreshInterval; + this.searchService.interval = this.refreshInterval; this.queryBuilder.setFromAndSize(this.pagingData.from, this.pagingData.size); } http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/metron-interface/metron-alerts/src/app/alerts/alerts-list/alerts-list.module.ts ---------------------------------------------------------------------- diff --git a/metron-interface/metron-alerts/src/app/alerts/alerts-list/alerts-list.module.ts b/metron-interface/metron-alerts/src/app/alerts/alerts-list/alerts-list.module.ts index adc8cbb..e6adae3 100644 --- a/metron-interface/metron-alerts/src/app/alerts/alerts-list/alerts-list.module.ts +++ b/metron-interface/metron-alerts/src/app/alerts/alerts-list/alerts-list.module.ts @@ -20,7 +20,7 @@ import {NgModule} from '@angular/core'; import {AlertsListComponent} from './alerts-list.component'; import {routing} from './alerts-list.routing'; import {SharedModule} from '../../shared/shared.module'; -import {AlertService} from '../../service/alert.service'; +import {SearchService} from '../../service/search.service'; import {MetronSorterModule} from '../../shared/metron-table/metron-sorter/metron-sorter.module'; import {ListGroupModule} from '../../shared/list-group/list-grup.module'; import {CollapseModule} from '../../shared/collapse/collapse.module'; @@ -32,7 +32,7 @@ import {ConfigureRowsModule} from '../configure-rows/configure-rows.module'; ListGroupModule, CollapseModule], exports: [AlertsListComponent], declarations: [AlertsListComponent], - providers: [AlertService], + providers: [SearchService], }) export class AlertsListModule { } http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/metron-interface/metron-alerts/src/app/alerts/configure-table/configure-table.component.ts ---------------------------------------------------------------------- diff --git a/metron-interface/metron-alerts/src/app/alerts/configure-table/configure-table.component.ts b/metron-interface/metron-alerts/src/app/alerts/configure-table/configure-table.component.ts index 941343e..0c0117b 100644 --- a/metron-interface/metron-alerts/src/app/alerts/configure-table/configure-table.component.ts +++ b/metron-interface/metron-alerts/src/app/alerts/configure-table/configure-table.component.ts @@ -24,6 +24,7 @@ import {ClusterMetaDataService} from '../../service/cluster-metadata.service'; import {ColumnMetadata} from '../../model/column-metadata'; import {ColumnNamesService} from '../../service/column-names.service'; import {ColumnNames} from '../../model/column-names'; +import {SearchService} from '../../service/search.service'; export enum AlertState { NEW, OPEN, ESCALATE, DISMISS, RESOLVE @@ -51,8 +52,11 @@ export class ConfigureTableComponent implements OnInit { allColumns: ColumnMetadataWrapper[] = []; - constructor(private router: Router, private activatedRoute: ActivatedRoute, private configureTableService: ConfigureTableService, - private clusterMetaDataService: ClusterMetaDataService, private columnNamesService: ColumnNamesService) { } + constructor(private router: Router, private activatedRoute: ActivatedRoute, + private configureTableService: ConfigureTableService, + private clusterMetaDataService: ClusterMetaDataService, + private columnNamesService: ColumnNamesService, + private searchService: SearchService) { } goBack() { this.router.navigateByUrl('/alerts-list'); @@ -80,7 +84,7 @@ export class ConfigureTableComponent implements OnInit { ngOnInit() { Observable.forkJoin( this.clusterMetaDataService.getDefaultColumns(), - this.clusterMetaDataService.getColumnMetaData(), + this.searchService.getColumnMetaData(), this.configureTableService.getTableMetadata() ).subscribe((response: any) => { this.prepareData(response[0], response[1], response[2].tableColumns); http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/metron-interface/metron-alerts/src/app/app.module.ts ---------------------------------------------------------------------- diff --git a/metron-interface/metron-alerts/src/app/app.module.ts b/metron-interface/metron-alerts/src/app/app.module.ts index 295c748..f16b2d1 100644 --- a/metron-interface/metron-alerts/src/app/app.module.ts +++ b/metron-interface/metron-alerts/src/app/app.module.ts @@ -36,7 +36,7 @@ import {ConfigureRowsModule} from './alerts/configure-rows/configure-rows.module import {SwitchModule} from './shared/switch/switch.module'; import {ColumnNamesService} from './service/column-names.service'; import {DataSource} from './service/data-source'; -import {RestApiImpl} from './service/rest-api-impl'; +import {ElasticSearchLocalstorageImpl} from './service/elasticsearch-localstorage-impl'; import {LoginModule} from './login/login.module'; import {AuthGuard} from './shared/auth-guard'; import {AuthenticationService} from './service/authentication.service'; @@ -67,7 +67,7 @@ export function initConfig(config: ColumnNamesService) { SwitchModule ], providers: [{ provide: APP_INITIALIZER, useFactory: initConfig, deps: [ColumnNamesService], multi: true }, - { provide: DataSource, useClass: RestApiImpl }, + { provide: DataSource, useClass: ElasticSearchLocalstorageImpl }, AuthenticationService, AuthGuard, LoginGuard, http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/metron-interface/metron-alerts/src/app/model/alerts-search-response.ts ---------------------------------------------------------------------- diff --git a/metron-interface/metron-alerts/src/app/model/alerts-search-response.ts b/metron-interface/metron-alerts/src/app/model/alerts-search-response.ts deleted file mode 100644 index 265f66b..0000000 --- a/metron-interface/metron-alerts/src/app/model/alerts-search-response.ts +++ /dev/null @@ -1,23 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -import {Alert} from './alert'; - -export class AlertsSearchResponse { - total = 0; - results: Alert[] = []; -} http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/metron-interface/metron-alerts/src/app/model/search-request.ts ---------------------------------------------------------------------- diff --git a/metron-interface/metron-alerts/src/app/model/search-request.ts b/metron-interface/metron-alerts/src/app/model/search-request.ts index 2150f2b..a37bd8d 100644 --- a/metron-interface/metron-alerts/src/app/model/search-request.ts +++ b/metron-interface/metron-alerts/src/app/model/search-request.ts @@ -1,4 +1,5 @@ import {SortField} from './sort-field'; +import {INDEXES} from '../utils/constants'; /** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file @@ -19,7 +20,7 @@ import {SortField} from './sort-field'; export class SearchRequest { // _source: string[]; //TODO: This needs to be removed from: number; - indices: string[] = ['websphere', 'snort', 'asa', 'bro', 'yaf']; + indices: string[] = INDEXES; query: string; size: number; sort: SortField[]; http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/metron-interface/metron-alerts/src/app/model/search-response.ts ---------------------------------------------------------------------- diff --git a/metron-interface/metron-alerts/src/app/model/search-response.ts b/metron-interface/metron-alerts/src/app/model/search-response.ts new file mode 100644 index 0000000..c3fea27 --- /dev/null +++ b/metron-interface/metron-alerts/src/app/model/search-response.ts @@ -0,0 +1,23 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import {Alert} from './alert'; + +export class SearchResponse { + total = 0; + results: Alert[] = []; +} http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/metron-interface/metron-alerts/src/app/service/alert.service.ts ---------------------------------------------------------------------- diff --git a/metron-interface/metron-alerts/src/app/service/alert.service.ts b/metron-interface/metron-alerts/src/app/service/alert.service.ts deleted file mode 100644 index 7dabc4f..0000000 --- a/metron-interface/metron-alerts/src/app/service/alert.service.ts +++ /dev/null @@ -1,72 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -import {Injectable, NgZone} from '@angular/core'; -import {Observable} from 'rxjs/Rx'; -import 'rxjs/add/observable/interval'; -import 'rxjs/add/operator/switchMap'; -import 'rxjs/add/operator/onErrorResumeNext'; - -import {Alert} from '../model/alert'; -import {Http} from '@angular/http'; -import {DataSource} from './data-source'; -import {AlertsSearchResponse} from '../model/alerts-search-response'; -import {SearchRequest} from '../model/search-request'; -import {AlertSource} from '../model/alert-source'; - -@Injectable() -export class AlertService { - - interval = 80000; - defaultHeaders = {'Content-Type': 'application/json', 'X-Requested-With': 'XMLHttpRequest'}; - - constructor(private http: Http, - private dataSource: DataSource, - private ngZone: NgZone) { } - - public search(searchRequest: SearchRequest): Observable<AlertsSearchResponse> { - return this.dataSource.getAlerts(searchRequest); - } - - public pollSearch(searchRequest: SearchRequest): Observable<AlertsSearchResponse> { - return this.ngZone.runOutsideAngular(() => { - return this.ngZone.run(() => { - return Observable.interval(this.interval * 1000).switchMap(() => { - return this.dataSource.getAlerts(searchRequest); - }); - }); - }); - } - - public getAlert(sourceType: string, alertId: string): Observable<AlertSource> { - return this.dataSource.getAlert(sourceType, alertId); - } - - public updateAlertState(alerts: Alert[], state: string, workflowId: string) { - let request = ''; - for (let alert of alerts) { - request += '{ "update" : { "sensorType" : "' + alert.source['source:type'] + '", "guid" : "' + alert.source.guid + '" } }\n' + - '{ "doc": { "alert_status": "' + state + '"'; - if (workflowId) { - request += ', "workflow_id": "' + workflowId + '"'; - } - request += ' }}\n'; - } - - return this.dataSource.updateAlertState(request); - } -} http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/metron-interface/metron-alerts/src/app/service/cluster-metadata.service.ts ---------------------------------------------------------------------- diff --git a/metron-interface/metron-alerts/src/app/service/cluster-metadata.service.ts b/metron-interface/metron-alerts/src/app/service/cluster-metadata.service.ts index 4077f30..ffd4ec1 100644 --- a/metron-interface/metron-alerts/src/app/service/cluster-metadata.service.ts +++ b/metron-interface/metron-alerts/src/app/service/cluster-metadata.service.ts @@ -18,11 +18,14 @@ import {Injectable} from '@angular/core'; import {Observable} from 'rxjs/Rx'; import {Http} from '@angular/http'; + + import {ColumnMetadata} from '../model/column-metadata'; import {DataSource} from './data-source'; @Injectable() export class ClusterMetaDataService { + defaultHeaders: {'Content-Type': 'application/json', 'X-Requested-With': 'XMLHttpRequest'}; constructor(private http: Http, private dataSource: DataSource) { @@ -31,8 +34,4 @@ export class ClusterMetaDataService { getDefaultColumns(): Observable<ColumnMetadata[]> { return this.dataSource.getDefaultAlertTableColumnNames(); } - - getColumnMetaData(): Observable<ColumnMetadata[]> { - return this.dataSource.getAllFieldNames(); - } } http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/metron-interface/metron-alerts/src/app/service/data-source.ts ---------------------------------------------------------------------- diff --git a/metron-interface/metron-alerts/src/app/service/data-source.ts b/metron-interface/metron-alerts/src/app/service/data-source.ts index 28ee384..f4f90ed 100644 --- a/metron-interface/metron-alerts/src/app/service/data-source.ts +++ b/metron-interface/metron-alerts/src/app/service/data-source.ts @@ -22,7 +22,7 @@ import {ColumnMetadata} from '../model/column-metadata'; import {ColumnNames} from '../model/column-names'; import {TableMetadata} from '../model/table-metadata'; import {SaveSearch} from '../model/save-search'; -import {AlertsSearchResponse} from '../model/alerts-search-response'; +import {SearchResponse} from '../model/search-response'; import {SearchRequest} from '../model/search-request'; import {AlertSource} from '../model/alert-source'; @@ -33,7 +33,7 @@ export abstract class DataSource { constructor(protected http: Http) {} // Calls to fetch alerts - abstract getAlerts(searchRequest: SearchRequest): Observable<AlertsSearchResponse> + abstract getAlerts(searchRequest: SearchRequest): Observable<SearchResponse> abstract getAlert(sourceType: string, alertId: string): Observable<AlertSource> abstract updateAlertState(request: any): Observable<{}> http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/metron-interface/metron-alerts/src/app/service/elasticsearch-localstorage-impl.ts ---------------------------------------------------------------------- diff --git a/metron-interface/metron-alerts/src/app/service/elasticsearch-localstorage-impl.ts b/metron-interface/metron-alerts/src/app/service/elasticsearch-localstorage-impl.ts index f4e43d5..7982102 100644 --- a/metron-interface/metron-alerts/src/app/service/elasticsearch-localstorage-impl.ts +++ b/metron-interface/metron-alerts/src/app/service/elasticsearch-localstorage-impl.ts @@ -30,7 +30,7 @@ import {ColumnNames} from '../model/column-names'; import {ColumnNamesService} from './column-names.service'; import {TableMetadata} from '../model/table-metadata'; import {SaveSearch} from '../model/save-search'; -import {AlertsSearchResponse} from '../model/alerts-search-response'; +import {SearchResponse} from '../model/search-response'; import {SearchRequest} from '../model/search-request'; import {AlertSource} from '../model/alert-source'; @@ -47,7 +47,7 @@ export class ElasticSearchLocalstorageImpl extends DataSource { new ColumnMetadata('alert_status', 'string') ]; - getAlerts(searchRequest: SearchRequest): Observable<AlertsSearchResponse> { + getAlerts(searchRequest: SearchRequest): Observable<SearchResponse> { let url = '/search/*' + ElasticsearchUtils.excludeIndexName + '/_search'; let request: any = JSON.parse(JSON.stringify(searchRequest)); request.query = { query_string: { query: searchRequest.query } }; http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/metron-interface/metron-alerts/src/app/service/rest-api-impl.ts ---------------------------------------------------------------------- diff --git a/metron-interface/metron-alerts/src/app/service/rest-api-impl.ts b/metron-interface/metron-alerts/src/app/service/rest-api-impl.ts deleted file mode 100644 index 061708d..0000000 --- a/metron-interface/metron-alerts/src/app/service/rest-api-impl.ts +++ /dev/null @@ -1,46 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -import {Observable} from 'rxjs/Rx'; -import {Headers, RequestOptions} from '@angular/http'; - -import {HttpUtil} from '../utils/httpUtil'; -import {AlertsSearchResponse} from '../model/alerts-search-response'; -import {SearchRequest} from '../model/search-request'; -import {ElasticSearchLocalstorageImpl} from './elasticsearch-localstorage-impl'; -import {AlertSource} from '../model/alert-source'; - -export class RestApiImpl extends ElasticSearchLocalstorageImpl { - - getAlerts(searchRequest: SearchRequest): Observable<AlertsSearchResponse> { - let url = '/api/v1/search/search'; - return this.http.post(url, searchRequest, new RequestOptions({headers: new Headers(this.defaultHeaders)})) - .map(HttpUtil.extractData) - .catch(HttpUtil.handleError) - .onErrorResumeNext(); - } - - getAlert(sourceType: string, alertId: string): Observable<AlertSource> { - let url = '/api/v1/search/findOne'; - let requestSchema = { guid: alertId, sensorType: sourceType}; - - return this.http.post(url, requestSchema, new RequestOptions({headers: new Headers(this.defaultHeaders)})) - .map(HttpUtil.extractData) - .catch(HttpUtil.handleError) - .onErrorResumeNext(); - } -} http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/metron-interface/metron-alerts/src/app/service/search.service.ts ---------------------------------------------------------------------- diff --git a/metron-interface/metron-alerts/src/app/service/search.service.ts b/metron-interface/metron-alerts/src/app/service/search.service.ts new file mode 100644 index 0000000..be3b1f6 --- /dev/null +++ b/metron-interface/metron-alerts/src/app/service/search.service.ts @@ -0,0 +1,112 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import {Injectable, NgZone} from '@angular/core'; +import {Headers, RequestOptions} from '@angular/http'; +import {Observable} from 'rxjs/Rx'; +import 'rxjs/add/observable/interval'; +import 'rxjs/add/operator/switchMap'; +import 'rxjs/add/operator/onErrorResumeNext'; + +import {HttpUtil} from '../utils/httpUtil'; +import {Alert} from '../model/alert'; +import {Http} from '@angular/http'; +import {DataSource} from './data-source'; +import {SearchResponse} from '../model/search-response'; +import {SearchRequest} from '../model/search-request'; +import {AlertSource} from '../model/alert-source'; +import {INDEXES} from '../utils/constants'; +import {ColumnMetadata} from '../model/column-metadata'; + +@Injectable() +export class SearchService { + + interval = 80000; + defaultHeaders = {'Content-Type': 'application/json', 'X-Requested-With': 'XMLHttpRequest'}; + + private static extractColumnNameDataFromRestApi(res: Response): ColumnMetadata[] { + let response: any = res || {}; + let processedKeys: string[] = []; + let columnMetadatas: ColumnMetadata[] = []; + + for (let index of Object.keys(response)) { + let indexMetaData = response[index]; + for (let key of Object.keys(indexMetaData)) { + if (processedKeys.indexOf(key) === -1) { + processedKeys.push(key); + columnMetadatas.push(new ColumnMetadata(key, indexMetaData[key])); + } + } + } + + return columnMetadatas; + } + + constructor(private http: Http, + private dataSource: DataSource, + private ngZone: NgZone) { } + + public getAlert(sourceType: string, alertId: string): Observable<AlertSource> { + let url = '/api/v1/search/findOne'; + let requestSchema = { guid: alertId, sensorType: sourceType}; + + return this.http.post(url, requestSchema, new RequestOptions({headers: new Headers(this.defaultHeaders)})) + .map(HttpUtil.extractData) + .catch(HttpUtil.handleError) + .onErrorResumeNext(); + } + + public getColumnMetaData(): Observable<ColumnMetadata[]> { + let url = '/api/v1/search/column/metadata'; + return this.http.post(url, INDEXES, new RequestOptions({headers: new Headers(this.defaultHeaders)})) + .map(HttpUtil.extractData) + .map(SearchService.extractColumnNameDataFromRestApi) + .catch(HttpUtil.handleError); + } + + public pollSearch(searchRequest: SearchRequest): Observable<SearchResponse> { + return this.ngZone.runOutsideAngular(() => { + return this.ngZone.run(() => { + return Observable.interval(this.interval * 1000).switchMap(() => { + return this.search(searchRequest); + }); + }); + }); + } + + public search(searchRequest: SearchRequest): Observable<SearchResponse> { + let url = '/api/v1/search/search'; + return this.http.post(url, searchRequest, new RequestOptions({headers: new Headers(this.defaultHeaders)})) + .map(HttpUtil.extractData) + .catch(HttpUtil.handleError) + .onErrorResumeNext(); + } + + public updateAlertState(alerts: Alert[], state: string, workflowId: string) { + let request = ''; + for (let alert of alerts) { + request += '{ "update" : { "sensorType" : "' + alert.source['source:type'] + '", "guid" : "' + alert.source.guid + '" } }\n' + + '{ "doc": { "alert_status": "' + state + '"'; + if (workflowId) { + request += ', "workflow_id": "' + workflowId + '"'; + } + request += ' }}\n'; + } + + return this.dataSource.updateAlertState(request); + } +} http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/metron-interface/metron-alerts/src/app/utils/constants.ts ---------------------------------------------------------------------- diff --git a/metron-interface/metron-alerts/src/app/utils/constants.ts b/metron-interface/metron-alerts/src/app/utils/constants.ts index da6d50c..a738a1d 100644 --- a/metron-interface/metron-alerts/src/app/utils/constants.ts +++ b/metron-interface/metron-alerts/src/app/utils/constants.ts @@ -20,3 +20,5 @@ export const ALERTS_RECENT_SEARCH = 'metron-alerts-recent-saved-search'; export const ALERTS_SAVED_SEARCH = 'metron-alerts-saved-search'; export const ALERTS_TABLE_METADATA = 'metron-alerts-table-metadata'; export const ALERTS_COLUMN_NAMES = 'metron-alerts-column-names'; + +export let INDEXES = ['websphere', 'snort', 'asa', 'bro', 'yaf']; http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/metron-interface/metron-alerts/src/app/utils/elasticsearch-utils.ts ---------------------------------------------------------------------- diff --git a/metron-interface/metron-alerts/src/app/utils/elasticsearch-utils.ts b/metron-interface/metron-alerts/src/app/utils/elasticsearch-utils.ts index a86907b..0896f32 100644 --- a/metron-interface/metron-alerts/src/app/utils/elasticsearch-utils.ts +++ b/metron-interface/metron-alerts/src/app/utils/elasticsearch-utils.ts @@ -16,7 +16,7 @@ * limitations under the License. */ import {ColumnMetadata} from '../model/column-metadata'; -import {AlertsSearchResponse} from '../model/alerts-search-response'; +import {SearchResponse} from '../model/search-response'; export class ElasticsearchUtils { @@ -54,9 +54,9 @@ export class ElasticsearchUtils { return columnMetadata; } - public static extractAlertsData(res: Response): AlertsSearchResponse { + public static extractAlertsData(res: Response): SearchResponse { let response: any = res || {}; - let alertsSearchResponse: AlertsSearchResponse = new AlertsSearchResponse(); + let alertsSearchResponse: SearchResponse = new SearchResponse(); alertsSearchResponse.total = response['hits']['total']; alertsSearchResponse.results = response['hits']['hits']; return alertsSearchResponse; http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/metron-interface/metron-config/README.md ---------------------------------------------------------------------- diff --git a/metron-interface/metron-config/README.md b/metron-interface/metron-config/README.md index 257faba..f0a69e3 100644 --- a/metron-interface/metron-config/README.md +++ b/metron-interface/metron-config/README.md @@ -49,12 +49,6 @@ This module provides a user interface for management functions in Metron. rpm -ih metron-config-$METRON_VERSION-*.noarch.rpm ``` -1. Install the [Express](https://expressjs.com/) web framework from the `package.json` file in `$METRON_HOME/web/expressjs`: - - ``` - npm --prefix $METRON_HOME/web/expressjs/ install - ``` - ## Configuration The Managment UI is configured in the `$METRON_HOME/config/management_ui.yml` file. Create this file and set the values to match your environment: @@ -87,7 +81,7 @@ The Management UI can also be started in development mode. This allows changes cd metron-interface/metron-config npm install ``` - + 1. Start the application: ``` @@ -113,4 +107,4 @@ The application will be available at http://localhost:4200/. The REST applicati ## License -This projects bundles Font Awesome which is available under the SIL Open Font License. See http://fontawesome.io/license/ for more details. \ No newline at end of file +This projects bundles Font Awesome which is available under the SIL Open Font License. See http://fontawesome.io/license/ for more details.
