http://git-wip-us.apache.org/repos/asf/metron/blob/8bf3b6ec/metron-analytics/metron-profiler-spark/src/test/java/org/apache/metron/profiler/spark/BatchProfilerIntegrationTest.java ---------------------------------------------------------------------- diff --git a/metron-analytics/metron-profiler-spark/src/test/java/org/apache/metron/profiler/spark/BatchProfilerIntegrationTest.java b/metron-analytics/metron-profiler-spark/src/test/java/org/apache/metron/profiler/spark/BatchProfilerIntegrationTest.java index c33644f..83800af 100644 --- a/metron-analytics/metron-profiler-spark/src/test/java/org/apache/metron/profiler/spark/BatchProfilerIntegrationTest.java +++ b/metron-analytics/metron-profiler-spark/src/test/java/org/apache/metron/profiler/spark/BatchProfilerIntegrationTest.java @@ -58,8 +58,11 @@ import static org.apache.metron.profiler.spark.BatchProfilerConfig.TELEMETRY_INP import static org.apache.metron.profiler.spark.BatchProfilerConfig.TELEMETRY_INPUT_END; import static org.apache.metron.profiler.spark.BatchProfilerConfig.TELEMETRY_INPUT_FORMAT; import static org.apache.metron.profiler.spark.BatchProfilerConfig.TELEMETRY_INPUT_PATH; +import static org.apache.metron.profiler.spark.BatchProfilerConfig.TELEMETRY_INPUT_READER; import static org.junit.Assert.assertTrue; +import static org.apache.metron.profiler.spark.reader.TelemetryReaders.*; + /** * An integration test for the {@link BatchProfiler}. */ @@ -159,8 +162,8 @@ public class BatchProfilerIntegrationTest { @Test public void testBatchProfilerWithJSON() throws Exception { // the input telemetry is text/json stored in the local filesystem + profilerProperties.put(TELEMETRY_INPUT_READER.getKey(), JSON.toString()); profilerProperties.put(TELEMETRY_INPUT_PATH.getKey(), "src/test/resources/telemetry.json"); - profilerProperties.put(TELEMETRY_INPUT_FORMAT.getKey(), "text"); BatchProfiler profiler = new BatchProfiler(); profiler.run(spark, profilerProperties, getGlobals(), readerProperties, getProfile()); @@ -170,20 +173,41 @@ public class BatchProfilerIntegrationTest { @Test public void testBatchProfilerWithORC() throws Exception { - // re-write the test data as ORC + // re-write the test data as column-oriented ORC String pathToORC = tempFolder.getRoot().getAbsolutePath(); spark.read() - .format("text") + .format("json") .load("src/test/resources/telemetry.json") - .as(Encoders.STRING()) .write() .mode("overwrite") .format("org.apache.spark.sql.execution.datasources.orc") .save(pathToORC); // tell the profiler to use the ORC input data + profilerProperties.put(TELEMETRY_INPUT_READER.getKey(), ORC.toString()); profilerProperties.put(TELEMETRY_INPUT_PATH.getKey(), pathToORC); - profilerProperties.put(TELEMETRY_INPUT_FORMAT.getKey(), "org.apache.spark.sql.execution.datasources.orc"); + + BatchProfiler profiler = new BatchProfiler(); + profiler.run(spark, profilerProperties, getGlobals(), readerProperties, getProfile()); + + validateProfiles(); + } + + @Test + public void testBatchProfilerWithParquet() throws Exception { + // re-write the test data as column-oriented ORC + String inputPath = tempFolder.getRoot().getAbsolutePath(); + spark.read() + .format("json") + .load("src/test/resources/telemetry.json") + .write() + .mode("overwrite") + .format("parquet") + .save(inputPath); + + // tell the profiler to use the ORC input data + profilerProperties.put(TELEMETRY_INPUT_READER.getKey(), PARQUET.toString()); + profilerProperties.put(TELEMETRY_INPUT_PATH.getKey(), inputPath); BatchProfiler profiler = new BatchProfiler(); profiler.run(spark, profilerProperties, getGlobals(), readerProperties, getProfile()); @@ -206,7 +230,9 @@ public class BatchProfilerIntegrationTest { .save(pathToCSV); // tell the profiler to use the CSV input data + // CSV is an example of needing to define both the reader and the input format profilerProperties.put(TELEMETRY_INPUT_PATH.getKey(), pathToCSV); + profilerProperties.put(TELEMETRY_INPUT_READER.getKey(), "text"); profilerProperties.put(TELEMETRY_INPUT_FORMAT.getKey(), "csv"); // set a reader property; tell the reader to expect a header
http://git-wip-us.apache.org/repos/asf/metron/blob/8bf3b6ec/metron-analytics/metron-profiler-spark/src/test/java/org/apache/metron/profiler/spark/function/reader/ColumnEncodedTelemetryReaderTest.java ---------------------------------------------------------------------- diff --git a/metron-analytics/metron-profiler-spark/src/test/java/org/apache/metron/profiler/spark/function/reader/ColumnEncodedTelemetryReaderTest.java b/metron-analytics/metron-profiler-spark/src/test/java/org/apache/metron/profiler/spark/function/reader/ColumnEncodedTelemetryReaderTest.java new file mode 100644 index 0000000..0bcfb3f --- /dev/null +++ b/metron-analytics/metron-profiler-spark/src/test/java/org/apache/metron/profiler/spark/function/reader/ColumnEncodedTelemetryReaderTest.java @@ -0,0 +1,118 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.apache.metron.profiler.spark.function.reader; + +import org.apache.metron.profiler.spark.reader.TelemetryReaders; +import org.apache.spark.SparkConf; +import org.apache.spark.sql.Dataset; +import org.apache.spark.sql.SparkSession; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import java.util.Properties; + +import static org.apache.metron.profiler.spark.BatchProfilerConfig.TELEMETRY_INPUT_FORMAT; +import static org.apache.metron.profiler.spark.BatchProfilerConfig.TELEMETRY_INPUT_PATH; + +/** + * Tests the {@link org.apache.metron.profiler.spark.reader.ColumnEncodedTelemetryReader} class. + */ +public class ColumnEncodedTelemetryReaderTest { + + @Rule + public TemporaryFolder tempFolder = new TemporaryFolder(); + private static SparkSession spark; + private Properties profilerProperties; + private Properties readerProperties; + + @BeforeClass + public static void setupSpark() { + SparkConf conf = new SparkConf() + .setMaster("local") + .setAppName("BatchProfilerIntegrationTest") + .set("spark.sql.shuffle.partitions", "8"); + spark = SparkSession + .builder() + .config(conf) + .getOrCreate(); + } + + @AfterClass + public static void tearDownSpark() { + if(spark != null) { + spark.close(); + } + } + + @Before + public void setup() { + readerProperties = new Properties(); + profilerProperties = new Properties(); + } + + @Test + public void testParquet() { + // re-write the test data as column-oriented ORC + String inputPath = tempFolder.getRoot().getAbsolutePath(); + spark.read() + .format("json") + .load("src/test/resources/telemetry.json") + .write() + .mode("overwrite") + .format("parquet") + .save(inputPath); + + // tell the profiler to use the CSV input data + profilerProperties.put(TELEMETRY_INPUT_PATH.getKey(), inputPath); + profilerProperties.put(TELEMETRY_INPUT_FORMAT.getKey(), "parquet"); + + // set a reader property; tell the reader to expect a header + readerProperties.put("header", "true"); + + // there should be 100 valid JSON records + Dataset<String> telemetry = TelemetryReaders.COLUMNAR.read(spark, profilerProperties, readerProperties); + Assert.assertEquals(100, telemetry.filter(new IsValidJSON()).count()); + } + + @Test + public void testORC() { + // re-write the test data as column-oriented ORC + String pathToORC = tempFolder.getRoot().getAbsolutePath(); + spark.read() + .format("json") + .load("src/test/resources/telemetry.json") + .write() + .mode("overwrite") + .format("org.apache.spark.sql.execution.datasources.orc") + .save(pathToORC); + + // tell the profiler to use the CSV input data + profilerProperties.put(TELEMETRY_INPUT_PATH.getKey(), pathToORC); + profilerProperties.put(TELEMETRY_INPUT_FORMAT.getKey(), "org.apache.spark.sql.execution.datasources.orc"); + + // there should be 100 valid JSON records + Dataset<String> telemetry = TelemetryReaders.COLUMNAR.read(spark, profilerProperties, readerProperties); + Assert.assertEquals(100, telemetry.filter(new IsValidJSON()).count()); + } +} http://git-wip-us.apache.org/repos/asf/metron/blob/8bf3b6ec/metron-analytics/metron-profiler-spark/src/test/java/org/apache/metron/profiler/spark/function/reader/IsValidJSON.java ---------------------------------------------------------------------- diff --git a/metron-analytics/metron-profiler-spark/src/test/java/org/apache/metron/profiler/spark/function/reader/IsValidJSON.java b/metron-analytics/metron-profiler-spark/src/test/java/org/apache/metron/profiler/spark/function/reader/IsValidJSON.java new file mode 100644 index 0000000..50144f9 --- /dev/null +++ b/metron-analytics/metron-profiler-spark/src/test/java/org/apache/metron/profiler/spark/function/reader/IsValidJSON.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.apache.metron.profiler.spark.function.reader; + +import org.apache.spark.api.java.function.FilterFunction; +import org.json.simple.JSONObject; +import org.json.simple.parser.JSONParser; + +/** + * A filter function that filters out invalid JSON records. + */ +public class IsValidJSON implements FilterFunction<String> { + + @Override + public boolean call(String text) throws Exception { + JSONParser parser = new JSONParser(); + JSONObject json = (JSONObject) parser.parse(text); + + // all of the test data has at least 32 fields in each JSON record + return json.keySet().size() >= 32; + } +} http://git-wip-us.apache.org/repos/asf/metron/blob/8bf3b6ec/metron-analytics/metron-profiler-spark/src/test/java/org/apache/metron/profiler/spark/function/reader/TelemetryReadersTest.java ---------------------------------------------------------------------- diff --git a/metron-analytics/metron-profiler-spark/src/test/java/org/apache/metron/profiler/spark/function/reader/TelemetryReadersTest.java b/metron-analytics/metron-profiler-spark/src/test/java/org/apache/metron/profiler/spark/function/reader/TelemetryReadersTest.java new file mode 100644 index 0000000..e525ae0 --- /dev/null +++ b/metron-analytics/metron-profiler-spark/src/test/java/org/apache/metron/profiler/spark/function/reader/TelemetryReadersTest.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.apache.metron.profiler.spark.function.reader; + +import org.apache.metron.profiler.spark.reader.ColumnEncodedTelemetryReader; +import org.apache.metron.profiler.spark.reader.TelemetryReaders; +import org.apache.metron.profiler.spark.reader.TextEncodedTelemetryReader; +import org.junit.Assert; +import org.junit.Test; + +import static org.apache.metron.profiler.spark.reader.TelemetryReaders.COLUMNAR; +import static org.apache.metron.profiler.spark.reader.TelemetryReaders.JSON; +import static org.apache.metron.profiler.spark.reader.TelemetryReaders.ORC; +import static org.apache.metron.profiler.spark.reader.TelemetryReaders.PARQUET; +import static org.apache.metron.profiler.spark.reader.TelemetryReaders.TEXT; + +public class TelemetryReadersTest { + + @Test + public void testJsonReader() { + String key = JSON.toString(); + Assert.assertTrue(TelemetryReaders.create(key) instanceof TextEncodedTelemetryReader); + } + + @Test + public void testJsonReaderLowerCase() { + String key = JSON.toString().toLowerCase(); + Assert.assertTrue(TelemetryReaders.create(key) instanceof TextEncodedTelemetryReader); + } + + @Test + public void testOrcReader() { + String key = ORC.toString(); + Assert.assertTrue(TelemetryReaders.create(key) instanceof ColumnEncodedTelemetryReader); + } + + + @Test + public void testOrcReaderLowerCase() { + String key = ORC.toString().toLowerCase(); + Assert.assertTrue(TelemetryReaders.create(key) instanceof ColumnEncodedTelemetryReader); + } + + @Test + public void testParquetReader() { + String key = PARQUET.toString(); + Assert.assertTrue(TelemetryReaders.create(key) instanceof ColumnEncodedTelemetryReader); + } + + @Test + public void testParquetReaderLowerCase() { + String key = PARQUET.toString().toLowerCase(); + Assert.assertTrue(TelemetryReaders.create(key) instanceof ColumnEncodedTelemetryReader); + } + + @Test + public void testTextReader() { + String key = TEXT.toString(); + Assert.assertTrue(TelemetryReaders.create(key) instanceof TextEncodedTelemetryReader); + } + + @Test + public void testColumnReader() { + String key = COLUMNAR.toString(); + Assert.assertTrue(TelemetryReaders.create(key) instanceof ColumnEncodedTelemetryReader); + } + + @Test(expected = IllegalArgumentException.class) + public void testInvalidReader() { + TelemetryReaders.create("invalid"); + Assert.fail("exception expected"); + } +} http://git-wip-us.apache.org/repos/asf/metron/blob/8bf3b6ec/metron-analytics/metron-profiler-spark/src/test/java/org/apache/metron/profiler/spark/function/reader/TextEncodedTelemetryReaderTest.java ---------------------------------------------------------------------- diff --git a/metron-analytics/metron-profiler-spark/src/test/java/org/apache/metron/profiler/spark/function/reader/TextEncodedTelemetryReaderTest.java b/metron-analytics/metron-profiler-spark/src/test/java/org/apache/metron/profiler/spark/function/reader/TextEncodedTelemetryReaderTest.java new file mode 100644 index 0000000..3b26bb9 --- /dev/null +++ b/metron-analytics/metron-profiler-spark/src/test/java/org/apache/metron/profiler/spark/function/reader/TextEncodedTelemetryReaderTest.java @@ -0,0 +1,114 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.apache.metron.profiler.spark.function.reader; + +import org.apache.metron.profiler.spark.reader.TelemetryReaders; +import org.apache.spark.SparkConf; +import org.apache.spark.sql.Dataset; +import org.apache.spark.sql.Encoders; +import org.apache.spark.sql.SparkSession; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import java.util.Properties; + +import static org.apache.metron.profiler.spark.BatchProfilerConfig.TELEMETRY_INPUT_FORMAT; +import static org.apache.metron.profiler.spark.BatchProfilerConfig.TELEMETRY_INPUT_PATH; + +/** + * Tests the {@link org.apache.metron.profiler.spark.reader.TextEncodedTelemetryReader} class. + */ +public class TextEncodedTelemetryReaderTest { + + @Rule + public TemporaryFolder tempFolder = new TemporaryFolder(); + private static SparkSession spark; + private Properties profilerProperties; + private Properties readerProperties; + + @BeforeClass + public static void setupSpark() { + SparkConf conf = new SparkConf() + .setMaster("local") + .setAppName("BatchProfilerIntegrationTest") + .set("spark.sql.shuffle.partitions", "8"); + spark = SparkSession + .builder() + .config(conf) + .getOrCreate(); + } + + @AfterClass + public static void tearDownSpark() { + if(spark != null) { + spark.close(); + } + } + + @Before + public void setup() { + readerProperties = new Properties(); + profilerProperties = new Properties(); + } + + @Test + public void testCSV() { + // re-write the test data as a CSV with a header record + String pathToCSV = tempFolder.getRoot().getAbsolutePath(); + spark.read() + .format("text") + .load("src/test/resources/telemetry.json") + .as(Encoders.STRING()) + .write() + .mode("overwrite") + .option("header", "true") + .format("csv") + .save(pathToCSV); + + // tell the profiler to use the CSV input data + profilerProperties.put(TELEMETRY_INPUT_PATH.getKey(), pathToCSV); + profilerProperties.put(TELEMETRY_INPUT_FORMAT.getKey(), "csv"); + + // set a reader property; tell the reader to expect a header + readerProperties.put("header", "true"); + + // there should be 100 valid JSON records + Dataset<String> telemetry = TelemetryReaders.TEXT.read(spark, profilerProperties, readerProperties); + Assert.assertEquals(100, telemetry.filter(new IsValidJSON()).count()); + } + + @Test + public void testJSON() { + // use the test telemetry that is stored as raw json + profilerProperties.put(TELEMETRY_INPUT_PATH.getKey(), "src/test/resources/telemetry.json"); + profilerProperties.put(TELEMETRY_INPUT_FORMAT.getKey(), "text"); + + // set a reader property; tell the reader to expect a header + readerProperties.put("header", "true"); + + // there should be 100 valid JSON records + Dataset<String> telemetry = TelemetryReaders.TEXT.read(spark, profilerProperties, readerProperties); + Assert.assertEquals(100, telemetry.filter(new IsValidJSON()).count()); + } +} http://git-wip-us.apache.org/repos/asf/metron/blob/8bf3b6ec/metron-deployment/Kerberos-manual-setup.md ---------------------------------------------------------------------- diff --git a/metron-deployment/Kerberos-manual-setup.md b/metron-deployment/Kerberos-manual-setup.md index 4bd4516..6d40552 100644 --- a/metron-deployment/Kerberos-manual-setup.md +++ b/metron-deployment/Kerberos-manual-setup.md @@ -563,17 +563,16 @@ X-Pack sudo -u hdfs hdfs dfs -chown metron:metron /apps/metron/elasticsearch/xpack-password ``` -1. New settings have been added to configure the Elasticsearch client. By default the client will run as the normal ES prebuilt transport client. If you enable X-Pack you should set the es.client.class as shown below. +1. New settings have been added to configure the Elasticsearch client. - Add the `es.client.settings` to global.json + Modify the `es.client.settings` key in global.json ``` $METRON_HOME/config/zookeeper/global.json -> "es.client.settings" : { - "es.client.class" : "org.elasticsearch.xpack.client.PreBuiltXPackTransportClient", - "es.xpack.username" : "transport_client_user", - "es.xpack.password.file" : "/apps/metron/elasticsearch/xpack-password" + "xpack.username" : "transport_client_user", + "xpack.password.file" : "/apps/metron/elasticsearch/xpack-password" } ``` @@ -583,151 +582,6 @@ X-Pack $METRON_HOME/bin/zk_load_configs.sh -m PUSH -i $METRON_HOME/config/zookeeper/ -z $ZOOKEEPER ``` -1. The last step before restarting the topology is to create a custom X-Pack shaded and relocated jar. This is up to you because of licensing restrictions, but here is a sample Maven pom file that should help. - - ``` - <?xml version="1.0" encoding="UTF-8"?> - <!-- - Licensed to the Apache Software - Foundation (ASF) under one or more contributor license agreements. See the - NOTICE file distributed with this work for additional information regarding - copyright ownership. The ASF licenses this file to You under the Apache License, - Version 2.0 (the "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - Unless required by applicable law or agreed to in writing, software distributed - under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES - OR CONDITIONS OF ANY KIND, either express or implied. See the License for - the specific language governing permissions and limitations under the License. - --> - <project xmlns="http://maven.apache.org/POM/4.0.0"; - xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; - xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> - <modelVersion>4.0.0</modelVersion> - <groupId>org.elasticsearch</groupId> - <artifactId>elasticsearch-xpack-shaded</artifactId> - <name>elasticsearch-xpack-shaded</name> - <packaging>jar</packaging> - <version>5.6.2</version> - <repositories> - <repository> - <id>elasticsearch-releases</id> - <url>https://artifacts.elastic.co/maven</url> - <releases> - <enabled>true</enabled> - </releases> - <snapshots> - <enabled>false</enabled> - </snapshots> - </repository> - </repositories> - <dependencies> - <dependency> - <groupId>org.elasticsearch.client</groupId> - <artifactId>x-pack-transport</artifactId> - <version>5.6.2</version> - <exclusions> - <exclusion> - <groupId>com.fasterxml.jackson.dataformat</groupId> - <artifactId>jackson-dataformat-yaml</artifactId> - </exclusion> - <exclusion> - <groupId>com.fasterxml.jackson.dataformat</groupId> - <artifactId>jackson-dataformat-cbor</artifactId> - </exclusion> - <exclusion> - <groupId>com.fasterxml.jackson.core</groupId> - <artifactId>jackson-core</artifactId> - </exclusion> - <exclusion> - <groupId>org.slf4j</groupId> - <artifactId>slf4j-api</artifactId> - </exclusion> - <exclusion> - <groupId>org.slf4j</groupId> - <artifactId>slf4j-log4j12</artifactId> - </exclusion> - <exclusion> - <groupId>log4j</groupId> - <artifactId>log4j</artifactId> - </exclusion> - </exclusions> - </dependency> - </dependencies> - <build> - <plugins> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-shade-plugin</artifactId> - <version>3.2.0</version> - <configuration> - <createDependencyReducedPom>true</createDependencyReducedPom> - </configuration> - <executions> - <execution> - <phase>package</phase> - <goals> - <goal>shade</goal> - </goals> - <configuration> - <filters> - <filter> - <artifact>*:*</artifact> - <excludes> - <exclude>META-INF/*.SF</exclude> - <exclude>META-INF/*.DSA</exclude> - <exclude>META-INF/*.RSA</exclude> - </excludes> - </filter> - </filters> - <relocations> - <relocation> - <pattern>io.netty</pattern> - <shadedPattern>org.apache.metron.io.netty</shadedPattern> - </relocation> - <relocation> - <pattern>org.apache.logging.log4j</pattern> - <shadedPattern>org.apache.metron.logging.log4j</shadedPattern> - </relocation> - </relocations> - <artifactSet> - <excludes> - <exclude>org.slf4j.impl*</exclude> - <exclude>org.slf4j:slf4j-log4j*</exclude> - </excludes> - </artifactSet> - <transformers> - <transformer - implementation="org.apache.maven.plugins.shade.resource.DontIncludeResourceTransformer"> - <resources> - <resource>.yaml</resource> - <resource>LICENSE.txt</resource> - <resource>ASL2.0</resource> - <resource>NOTICE.txt</resource> - </resources> - </transformer> - <transformer - implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/> - <transformer - implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer"> - <mainClass></mainClass> - </transformer> - </transformers> - </configuration> - </execution> - </executions> - </plugin> - </plugins> - </build> - </project> - ``` - -1. Once you've built the `elasticsearch-xpack-shaded-5.6.2.jar`, it needs to be made available to Storm when you submit the topology. Create a contrib directory for indexing and put the jar file in this directory. - - ``` - mkdir $METRON_HOME/indexing_contrib - cp elasticsearch-xpack-shaded-5.6.2.jar $METRON_HOME/indexing_contrib/elasticsearch-xpack-shaded-5.6.2.jar - ``` - 1. Now you can restart the Elasticsearch topology. Note, you should perform this step manually, as follows. ``` http://git-wip-us.apache.org/repos/asf/metron/blob/8bf3b6ec/metron-deployment/ansible/roles/ambari_master/defaults/main.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/ansible/roles/ambari_master/defaults/main.yml b/metron-deployment/ansible/roles/ambari_master/defaults/main.yml index 8bae8ce..f6d76e5 100644 --- a/metron-deployment/ansible/roles/ambari_master/defaults/main.yml +++ b/metron-deployment/ansible/roles/ambari_master/defaults/main.yml @@ -19,5 +19,6 @@ ambari_server_mem: 2048 ambari_mpack_version: 0.6.1.0 metron_mpack_name: metron_mpack-{{ ambari_mpack_version }}.tar.gz metron_mpack_path: "{{ playbook_dir }}/../../packaging/ambari/metron-mpack/target/{{ metron_mpack_name }}" +elasticsearch_mpack_version: 5.6.2 elasticsearch_mpack_name: elasticsearch_mpack-{{ ambari_mpack_version }}.tar.gz elasticsearch_mpack_path: "{{ playbook_dir }}/../../packaging/ambari/elasticsearch-mpack/target/{{ elasticsearch_mpack_name }}" http://git-wip-us.apache.org/repos/asf/metron/blob/8bf3b6ec/metron-deployment/ansible/roles/ambari_master/tasks/elasticsearch_mpack.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/ansible/roles/ambari_master/tasks/elasticsearch_mpack.yml b/metron-deployment/ansible/roles/ambari_master/tasks/elasticsearch_mpack.yml index 7ce1a13..ea5b0f4 100644 --- a/metron-deployment/ansible/roles/ambari_master/tasks/elasticsearch_mpack.yml +++ b/metron-deployment/ansible/roles/ambari_master/tasks/elasticsearch_mpack.yml @@ -21,6 +21,6 @@ dest: /tmp - name: Install Elasticsearch MPack on Ambari Host - shell: ambari-server install-mpack --mpack=/tmp/elasticsearch_mpack-{{ ambari_mpack_version }}.tar.gz + shell: "ambari-server install-mpack --mpack=/tmp/{{ elasticsearch_mpack_name }}" args: - creates: /var/lib/ambari-server/resources/mpacks/elasticsearch-ambari.mpack-{{ ambari_mpack_version }}/addon-services + creates: /var/lib/ambari-server/resources/mpacks/elasticsearch-ambari.mpack-{{ elasticsearch_mpack_version }}/addon-services http://git-wip-us.apache.org/repos/asf/metron/blob/8bf3b6ec/metron-deployment/ansible/roles/bro/tasks/bro.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/ansible/roles/bro/tasks/bro.yml b/metron-deployment/ansible/roles/bro/tasks/bro.yml index 222ef0e..9ba3ffa 100644 --- a/metron-deployment/ansible/roles/bro/tasks/bro.yml +++ b/metron-deployment/ansible/roles/bro/tasks/bro.yml @@ -30,8 +30,8 @@ - name: Compile and Install bro shell: "{{ item }}" environment: - CXX: /opt/rh/devtoolset-4/root/usr/bin/g++ - CC: /opt/rh/devtoolset-4/root/usr/bin/gcc + CXX: /opt/rh/devtoolset-6/root/usr/bin/g++ + CC: /opt/rh/devtoolset-6/root/usr/bin/gcc args: chdir: "/tmp/bro-{{ bro_version }}" creates: "{{ bro_home }}/bin/bro" http://git-wip-us.apache.org/repos/asf/metron/blob/8bf3b6ec/metron-deployment/ansible/roles/bro/tasks/dependencies.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/ansible/roles/bro/tasks/dependencies.yml b/metron-deployment/ansible/roles/bro/tasks/dependencies.yml index 3cd3bae..f8b1e28 100644 --- a/metron-deployment/ansible/roles/bro/tasks/dependencies.yml +++ b/metron-deployment/ansible/roles/bro/tasks/dependencies.yml @@ -38,8 +38,8 @@ - perl - crontabs - net-tools - - devtoolset-4-gcc - - devtoolset-4-gcc-c++ + - devtoolset-6-gcc + - devtoolset-6-gcc-c++ - python27 - rh-git29 register: result http://git-wip-us.apache.org/repos/asf/metron/blob/8bf3b6ec/metron-deployment/ansible/roles/bro/tasks/metron-bro-plugin-kafka.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/ansible/roles/bro/tasks/metron-bro-plugin-kafka.yml b/metron-deployment/ansible/roles/bro/tasks/metron-bro-plugin-kafka.yml index 7043387..ec7af97 100644 --- a/metron-deployment/ansible/roles/bro/tasks/metron-bro-plugin-kafka.yml +++ b/metron-deployment/ansible/roles/bro/tasks/metron-bro-plugin-kafka.yml @@ -30,8 +30,8 @@ environment: PATH: "{{ git29_bin }}:{{ bro_bin }}:{{ ansible_env.PATH }}" LD_LIBRARY_PATH: "{{ httpd24_lib }}:{{ python27_lib }}" - CXX: /opt/rh/devtoolset-4/root/usr/bin/g++ - CC: /opt/rh/devtoolset-4/root/usr/bin/gcc + CXX: /opt/rh/devtoolset-6/root/usr/bin/g++ + CC: /opt/rh/devtoolset-6/root/usr/bin/gcc command: "{{ python27_bin }}/bro-pkg install apache/metron-bro-plugin-kafka --force --version {{ metron_bro_plugin_kafka_version }}" - name: Configure bro-kafka plugin http://git-wip-us.apache.org/repos/asf/metron/blob/8bf3b6ec/metron-deployment/ansible/roles/librdkafka/defaults/main.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/ansible/roles/librdkafka/defaults/main.yml b/metron-deployment/ansible/roles/librdkafka/defaults/main.yml index 063c22f..6e25cb3 100644 --- a/metron-deployment/ansible/roles/librdkafka/defaults/main.yml +++ b/metron-deployment/ansible/roles/librdkafka/defaults/main.yml @@ -15,6 +15,6 @@ # limitations under the License. # --- -librdkafka_version: 0.9.4 -librdkafka_url: https://github.com/edenhill/librdkafka/archive/v0.9.4.tar.gz +librdkafka_version: 0.11.5 +librdkafka_url: https://github.com/edenhill/librdkafka/archive/v0.11.5.tar.gz librdkafka_home: /usr/local http://git-wip-us.apache.org/repos/asf/metron/blob/8bf3b6ec/metron-deployment/development/README.md ---------------------------------------------------------------------- diff --git a/metron-deployment/development/README.md b/metron-deployment/development/README.md index 2a04e5f..b86a5c4 100644 --- a/metron-deployment/development/README.md +++ b/metron-deployment/development/README.md @@ -27,3 +27,37 @@ This directory contains environments useful for Metron developers. These enviro ## Vagrant Cachier recommendations The development boxes are designed to be spun up and destroyed on a regular basis as part of the development cycle. In order to avoid the overhead of re-downloading many of the heavy platform dependencies, Vagrant can use the [vagrant-cachier](http://fgrehm.viewdocs.io/vagrant-cachier/) plugin to store package caches between builds. If the plugin has been installed to your vagrant it will be used, and packages will be cached in ~/.vagrant/cache. + +## Knox Demo LDAP + +The development environment can be set up to authenticate against Knox's demo LDAP. + +A couple notes +* A custom LDIF file is used to setup users. This is to get the roles and passwords setup correctly. +* The demo LDAP uses plaintext passwords with no encryption prefix (e.g. {SSHA}). +* You may need or want to shut down any or all of the topologies. This is optional, but clears some room + +To setup this up, start full dev. +* In Ambari, add the Knox service (Actions -> +Add Service). Accept all defaults and let it install. The configs that will be set how we need by default are: + * LDAP URL = ldap://localhost:33389 + * User dn pattern = uid={0},ou=people,dc=hadoop,dc=apache,dc=org + * LDAP user searchbase = ou=people,dc=hadoop,dc=apache,dc=org + * Group Search Base = ou=groups,dc=hadoop,dc=apache,dc=org + * Group Search Filter = member={0} + * User Base DN = uid=admin,ou=people,dc=hadoop,dc=apache,dc=org + * User Search Filter is empty + * User password attribute = userPassword + * LDAP group role attribute = cn + * Bind User = uid=admin,ou=people,dc=hadoop,dc=apache,dc=org + * LDAP Truststore is empty + * LDAP Truststore Password is empty + +* In the Knox configuration, go to "Advanced users-ldif". We have a custom ldif file "knox-demo-ldap.ldif" in "metron-deployment/development" that contains a customized variant of the users and groups defined here. Replace the default ldif configuration with the contents of "knox-demo-ldap.ldif" +* Start the Demo LDAP (In Knox, "Service Actions -> Start Demo LDAP) +* In Metron's configs, we're going to make two changes + * Set "LDAP Enabled" to "On" + * In Security, set "Bind user password" to match the admin user's password from the ldif file (admin-password). +* Restart the REST application + +Now, when you go to Swagger or the UIs, you should be able to give a user and password. +"admin" will have the roles ROLE_ADMIN and ROLE_USER, which can be verified via the "/whoami/roles" endpoint in Swagger. Similarly, there is a user "sam" that only has ROLE_USER. A third user, "tom" has neither role. http://git-wip-us.apache.org/repos/asf/metron/blob/8bf3b6ec/metron-deployment/development/centos6/README.md ---------------------------------------------------------------------- diff --git a/metron-deployment/development/centos6/README.md b/metron-deployment/development/centos6/README.md index e873452..99ec967 100644 --- a/metron-deployment/development/centos6/README.md +++ b/metron-deployment/development/centos6/README.md @@ -97,8 +97,30 @@ In addition to re-running the entire provisioning play book, you may now re-run vagrant --ansible-tags="sensor-stubs" provision ``` -Tags are listed in the playbooks, some frequently used tags: +Tags are listed in the playbooks. Here are some frequently used tags: + `hdp-install` - Install HDP + `hdp-deploy` - Deploy and Start HDP Services (will start all Hadoop Services) -+ `sensors` - Deploy and start the sensors. ++ `sensors` - Deploy the sensors (see [Sensors](#sensors) for more details regarding this tag) + `sensor-stubs` - Deploy and start the sensor stubs. + +#### Sensors + +By default, the Metron development environment uses sensor stubs to mimic the behavior of the full sensors. This is done because the full sensors take a significant amount of time and CPU to build, install, and run. + +From time to time you may want to install the full sensors for testing (see the specifics of what that means [here](../../ansible/playbooks/sensor_install.yml)). This can be done by running the following command: + +``` +vagrant --ansible-skip-tags="sensor-stubs" up +``` + +This will skip only the `sensor-stubs` tag, allowing the ansible roles with the `sensors` tag to be run. This provisions the full sensors in a 'testing mode' so that they are more active, and thus more useful for testing (more details on that [here](../../ansible/roles/sensor-test-mode/)). **However**, when vagrant completes the sensors will NOT be running. In order to start the sensors and simulate traffic through them (which will create a fair amount of load on your test system), complete the below steps: + +``` +vagrant ssh +sudo su - +service pcap-replay restart +service yaf restart +service snortd restart +service snort-producer restart +``` + http://git-wip-us.apache.org/repos/asf/metron/blob/8bf3b6ec/metron-deployment/development/knox-demo-ldap.ldif ---------------------------------------------------------------------- diff --git a/metron-deployment/development/knox-demo-ldap.ldif b/metron-deployment/development/knox-demo-ldap.ldif new file mode 100644 index 0000000..3097a64 --- /dev/null +++ b/metron-deployment/development/knox-demo-ldap.ldif @@ -0,0 +1,101 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +version: 1 + +# Please replace with site specific values +dn: dc=hadoop,dc=apache,dc=org +objectclass: organization +objectclass: dcObject +o: Hadoop +dc: hadoop + +# Entry for a sample people container +# Please replace with site specific values +dn: ou=people,dc=hadoop,dc=apache,dc=org +objectclass:top +objectclass:organizationalUnit +ou: people + +# Entry for a sample end user +# Please replace with site specific values +dn: uid=guest,ou=people,dc=hadoop,dc=apache,dc=org +objectclass:top +objectclass:person +objectclass:organizationalPerson +objectclass:inetOrgPerson +cn: Guest +sn: User +uid: guest +userPassword:guest-password + + +# entry for sample user admin +dn: uid=admin,ou=people,dc=hadoop,dc=apache,dc=org +objectclass:top +objectclass:person +objectclass:organizationalPerson +objectclass:inetOrgPerson +cn: Admin +sn: Admin +uid: admin +userPassword:admin-password + +# entry for sample user sam +dn: uid=sam,ou=people,dc=hadoop,dc=apache,dc=org +objectclass:top +objectclass:person +objectclass:organizationalPerson +objectclass:inetOrgPerson +cn: sam +sn: sam +uid: sam +userPassword:sam-password + +# entry for sample user tom +dn: uid=tom,ou=people,dc=hadoop,dc=apache,dc=org +objectclass:top +objectclass:person +objectclass:organizationalPerson +objectclass:inetOrgPerson +cn: tom +sn: tom +uid: tom +userPassword:tom-password + +# create FIRST Level groups branch +dn: ou=groups,dc=hadoop,dc=apache,dc=org +objectclass:top +objectclass:organizationalUnit +ou: groups +description: generic groups branch + +# create the admin group under groups +dn: cn=admin,ou=groups,dc=hadoop,dc=apache,dc=org +objectclass:top +objectclass: groupofnames +cn: admin +description:admin group +member: uid=admin,ou=people,dc=hadoop,dc=apache,dc=org + +# create the user group under groups +dn: cn=user,ou=groups,dc=hadoop,dc=apache,dc=org +objectclass:top +objectclass: groupofnames +cn: user +description: user group +member: uid=sam,ou=people,dc=hadoop,dc=apache,dc=org +member: uid=admin,ou=people,dc=hadoop,dc=apache,dc=org http://git-wip-us.apache.org/repos/asf/metron/blob/8bf3b6ec/metron-deployment/development/ubuntu14/README.md ---------------------------------------------------------------------- diff --git a/metron-deployment/development/ubuntu14/README.md b/metron-deployment/development/ubuntu14/README.md index 494937e..53bea67 100644 --- a/metron-deployment/development/ubuntu14/README.md +++ b/metron-deployment/development/ubuntu14/README.md @@ -98,8 +98,30 @@ In addition to re-running the entire provisioning play book, you may now re-run vagrant --ansible-tags="sensor-stubs" provision ``` -Tags are listed in the playbooks, some frequently used tags: +Tags are listed in the playbooks. Here are some frequently used tags: + `hdp-install` - Install HDP + `hdp-deploy` - Deploy and Start HDP Services (will start all Hadoop Services) -+ `sensors` - Deploy and start the sensors. ++ `sensors` - Deploy the sensors (see [Sensors](#sensors) for more details regarding this tag) + `sensor-stubs` - Deploy and start the sensor stubs. + +#### Sensors + +By default, the Metron development environment uses sensor stubs to mimic the behavior of the full sensors. This is done because the full sensors take a significant amount of time and CPU to build, install, and run. + +From time to time you may want to install the full sensors for testing (see the specifics of what that means [here](../../ansible/playbooks/sensor_install.yml)). This can be done by running the following command: + +``` +vagrant --ansible-skip-tags="sensor-stubs" up +``` + +This will skip only the `sensor-stubs` tag, allowing the ansible roles with the `sensors` tag to be run. This provisions the full sensors in a 'testing mode' so that they are more active, and thus more useful for testing (more details on that [here](../../ansible/roles/sensor-test-mode/)). **However**, when vagrant completes the sensors will NOT be running. In order to start the sensors and simulate traffic through them (which will create a fair amount of load on your test system), complete the below steps: + +``` +vagrant ssh +sudo su - +service pcap-replay restart +service yaf restart +service snortd restart +service snort-producer restart +``` + http://git-wip-us.apache.org/repos/asf/metron/blob/8bf3b6ec/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-enrichment-env.xml ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-enrichment-env.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-enrichment-env.xml index b41c455..69dce3f 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-enrichment-env.xml +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-enrichment-env.xml @@ -165,17 +165,17 @@ </property> <property> <name>enrichment_topology</name> - <description>Which Enrichment topology to execute</description> - <value>Split-Join</value> + <description>Which Enrichment topology to execute. Note: Split-Join is deprecated in favor of the Unified topology.</description> + <value>Unified</value> <display-name>Enrichment Topology</display-name> <value-attributes> <type>value-list</type> <entries> <entry> - <value>Split-Join</value> + <value>Unified</value> </entry> <entry> - <value>Unified</value> + <value>Split-Join</value> </entry> </entries> <selection-cardinality>1</selection-cardinality> http://git-wip-us.apache.org/repos/asf/metron/blob/8bf3b6ec/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-indexing-env.xml ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-indexing-env.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-indexing-env.xml index 3d268b7..ebc9759 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-indexing-env.xml +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-indexing-env.xml @@ -146,7 +146,7 @@ <property> <name>ra_indexing_topology_max_spout_pending</name> <description>Indexing Topology Spout Max Pending Tuples</description> - <value/> + <value>300</value> <display-name>Indexing Max Pending for Random Access</display-name> <value-attributes> <empty-value-valid>true</empty-value-valid> @@ -155,7 +155,7 @@ <property> <name>batch_indexing_topology_max_spout_pending</name> <description>Indexing Topology Spout Max Pending Tuples</description> - <value/> + <value>300</value> <display-name>Indexing Max Pending for HDFS</display-name> <value-attributes> <empty-value-valid>true</empty-value-valid> http://git-wip-us.apache.org/repos/asf/metron/blob/8bf3b6ec/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-profiler-env.xml ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-profiler-env.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-profiler-env.xml index 07b8d11..c45d799 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-profiler-env.xml +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-profiler-env.xml @@ -165,7 +165,7 @@ <name>profiler_topology_max_spout_pending</name> <description>Profiler Topology Spout Max Pending Tuples</description> <display-name>Spout Max Pending Tuples</display-name> - <value/> + <value>300</value> <value-attributes> <empty-value-valid>true</empty-value-valid> </value-attributes> http://git-wip-us.apache.org/repos/asf/metron/blob/8bf3b6ec/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-rest-env.xml ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-rest-env.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-rest-env.xml index f4b2327..55b880f 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-rest-env.xml +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-rest-env.xml @@ -35,32 +35,32 @@ </property> <property> <name>metron_spring_profiles_active</name> - <description>Active Spring profiles</description> + <description>Active Spring profiles. 'ldap' is used to enable authentication via LDAP.</description> <display-name>Active Spring profiles</display-name> <value/> <value-attributes> <empty-value-valid>true</empty-value-valid> </value-attributes> </property> - <property require-input="true"> + <property> <name>metron_jdbc_driver</name> <value></value> <description>Class name of the JDBC Driver used by Metron</description> <display-name>Metron JDBC Driver</display-name> </property> - <property require-input="true"> + <property> <name>metron_jdbc_url</name> <value></value> <description>JDBC Connection URL used by Metron</description> <display-name>Metron JDBC URL</display-name> </property> - <property require-input="true"> + <property> <name>metron_jdbc_username</name> <value></value> <description>Metron JDBC Username</description> <display-name>Metron JDBC username</display-name> </property> - <property require-input="true"> + <property> <name>metron_jdbc_password</name> <value></value> <property-type>PASSWORD</property-type> http://git-wip-us.apache.org/repos/asf/metron/blob/8bf3b6ec/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-security-env.xml ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-security-env.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-security-env.xml new file mode 100644 index 0000000..ab1fe6c --- /dev/null +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-security-env.xml @@ -0,0 +1,186 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<configuration supports_final="true"> + <property> + <name>metron.ldap.enabled</name> + <display-name>LDAP Enabled</display-name> + <value>false</value> + <description>Enable LDAP for Authentication</description> + <value-attributes> + <type>value-list</type> + <entries> + <entry> + <value>true</value> + <label>On</label> + </entry> + <entry> + <value>false</value> + <label>Off</label> + </entry> + </entries> + <selection-cardinality>1</selection-cardinality> + <overridable>false</overridable> + </value-attributes> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>metron.ldap.url</name> + <display-name>LDAP URL</display-name> + <value>ldap://localhost:33389</value> + <description>LDAP Server URL</description> + <value-attributes> + <overridable>false</overridable> + </value-attributes> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>metron.ldap.user.dnpattern</name> + <value>uid={0},ou=people,dc=hadoop,dc=apache,dc=org</value> + <display-name>User dn pattern</display-name> + <description>LDAP user DN</description> + <value-attributes> + <overridable>false</overridable> + </value-attributes> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>metron.ldap.user.searchbase</name> + <display-name>User Search Base</display-name> + <value>ou=people,dc=hadoop,dc=apache,dc=org</value> + <description>LDAP user searchbase</description> + <value-attributes> + <overridable>false</overridable> + <empty-value-valid>true</empty-value-valid> + </value-attributes> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>metron.ldap.group.searchbase</name> + <display-name>Group Search Base</display-name> + <value>ou=groups,dc=hadoop,dc=apache,dc=org</value> + <description>LDAP group searchbase</description> + <value-attributes> + <overridable>false</overridable> + <empty-value-valid>true</empty-value-valid> + </value-attributes> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>metron.ldap.group.searchfilter</name> + <display-name>Group Search Filter</display-name> + <value>member={0}</value> + <description>LDAP group search filter</description> + <value-attributes> + <overridable>false</overridable> + <empty-value-valid>true</empty-value-valid> + </value-attributes> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>metron.ldap.user.basedn</name> + <display-name>User Base DN</display-name> + <value>uid=admin,ou=people,dc=hadoop,dc=apache,dc=org</value> + <description>LDAP User Base DN</description> + <value-attributes> + <overridable>false</overridable> + <empty-value-valid>true</empty-value-valid> + </value-attributes> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>metron.ldap.user.searchfilter</name> + <display-name>User Search Filter</display-name> + <value></value> + <description>Search filter used for Bind Authentication</description> + <value-attributes> + <overridable>false</overridable> + <empty-value-valid>true</empty-value-valid> + </value-attributes> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>metron.ldap.user.password</name> + <value>userPassword</value> + <display-name>User password attribute</display-name> + <description>LDAP attribute for the user password</description> + <value-attributes> + <overridable>false</overridable> + </value-attributes> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>metron.ldap.group.roleattribute</name> + <display-name>LDAP group role attribute</display-name> + <value>cn</value> + <description>The LDAP group attribute to be used for determining roles</description> + <value-attributes> + <overridable>false</overridable> + </value-attributes> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>metron.ldap.bind.dn</name> + <display-name>Bind User</display-name> + <value>uid=admin,ou=people,dc=hadoop,dc=apache,dc=org</value> + <description>Full distinguished name (DN), of an LDAP user account that has privileges to search for users. </description> + <on-ambari-upgrade add="true"/> + <value-attributes> + <overridable>false</overridable> + <empty-value-valid>true</empty-value-valid> + </value-attributes> + </property> + <property> + <name>metron.ldap.bind.password</name> + <display-name>Bind User Password</display-name> + <value></value> + <property-type>PASSWORD</property-type> + <description>Password for the account that can search for users</description> + <value-attributes> + <overridable>false</overridable> + <empty-value-valid>true</empty-value-valid> + </value-attributes> + <on-ambari-upgrade add="true"/> + </property> + + <property> + <name>metron.ldap.ssl.truststore</name> + <display-name>LDAP Truststore</display-name> + <value></value> + <description>Path of truststore with SSL certs for LDAP</description> + <value-attributes> + <overridable>false</overridable> + <empty-value-valid>true</empty-value-valid> + </value-attributes> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>metron.ldap.ssl.truststore.password</name> + <display-name>LDAP Truststore Password</display-name> + <value></value> + <property-type>PASSWORD</property-type> + <description>Password for the truststore with SSL certs for LDAP</description> + <value-attributes> + <overridable>false</overridable> + <empty-value-valid>true</empty-value-valid> + </value-attributes> + <on-ambari-upgrade add="true"/> + </property> + +</configuration> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/metron/blob/8bf3b6ec/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/metainfo.xml ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/metainfo.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/metainfo.xml index ad1f7a9..97b5749 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/metainfo.xml +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/metainfo.xml @@ -322,6 +322,7 @@ <scriptType>PYTHON</scriptType> </commandScript> <configuration-dependencies> + <config-type>metron-security-env</config-type> <config-type>metron-indexing-env</config-type> <config-type>metron-rest-env</config-type> <config-type>metron-pcap-env</config-type> @@ -371,6 +372,7 @@ <scriptType>PYTHON</scriptType> </commandScript> <configuration-dependencies> + <config-type>metron-security-env</config-type> <config-type>metron-rest-env</config-type> <config-type>metron-management-ui-env</config-type> </configuration-dependencies> @@ -397,6 +399,7 @@ <scriptType>PYTHON</scriptType> </commandScript> <configuration-dependencies> + <config-type>metron-security-env</config-type> <config-type>metron-rest-env</config-type> <config-type>metron-alerts-ui-env</config-type> </configuration-dependencies> http://git-wip-us.apache.org/repos/asf/metron/blob/8bf3b6ec/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/bro_index.template ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/bro_index.template b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/bro_index.template index 17ad4d2..48e76c7 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/bro_index.template +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/bro_index.template @@ -133,6 +133,9 @@ "source:type": { "type": "keyword" }, + "alert_status": { + "type": "keyword" + }, /* * Widely-used Bro fields (potentially renamed during Metron ingest) */ http://git-wip-us.apache.org/repos/asf/metron/blob/8bf3b6ec/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/metaalert_index.template ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/metaalert_index.template b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/metaalert_index.template index 05d5e32..040c411 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/metaalert_index.template +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/metaalert_index.template @@ -5,8 +5,8 @@ "dynamic_templates": [ { "alert_template": { - "path_match": "alert.*", - "match_mapping_type": "string", + "path_match": "metron_alert.*", + "match_mapping_type": "*", "mapping": { "type": "keyword" } http://git-wip-us.apache.org/repos/asf/metron/blob/8bf3b6ec/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/snort_index.template ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/snort_index.template b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/snort_index.template index f7c6e59..139dbc9 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/snort_index.template +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/snort_index.template @@ -123,6 +123,9 @@ "source:type": { "type": "keyword" }, + "alert_status": { + "type": "keyword" + }, "ip_dst_addr": { "type": "ip" }, http://git-wip-us.apache.org/repos/asf/metron/blob/8bf3b6ec/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/yaf_index.template ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/yaf_index.template b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/yaf_index.template index f4093ba..4a3e5f9 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/yaf_index.template +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/yaf_index.template @@ -123,6 +123,9 @@ "source:type": { "type": "keyword" }, + "alert_status": { + "type": "keyword" + }, "ip_dst_addr": { "type": "ip" }, http://git-wip-us.apache.org/repos/asf/metron/blob/8bf3b6ec/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py index dd00e9c..5635330 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py @@ -58,7 +58,17 @@ metron_management_ui_port = status_params.metron_management_ui_port metron_alerts_ui_host = status_params.metron_alerts_ui_host metron_alerts_ui_port = status_params.metron_alerts_ui_port metron_jvm_flags = config['configurations']['metron-rest-env']['metron_jvm_flags'] -metron_spring_profiles_active = config['configurations']['metron-rest-env']['metron_spring_profiles_active'] + +# Construct the profiles as a temp variable first. Only the first time it's set will carry through +metron_spring_profiles_temp = config['configurations']['metron-rest-env']['metron_spring_profiles_active'] +if config['configurations']['metron-security-env']['metron.ldap.enabled']: + if metron_spring_profiles_temp: + metron_spring_profiles_active = metron_spring_profiles_temp + ',ldap' + else: + metron_spring_profiles_active = 'ldap' +else: + metron_spring_profiles_active = metron_spring_profiles_temp + metron_jdbc_driver = config['configurations']['metron-rest-env']['metron_jdbc_driver'] metron_jdbc_url = config['configurations']['metron-rest-env']['metron_jdbc_url'] metron_jdbc_username = config['configurations']['metron-rest-env']['metron_jdbc_username'] @@ -266,6 +276,21 @@ if security_enabled: if 'solr-config-env' in config['configurations']: solr_principal_name = solr_principal_name.replace('_HOST', hostname_lowercase) +# LDAP +metron_ldap_url = config['configurations']['metron-security-env']['metron.ldap.url'] +metron_ldap_userdn = config['configurations']['metron-security-env']['metron.ldap.bind.dn'] +metron_ldap_password = config['configurations']['metron-security-env']['metron.ldap.bind.password'] +metron_ldap_user_pattern = config['configurations']['metron-security-env']['metron.ldap.user.dnpattern'] +metron_ldap_user_password = config['configurations']['metron-security-env']['metron.ldap.user.password'] +metron_ldap_user_dnbase = config['configurations']['metron-security-env']['metron.ldap.user.basedn'] +metron_ldap_user_searchbase = config['configurations']['metron-security-env']['metron.ldap.user.searchbase'] +metron_ldap_user_searchfilter = config['configurations']['metron-security-env']['metron.ldap.user.searchfilter'] +metron_ldap_group_searchbase = config['configurations']['metron-security-env']['metron.ldap.group.searchbase'] +metron_ldap_group_searchfilter = config['configurations']['metron-security-env']['metron.ldap.group.searchfilter'] +metron_ldap_group_role = config['configurations']['metron-security-env']['metron.ldap.group.roleattribute'] +metron_ldap_ssl_truststore = config['configurations']['metron-security-env']['metron.ldap.ssl.truststore'] +metron_ldap_ssl_truststore_password = config['configurations']['metron-security-env']['metron.ldap.ssl.truststore.password'] + # Management UI metron_rest_host = default("/clusterHostInfo/metron_rest_hosts", [hostname])[0] http://git-wip-us.apache.org/repos/asf/metron/blob/8bf3b6ec/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_commands.py ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_commands.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_commands.py index c410b94..bab9129 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_commands.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_commands.py @@ -145,8 +145,12 @@ class RestCommands: "export METRON_PID_FILE={pid_file};" "export HDP_VERSION={hdp_version};" "export METRON_RA_INDEXING_WRITER={ra_indexing_writer};" + "export METRON_LDAP_PASSWORD={metron_ldap_password!p};" + "export METRON_LDAP_SSL_TRUSTSTORE_PASSWORD={metron_ldap_ssl_truststore_password!p};" "{metron_home}/bin/metron-rest.sh;" "unset METRON_JDBC_PASSWORD;" + "unset METRON_LDAP_PASSWORD;" + "unset METRON_LDAP_SSL_TRUSTSTORE_PASSWORD;" )) Execute(cmd, http://git-wip-us.apache.org/repos/asf/metron/blob/8bf3b6ec/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/metron.j2 ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/metron.j2 b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/metron.j2 index a7d01e5..08d4281 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/metron.j2 +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/metron.j2 @@ -21,6 +21,8 @@ METRON_PID_DIR="{{metron_pid_dir}}" METRON_REST_PORT={{metron_rest_port}} METRON_JVMFLAGS="{{metron_jvm_flags}}" METRON_SPRING_PROFILES_ACTIVE="{{metron_spring_profiles_active}}" + +#JDBC METRON_JDBC_DRIVER="{{metron_jdbc_driver}}" METRON_JDBC_URL="{{metron_jdbc_url}}" METRON_JDBC_USERNAME="{{metron_jdbc_username}}" @@ -28,6 +30,20 @@ METRON_JDBC_PLATFORM="{{metron_jdbc_platform}}" METRON_JDBC_CLIENT_PATH="{{metron_jdbc_client_path}}" METRON_TEMP_GROK_PATH="{{metron_temp_grok_path}}" METRON_SPRING_OPTIONS="{{metron_spring_options}}" + +#LDAP +METRON_LDAP_URL="{{metron_ldap_url}}" +METRON_LDAP_USERDN="{{metron_ldap_userdn}}" +METRON_LDAP_USER_PATTERN="{{metron_ldap_user_pattern}}" +METRON_LDAP_USER_DNBASE="{{metron_ldap_user_dnbase}}" +METRON_LDAP_USER_SEARCHBASE="{{metron_ldap_user_searchbase}}" +METRON_LDAP_USER_PASSWORD="{{metron_ldap_user_password}}" +METRON_LDAP_USER_SEARCHFILTER="{{metron_ldap_user_searchfilter}}" +METRON_LDAP_GROUP_SEARCHBASE="{{metron_ldap_group_searchbase}}" +METRON_LDAP_GROUP_SEARCHFILTER="{{metron_ldap_group_searchfilter}}" +METRON_LDAP_GROUP_ROLE="{{metron_ldap_group_role}}" +METRON_LDAP_SSL_TRUSTSTORE="{{metron_ldap_ssl_truststore}}" + ZOOKEEPER="{{zookeeper_quorum}}" BROKERLIST="{{kafka_brokers}}" HADOOP_CONF_DIR="/etc/hadoop/conf/" http://git-wip-us.apache.org/repos/asf/metron/blob/8bf3b6ec/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/themes/metron_theme.json ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/themes/metron_theme.json b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/themes/metron_theme.json index 26c7f4e..7f84f1d 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/themes/metron_theme.json +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/themes/metron_theme.json @@ -125,7 +125,7 @@ ] }, { - "name": "section-enrichment-splitjoin", + "name": "section-enrichment-unified", "row-index": "3", "column-index": "0", "row-span": "1", @@ -134,8 +134,8 @@ "section-rows": "1", "subsections": [ { - "name": "subsection-enrichment-splitjoin", - "display-name": "Split Join Topology", + "name": "subsection-enrichment-unified", + "display-name": "Unified Topology", "row-index": "0", "column-index": "0", "row-span": "1", @@ -144,7 +144,7 @@ ] }, { - "name": "section-enrichment-unified", + "name": "section-enrichment-splitjoin", "row-index": "4", "column-index": "0", "row-span": "1", @@ -153,8 +153,8 @@ "section-rows": "1", "subsections": [ { - "name": "subsection-enrichment-unified", - "display-name": "Unified Topology", + "name": "subsection-enrichment-splitjoin", + "display-name": "Split Join Topology", "row-index": "0", "column-index": "0", "row-span": "1", @@ -422,6 +422,35 @@ } }, { + "name": "security", + "display-name": "Security", + "layout": { + "tab-columns": "2", + "tab-rows": "1", + "sections": [ + { + "name": "section-security-ldap", + "row-index": "0", + "column-index": "0", + "row-span": "1", + "column-span": "1", + "section-columns": "1", + "section-rows": "1", + "subsections": [ + { + "name": "subsection-security-ldap", + "display-name": "LDAP", + "row-index": "0", + "column-index": "0", + "row-span": "1", + "column-span": "1" + } + ] + } + ] + } + }, + { "name": "metron-pcap", "display-name": "PCAP", "layout": { @@ -852,6 +881,58 @@ "subsection-name": "subsection-alerts-ui" }, { + "config": "metron-security-env/metron.ldap.enabled", + "subsection-name": "subsection-security-ldap" + }, + { + "config": "metron-security-env/metron.ldap.url", + "subsection-name": "subsection-security-ldap" + }, + { + "config": "metron-security-env/metron.ldap.bind.dn", + "subsection-name": "subsection-security-ldap" + }, + { + "config": "metron-security-env/metron.ldap.bind.password", + "subsection-name": "subsection-security-ldap" + }, + { + "config": "metron-security-env/metron.ldap.user.dnpattern", + "subsection-name": "subsection-security-ldap" + }, + { + "config": "metron-security-env/metron.ldap.user.password", + "subsection-name": "subsection-security-ldap" + }, + { + "config": "metron-security-env/metron.ldap.user.searchbase", + "subsection-name": "subsection-security-ldap" + }, + { + "config": "metron-security-env/metron.ldap.user.searchfilter", + "subsection-name": "subsection-security-ldap" + }, + { + "config": "metron-security-env/metron.ldap.group.searchbase", + "subsection-name": "subsection-security-ldap" + }, + { + "config": "metron-security-env/metron.ldap.group.searchfilter", + "subsection-name": "subsection-security-ldap" + }, + { + "config": "metron-security-env/metron.ldap.group.roleattribute", + "subsection-name": "subsection-security-ldap" + }, + { + "config": "metron-security-env/metron.ldap.ssl.truststore", + "subsection-name": "subsection-security-ldap" + }, + { + "config": "metron-security-env/metron.ldap.ssl.truststore.password", + "subsection-name": "subsection-security-ldap" + }, + { "config": "metron-pcap-env/pcap_topology_workers", "subsection-name": "subsection-pcap" }, @@ -1529,6 +1610,84 @@ } }, { + "config": "metron-security-env/metron.ldap.enabled", + "widget": { + "type": "toggle" + } + }, + { + "config": "metron-security-env/metron.ldap.url", + "widget": { + "type": "text-field" + } + }, + { + "config": "metron-security-env/metron.ldap.bind.dn", + "widget": { + "type": "text-field" + } + }, + { + "config": "metron-security-env/metron.ldap.bind.password", + "widget": { + "type": "password" + } + }, + { + "config": "metron-security-env/metron.ldap.ssl.truststore", + "widget": { + "type": "text-field" + } + }, + { + "config": "metron-security-env/metron.ldap.ssl.truststore.password", + "widget": { + "type": "password" + } + }, + { + "config": "metron-security-env/metron.ldap.user.dnpattern", + "widget": { + "type": "text-field" + } + }, + { + "config": "metron-security-env/metron.ldap.user.password", + "widget": { + "type": "text-field" + } + }, + { + "config": "metron-security-env/metron.ldap.user.searchbase", + "widget": { + "type": "text-field" + } + }, + { + "config": "metron-security-env/metron.ldap.user.searchfilter", + "widget": { + "type": "text-field" + } + }, + { + "config": "metron-security-env/metron.ldap.group.searchbase", + "widget": { + "type": "text-field" + } + }, + { + "config": "metron-security-env/metron.ldap.group.searchfilter", + "widget": { + "type": "text-field" + } + }, + { + "config": "metron-security-env/metron.ldap.group.roleattribute", + "widget": { + "type": "text-field" + } + }, + { "config": "metron-pcap-env/pcap_topology_workers", "widget": { "type": "text-field" http://git-wip-us.apache.org/repos/asf/metron/blob/8bf3b6ec/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec b/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec index ed22a28..14c08a1 100644 --- a/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec +++ b/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec @@ -461,15 +461,16 @@ This package installs the Metron Management UI %{metron_home} %attr(0755,root,root) %{metron_home}/web/expressjs/node_modules/.bin/* %attr(0755,root,root) %{metron_home}/web/expressjs/server.js %attr(0644,root,root) %{metron_home}/web/expressjs/package.json +%attr(0644,root,root) %{metron_home}/web/management-ui/styles.*.css %attr(0644,root,root) %{metron_home}/web/management-ui/favicon.ico %attr(0644,root,root) %{metron_home}/web/management-ui/index.html %attr(0644,root,root) %{metron_home}/web/management-ui/*.js -%attr(0644,root,root) %{metron_home}/web/management-ui/*.js.gz %attr(0644,root,root) %{metron_home}/web/management-ui/*.ttf %attr(0644,root,root) %{metron_home}/web/management-ui/*.svg %attr(0644,root,root) %{metron_home}/web/management-ui/*.eot %attr(0644,root,root) %{metron_home}/web/management-ui/*.woff %attr(0644,root,root) %{metron_home}/web/management-ui/*.woff2 +%attr(0644,root,root) %{metron_home}/web/management-ui/3rdpartylicenses.txt %attr(0644,root,root) %{metron_home}/web/management-ui/assets/ace/*.js %attr(0644,root,root) %{metron_home}/web/management-ui/assets/ace/LICENSE %attr(0644,root,root) %{metron_home}/web/management-ui/assets/ace/snippets/*.js @@ -589,6 +590,8 @@ chkconfig --del metron-management-ui chkconfig --del metron-alerts-ui %changelog +* Wed Oct 31 2018 Apache Metron <d...@metron.apache.org> - 0.6.1 +- Update files in Management UI from Angular upgrade * Thu Aug 30 2018 Apache Metron <d...@metron.apache.org> - 0.6.1 - Update compiled css file name for Alerts UI * Fri Aug 24 2018 Apache Metron <d...@metron.apache.org> - 0.6.1 http://git-wip-us.apache.org/repos/asf/metron/blob/8bf3b6ec/metron-interface/metron-alerts/cypress.json ---------------------------------------------------------------------- diff --git a/metron-interface/metron-alerts/cypress.json b/metron-interface/metron-alerts/cypress.json new file mode 100644 index 0000000..7c0410d --- /dev/null +++ b/metron-interface/metron-alerts/cypress.json @@ -0,0 +1,7 @@ +{ + "viewportWidth": 1435, + "viewportHeight": 850, + "video": false, + "supportFile": false, + "pluginsFile": false +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/metron/blob/8bf3b6ec/metron-interface/metron-alerts/cypress/fixtures/config.json ---------------------------------------------------------------------- diff --git a/metron-interface/metron-alerts/cypress/fixtures/config.json b/metron-interface/metron-alerts/cypress/fixtures/config.json new file mode 100644 index 0000000..190e514 --- /dev/null +++ b/metron-interface/metron-alerts/cypress/fixtures/config.json @@ -0,0 +1,23 @@ +{ + "es.clustername":"metron", + "es.ip":"node1:9300", + "es.date.format":"yyyy.MM.dd.HH", + "parser.error.topic":"indexing", + "update.hbase.table":"metron_update", + "update.hbase.cf":"t", + "es.client.settings":{ + "client.transport.ping_timeout":"500s" + }, + "profiler.client.period.duration":"15", + "profiler.client.period.duration.units":"MINUTES", + "user.settings.hbase.table":"user_settings", + "user.settings.hbase.cf":"cf", + "bootstrap.servers":"node1:6667", + "source.type.field":"source:type", + "threat.triage.score.field":"threat:triage:score", + "enrichment.writer.batchSize":"15", + "enrichment.writer.batchTimeout":"0", + "profiler.writer.batchSize":"15", + "profiler.writer.batchTimeout":"0", + "geo.hdfs.file":"/apps/metron/geo/default/GeoLite2-City.mmdb.gz" +} \ No newline at end of file
