METRON-1051: Enable the ability to update indexed messages closes apache/incubator-metron#666
Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/813adf2d Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/813adf2d Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/813adf2d Branch: refs/heads/master Commit: 813adf2dac1b4c07c0e4ae134366c61388ff5b67 Parents: 3d47513 Author: cstella <[email protected]> Authored: Mon Aug 7 14:14:44 2017 -0400 Committer: cstella <[email protected]> Committed: Mon Aug 7 14:14:44 2017 -0400 ---------------------------------------------------------------------- metron-analytics/metron-profiler-client/pom.xml | 13 + .../metron/profiler/client/GetProfileTest.java | 23 +- .../client/HBaseProfilerClientTest.java | 4 +- metron-analytics/metron-profiler/pom.xml | 13 + .../integration/ProfilerIntegrationTest.java | 30 +- .../configuration/metron-indexing-env.xml | 12 + .../package/scripts/indexing_commands.py | 64 ++ .../CURRENT/package/scripts/indexing_master.py | 5 + .../package/scripts/params/params_linux.py | 5 + .../package/scripts/params/status_params.py | 4 + .../CURRENT/package/templates/global.json.j2 | 4 +- .../METRON/CURRENT/themes/metron_theme.json | 27 +- metron-interface/metron-rest/README.md | 70 ++ metron-interface/metron-rest/pom.xml | 16 + .../apache/metron/rest/MetronRestConstants.java | 1 + .../apache/metron/rest/config/IndexConfig.java | 40 +- .../rest/controller/SearchController.java | 22 + .../rest/controller/UpdateController.java | 70 ++ .../metron/rest/service/SearchService.java | 5 + .../metron/rest/service/UpdateService.java | 29 + .../service/impl/IndexDaoSearchServiceImpl.java | 75 -- .../rest/service/impl/SearchServiceImpl.java | 87 +++ .../rest/service/impl/UpdateServiceImpl.java | 58 ++ .../src/main/resources/application-test.yml | 7 +- .../src/main/resources/application.yml | 3 +- .../metron-rest/src/main/scripts/metron-rest | 3 +- .../apache/metron/rest/config/TestConfig.java | 19 +- .../rest/controller/DaoControllerTest.java | 58 ++ .../SearchControllerIntegrationTest.java | 51 +- .../UpdateControllerIntegrationTest.java | 192 +++++ .../src/test/resources/zookeeper/global.json | 4 + .../apache/metron/common/utils/JSONUtils.java | 10 + metron-platform/metron-data-management/pom.xml | 7 + .../nonbulk/taxii/TaxiiIntegrationTest.java | 9 +- metron-platform/metron-elasticsearch/pom.xml | 17 + .../elasticsearch/dao/ElasticsearchDao.java | 114 ++- .../elasticsearch/utils/ElasticsearchUtils.java | 29 +- .../writer/ElasticsearchWriter.java | 12 + .../ElasticsearchDaoIntegrationTest.java | 147 ---- .../ElasticsearchSearchIntegrationTest.java | 147 ++++ .../ElasticsearchUpdateIntegrationTest.java | 219 ++++++ .../components/ElasticSearchComponent.java | 36 + metron-platform/metron-enrichment/pom.xml | 7 + .../simplehbase/SimpleHBaseAdapterTest.java | 7 +- .../threatintel/ThreatIntelAdapterTest.java | 7 +- .../integration/EnrichmentIntegrationTest.java | 24 +- .../components/ConfigUploadComponent.java | 12 +- .../integration/mock/MockTableProvider.java | 45 -- .../SimpleHBaseEnrichmentFunctionsTest.java | 17 +- metron-platform/metron-hbase-client/pom.xml | 100 +++ metron-platform/metron-hbase/pom.xml | 16 + .../org/apache/metron/hbase/TableProvider.java | 11 +- .../hbase/mock/MockHBaseTableProvider.java | 49 ++ .../apache/metron/hbase/mock/MockHTable.java | 701 ++++++++++++++++++ metron-platform/metron-indexing/README.md | 24 + metron-platform/metron-indexing/pom.xml | 46 ++ .../metron/indexing/dao/AccessConfig.java | 39 + .../apache/metron/indexing/dao/HBaseDao.java | 135 ++++ .../apache/metron/indexing/dao/IndexDao.java | 113 ++- .../metron/indexing/dao/IndexDaoFactory.java | 38 +- .../indexing/dao/IndexUpdateCallback.java | 27 + .../metron/indexing/dao/MultiIndexDao.java | 159 ++++ .../metron/indexing/dao/search/GetRequest.java | 47 ++ .../indexing/dao/search/SearchRequest.java | 20 + .../indexing/dao/search/SearchResponse.java | 8 + .../indexing/dao/search/SearchResult.java | 25 + .../metron/indexing/dao/update/Document.java | 88 +++ .../dao/update/OriginalNotFoundException.java | 28 + .../indexing/dao/update/PatchRequest.java | 101 +++ .../indexing/dao/update/ReplaceRequest.java | 77 ++ .../apache/metron/indexing/dao/InMemoryDao.java | 34 +- .../dao/IndexingDaoIntegrationTest.java | 475 ------------ .../indexing/dao/SearchIntegrationTest.java | 477 ++++++++++++ .../src/main/config/zookeeper/global.json | 4 +- metron-platform/metron-parsers/pom.xml | 7 + .../SimpleHBaseEnrichmentWriterTest.java | 11 +- ...pleHbaseEnrichmentWriterIntegrationTest.java | 10 +- .../org/apache/metron/test/mock/MockHTable.java | 722 ------------------- metron-platform/pom.xml | 1 + 79 files changed, 3844 insertions(+), 1629 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/metron/blob/813adf2d/metron-analytics/metron-profiler-client/pom.xml ---------------------------------------------------------------------- diff --git a/metron-analytics/metron-profiler-client/pom.xml b/metron-analytics/metron-profiler-client/pom.xml index 61d93f8..bba881d 100644 --- a/metron-analytics/metron-profiler-client/pom.xml +++ b/metron-analytics/metron-profiler-client/pom.xml @@ -81,6 +81,19 @@ </exclusions> </dependency> <dependency> + <groupId>org.apache.metron</groupId> + <artifactId>metron-hbase</artifactId> + <version>${project.parent.version}</version> + <scope>test</scope> + <type>test-jar</type> + <exclusions> + <exclusion> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-log4j12</artifactId> + </exclusion> + </exclusions> + </dependency> + <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-mapreduce-client-core</artifactId> <version>${global_hadoop_version}</version> http://git-wip-us.apache.org/repos/asf/metron/blob/813adf2d/metron-analytics/metron-profiler-client/src/test/java/org/apache/metron/profiler/client/GetProfileTest.java ---------------------------------------------------------------------- diff --git a/metron-analytics/metron-profiler-client/src/test/java/org/apache/metron/profiler/client/GetProfileTest.java b/metron-analytics/metron-profiler-client/src/test/java/org/apache/metron/profiler/client/GetProfileTest.java index 917a5ca..00d842c 100644 --- a/metron-analytics/metron-profiler-client/src/test/java/org/apache/metron/profiler/client/GetProfileTest.java +++ b/metron-analytics/metron-profiler-client/src/test/java/org/apache/metron/profiler/client/GetProfileTest.java @@ -20,12 +20,11 @@ package org.apache.metron.profiler.client; -import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.client.HTableInterface; +import org.apache.metron.hbase.mock.MockHBaseTableProvider; import org.apache.metron.stellar.dsl.Context; import org.apache.metron.stellar.dsl.functions.resolver.SimpleFunctionResolver; import org.apache.metron.stellar.dsl.functions.resolver.SingletonFunctionResolver; -import org.apache.metron.hbase.TableProvider; import org.apache.metron.profiler.ProfileMeasurement; import org.apache.metron.profiler.client.stellar.FixedLookback; import org.apache.metron.profiler.client.stellar.GetProfile; @@ -35,13 +34,10 @@ import org.apache.metron.profiler.hbase.SaltyRowKeyBuilder; import org.apache.metron.profiler.hbase.ValueOnlyColumnBuilder; import org.apache.metron.stellar.common.DefaultStellarStatefulExecutor; import org.apache.metron.stellar.common.StellarStatefulExecutor; -import org.apache.metron.test.mock.MockHTable; import org.junit.Assert; import org.junit.Before; import org.junit.Test; -import java.io.IOException; -import java.io.Serializable; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; @@ -69,18 +65,7 @@ public class GetProfileTest { private static final TimeUnit periodUnits2 = TimeUnit.HOURS; private static final int saltDivisor2 = 2050; - /** - * A TableProvider that allows us to mock HBase. - */ - public static class MockTableProvider implements TableProvider, Serializable { - MockHTable.Provider provider = new MockHTable.Provider(); - - @Override - public HTableInterface getTable(Configuration config, String tableName) throws IOException { - return provider.getTable(config, tableName); - } - } private <T> T run(String expression, Class<T> clazz) { return executor.execute(expression, state, clazz); @@ -100,7 +85,7 @@ public class GetProfileTest { @Before public void setup() { state = new HashMap<>(); - final HTableInterface table = MockHTable.Provider.addToCache(tableName, columnFamily); + final HTableInterface table = MockHBaseTableProvider.addToCache(tableName, columnFamily); // used to write values to be read during testing RowKeyBuilder rowKeyBuilder = new SaltyRowKeyBuilder(); @@ -111,7 +96,7 @@ public class GetProfileTest { Map<String, Object> global = new HashMap<String, Object>() {{ put(PROFILER_HBASE_TABLE.getKey(), tableName); put(PROFILER_COLUMN_FAMILY.getKey(), columnFamily); - put(PROFILER_HBASE_TABLE_PROVIDER.getKey(), MockTableProvider.class.getName()); + put(PROFILER_HBASE_TABLE_PROVIDER.getKey(), MockHBaseTableProvider.class.getName()); put(PROFILER_PERIOD.getKey(), Long.toString(periodDuration)); put(PROFILER_PERIOD_UNITS.getKey(), periodUnits.toString()); put(PROFILER_SALT_DIVISOR.getKey(), Integer.toString(saltDivisor)); @@ -152,7 +137,7 @@ public class GetProfileTest { Map<String, Object> global = new HashMap<String, Object>() {{ put(PROFILER_HBASE_TABLE.getKey(), tableName); put(PROFILER_COLUMN_FAMILY.getKey(), columnFamily); - put(PROFILER_HBASE_TABLE_PROVIDER.getKey(), MockTableProvider.class.getName()); + put(PROFILER_HBASE_TABLE_PROVIDER.getKey(), MockHBaseTableProvider.class.getName()); put(PROFILER_PERIOD.getKey(), Long.toString(periodDuration2)); put(PROFILER_PERIOD_UNITS.getKey(), periodUnits2.toString()); put(PROFILER_SALT_DIVISOR.getKey(), Integer.toString(saltDivisor2)); http://git-wip-us.apache.org/repos/asf/metron/blob/813adf2d/metron-analytics/metron-profiler-client/src/test/java/org/apache/metron/profiler/client/HBaseProfilerClientTest.java ---------------------------------------------------------------------- diff --git a/metron-analytics/metron-profiler-client/src/test/java/org/apache/metron/profiler/client/HBaseProfilerClientTest.java b/metron-analytics/metron-profiler-client/src/test/java/org/apache/metron/profiler/client/HBaseProfilerClientTest.java index 960e4d2..8519f10 100644 --- a/metron-analytics/metron-profiler-client/src/test/java/org/apache/metron/profiler/client/HBaseProfilerClientTest.java +++ b/metron-analytics/metron-profiler-client/src/test/java/org/apache/metron/profiler/client/HBaseProfilerClientTest.java @@ -20,6 +20,7 @@ package org.apache.metron.profiler.client; +import org.apache.metron.hbase.mock.MockHTable; import org.apache.metron.profiler.ProfileMeasurement; import org.apache.metron.profiler.hbase.ColumnBuilder; import org.apache.metron.profiler.hbase.RowKeyBuilder; @@ -27,11 +28,8 @@ import org.apache.metron.profiler.hbase.SaltyRowKeyBuilder; import org.apache.metron.profiler.hbase.ValueOnlyColumnBuilder; import org.apache.metron.stellar.common.DefaultStellarStatefulExecutor; import org.apache.metron.stellar.common.StellarStatefulExecutor; -import org.apache.metron.test.mock.MockHTable; import org.junit.After; -import org.junit.AfterClass; import org.junit.Before; -import org.junit.BeforeClass; import org.junit.Test; import java.util.Arrays; http://git-wip-us.apache.org/repos/asf/metron/blob/813adf2d/metron-analytics/metron-profiler/pom.xml ---------------------------------------------------------------------- diff --git a/metron-analytics/metron-profiler/pom.xml b/metron-analytics/metron-profiler/pom.xml index bd3ab9c..41888a1 100644 --- a/metron-analytics/metron-profiler/pom.xml +++ b/metron-analytics/metron-profiler/pom.xml @@ -130,6 +130,19 @@ </exclusions> </dependency> <dependency> + <groupId>org.apache.metron</groupId> + <artifactId>metron-hbase</artifactId> + <version>${project.parent.version}</version> + <scope>test</scope> + <type>test-jar</type> + <exclusions> + <exclusion> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-log4j12</artifactId> + </exclusion> + </exclusions> + </dependency> + <dependency> <groupId>com.esotericsoftware</groupId> <artifactId>kryo-shaded</artifactId> <version>${global_kryo_version}</version> http://git-wip-us.apache.org/repos/asf/metron/blob/813adf2d/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/integration/ProfilerIntegrationTest.java ---------------------------------------------------------------------- diff --git a/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/integration/ProfilerIntegrationTest.java b/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/integration/ProfilerIntegrationTest.java index b863ebc..ad96857 100644 --- a/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/integration/ProfilerIntegrationTest.java +++ b/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/integration/ProfilerIntegrationTest.java @@ -23,14 +23,13 @@ package org.apache.metron.profiler.integration; import com.google.common.base.Joiner; import org.adrianwalker.multilinestring.Multiline; import org.apache.commons.math.util.MathUtils; -import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.Cell; -import org.apache.hadoop.hbase.client.HTableInterface; import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.util.Bytes; import org.apache.metron.common.Constants; import org.apache.metron.common.utils.SerDeUtils; -import org.apache.metron.hbase.TableProvider; +import org.apache.metron.hbase.mock.MockHTable; +import org.apache.metron.hbase.mock.MockHBaseTableProvider; import org.apache.metron.integration.BaseIntegrationTest; import org.apache.metron.integration.ComponentRunner; import org.apache.metron.integration.UnableToStartException; @@ -40,7 +39,6 @@ import org.apache.metron.integration.components.ZKServerComponent; import org.apache.metron.profiler.hbase.ColumnBuilder; import org.apache.metron.profiler.hbase.ValueOnlyColumnBuilder; import org.apache.metron.statistics.OnlineStatisticsProvider; -import org.apache.metron.test.mock.MockHTable; import org.junit.After; import org.junit.AfterClass; import org.junit.Assert; @@ -49,8 +47,6 @@ import org.junit.BeforeClass; import org.junit.Test; import java.io.File; -import java.io.IOException; -import java.io.Serializable; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -119,18 +115,6 @@ public class ProfilerIntegrationTest extends BaseIntegrationTest { private static final String inputTopic = Constants.INDEXING_TOPIC; private static final String outputTopic = "profiles"; - /** - * A TableProvider that allows us to mock HBase. - */ - public static class MockTableProvider implements TableProvider, Serializable { - - MockHTable.Provider provider = new MockHTable.Provider(); - - @Override - public HTableInterface getTable(Configuration config, String tableName) throws IOException { - return provider.getTable(config, tableName); - } - } /** * Tests the first example contained within the README. @@ -316,13 +300,13 @@ public class ProfilerIntegrationTest extends BaseIntegrationTest { setProperty("profiler.hbase.batch", "10"); setProperty("profiler.hbase.flush.interval.seconds", "1"); setProperty("profiler.profile.ttl", "20"); - setProperty("hbase.provider.impl", "" + MockTableProvider.class.getName()); + setProperty("hbase.provider.impl", "" + MockHBaseTableProvider.class.getName()); setProperty("storm.auto.credentials", "[]"); setProperty("kafka.security.protocol", "PLAINTEXT"); }}; // create the mock table - profilerTable = (MockHTable) MockHTable.Provider.addToCache(tableName, columnFamily); + profilerTable = (MockHTable) MockHBaseTableProvider.addToCache(tableName, columnFamily); zkComponent = getZKServerComponent(topologyProperties); @@ -363,7 +347,7 @@ public class ProfilerIntegrationTest extends BaseIntegrationTest { @AfterClass public static void tearDownAfterClass() throws Exception { - MockHTable.Provider.clear(); + MockHBaseTableProvider.clear(); if (runner != null) { runner.stop(); } @@ -372,12 +356,12 @@ public class ProfilerIntegrationTest extends BaseIntegrationTest { @Before public void setup() { // create the mock table - profilerTable = (MockHTable) MockHTable.Provider.addToCache(tableName, columnFamily); + profilerTable = (MockHTable) MockHBaseTableProvider.addToCache(tableName, columnFamily); } @After public void tearDown() throws Exception { - MockHTable.Provider.clear(); + MockHBaseTableProvider.clear(); profilerTable.clear(); if (runner != null) { runner.reset(); http://git-wip-us.apache.org/repos/asf/metron/blob/813adf2d/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-indexing-env.xml ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-indexing-env.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-indexing-env.xml index e28e328..e36730a 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-indexing-env.xml +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-indexing-env.xml @@ -59,6 +59,18 @@ <description>Indexing Writer Class Name</description> <value>org.apache.metron.elasticsearch.writer.ElasticsearchWriter</value> <display-name>Indexing Writer Class Name</display-name> + </property> + <property> + <name>update_table</name> + <description>The HBase table which will hold edits to indexed data</description> + <value>metron_update</value> + <display-name>Indexing Update Table</display-name> + </property> + <property> + <name>update_cf</name> + <description>The HBase column family which will hold edits to indexed data</description> + <value>t</value> + <display-name>Indexing Update Column Family</display-name> </property> <property> <name>indexing_workers</name> http://git-wip-us.apache.org/repos/asf/metron/blob/813adf2d/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py index 711d4fc..1d8e914 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py @@ -33,6 +33,8 @@ class IndexingCommands: __configured = False __acl_configured = False __hdfs_perm_configured = False + __hbase_configured = False + __hbase_acl_configured = False def __init__(self, params): if params is None: @@ -42,6 +44,8 @@ class IndexingCommands: self.__indexing_topic = params.indexing_input_topic self.__configured = os.path.isfile(self.__params.indexing_configured_flag_file) self.__acl_configured = os.path.isfile(self.__params.indexing_acl_configured_flag_file) + self.__hbase_configured = os.path.isfile(self.__params.indexing_hbase_configured_flag_file) + self.__hbase_acl_configured = os.path.isfile(self.__params.indexing_hbase_acl_configured_flag_file) def is_configured(self): return self.__configured @@ -58,6 +62,66 @@ class IndexingCommands: owner=self.__params.metron_user, mode=0755) + def is_hbase_configured(self): + return self.__hbase_configured + + def is_hbase_acl_configured(self): + return self.__hbase_acl_configured + + def set_hbase_configured(self): + Logger.info("Setting HBase Configured to True") + File(self.__params.indexing_hbase_configured_flag_file, + content="", + owner=self.__params.metron_user, + mode=0755) + + def set_hbase_acl_configured(self): + Logger.info("Setting HBase ACL Configured to True") + File(self.__params.indexing_hbase_acl_configured_flag_file, + content="", + owner=self.__params.metron_user, + mode=0755) + + def create_hbase_tables(self): + Logger.info("Creating HBase Tables") + if self.__params.security_enabled: + kinit(self.__params.kinit_path_local, + self.__params.hbase_keytab_path, + self.__params.hbase_principal_name, + execute_user=self.__params.hbase_user) + cmd = "echo \"create '{0}','{1}'\" | hbase shell -n" + add_update_cmd = cmd.format(self.__params.update_table, self.__params.update_cf) + Execute(add_update_cmd, + tries=3, + try_sleep=5, + logoutput=False, + path='/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin', + user=self.__params.hbase_user + ) + + Logger.info("Done creating HBase Tables") + self.set_hbase_configured() + + def set_hbase_acls(self): + Logger.info("Setting HBase ACLs") + if self.__params.security_enabled: + kinit(self.__params.kinit_path_local, + self.__params.hbase_keytab_path, + self.__params.hbase_principal_name, + execute_user=self.__params.hbase_user) + cmd = "echo \"grant '{0}', 'RW', '{1}'\" | hbase shell -n" + add_update_acl_cmd = cmd.format(self.__params.metron_user, self.__params.update_table) + Execute(add_update_acl_cmd, + tries=3, + try_sleep=5, + logoutput=False, + path='/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin', + user=self.__params.hbase_user + ) + + Logger.info("Done setting HBase ACLs") + self.set_hbase_acl_configured() + def set_acl_configured(self): File(self.__params.indexing_acl_configured_flag_file, content="", http://git-wip-us.apache.org/repos/asf/metron/blob/813adf2d/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py index 7e111cf..71dcc74 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py @@ -65,6 +65,11 @@ class Indexing(Script): commands.init_kafka_acls() commands.set_acl_configured() + if not commands.is_hbase_configured(): + commands.create_hbase_tables() + if params.security_enabled and not commands.is_hbase_acl_configured(): + commands.set_hbase_acls() + Logger.info("Calling security setup") storm_security_setup(params) http://git-wip-us.apache.org/repos/asf/metron/blob/813adf2d/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py index 3e75cb6..22ccabc 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py @@ -71,6 +71,8 @@ enrichment_hbase_acl_configured_flag_file = status_params.enrichment_hbase_acl_c enrichment_geo_configured_flag_file = status_params.enrichment_geo_configured_flag_file indexing_configured_flag_file = status_params.indexing_configured_flag_file indexing_acl_configured_flag_file = status_params.indexing_acl_configured_flag_file +indexing_hbase_configured_flag_file = status_params.indexing_hbase_configured_flag_file +indexing_hbase_acl_configured_flag_file = status_params.indexing_hbase_acl_configured_flag_file indexing_hdfs_perm_configured_flag_file = status_params.indexing_hdfs_perm_configured_flag_file global_json_template = config['configurations']['metron-env']['global-json'] global_properties_template = config['configurations']['metron-env']['elasticsearch-properties'] @@ -167,6 +169,9 @@ HdfsResource = functools.partial( enrichment_hbase_provider_impl = 'org.apache.metron.hbase.HTableProvider' enrichment_table = status_params.enrichment_table enrichment_cf = status_params.enrichment_cf +update_table = status_params.update_table +update_cf = status_params.update_cf + threatintel_table = status_params.threatintel_table threatintel_cf = status_params.threatintel_cf http://git-wip-us.apache.org/repos/asf/metron/blob/813adf2d/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py index c5e36b0..dd2ba9b 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py @@ -49,6 +49,8 @@ enrichment_table = 'enrichment' enrichment_cf = 't' threatintel_table = 'threatintel' threatintel_cf = 't' +update_table = 'metron_update' +update_cf = 't' # Indexing metron_indexing_topology = 'indexing' @@ -56,6 +58,8 @@ indexing_input_topic = config['configurations']['metron-indexing-env']['indexing indexing_configured_flag_file = metron_zookeeper_config_path + '/../metron_indexing_configured' indexing_acl_configured_flag_file = metron_zookeeper_config_path + '/../metron_indexing_acl_configured' indexing_hdfs_perm_configured_flag_file = metron_zookeeper_config_path + '/../metron_indexing_hdfs_perm_configured' +indexing_hbase_configured_flag_file = metron_zookeeper_config_path + '/../metron_indexing_hbase_configured' +indexing_hbase_acl_configured_flag_file = metron_zookeeper_config_path + '/../metron_indexing_hbase_acl_configured' # REST metron_rest_port = config['configurations']['metron-rest-env']['metron_rest_port'] http://git-wip-us.apache.org/repos/asf/metron/blob/813adf2d/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/global.json.j2 ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/global.json.j2 b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/global.json.j2 index 61e1416..67226ff 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/global.json.j2 +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/global.json.j2 @@ -2,5 +2,7 @@ "es.clustername": "{{ es_cluster_name }}", "es.ip": "{{ es_url }}", "es.date.format": "{{es_date_format}}", - "parser.error.topic": "{{parser_error_topic}}" + "parser.error.topic": "{{parser_error_topic}}", + "update.hbase.table": "{{update_table}}", + "update.hbase.cf": "{{update_cf}}" } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/metron/blob/813adf2d/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/themes/metron_theme.json ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/themes/metron_theme.json b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/themes/metron_theme.json index e7ac630..0e5457d 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/themes/metron_theme.json +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/themes/metron_theme.json @@ -154,6 +154,25 @@ ] }, { + "name": "section-indexing-update", + "row-index": "0", + "column-index": "0", + "row-span": "1", + "column-span": "1", + "section-columns": "1", + "section-rows": "1", + "subsections": [ + { + "name": "subsection-indexing-update", + "display-name": "Index Updates", + "row-index": "0", + "column-index": "0", + "row-span": "1", + "column-span": "1" + } + ] + }, + { "name": "section-indexing-storm", "row-index": "1", "column-index": "0", @@ -378,8 +397,12 @@ "subsection-name": "subsection-indexing-kafka" }, { - "config": "metron-indexing-env/indexing_error_topic", - "subsection-name": "subsection-indexing-kafka" + "config": "metron-indexing-env/update_table", + "subsection-name": "subsection-indexing-update" + }, + { + "config": "metron-indexing-env/update_cf", + "subsection-name": "subsection-indexing-update" }, { "config": "metron-indexing-env/indexing_writer_class_name", http://git-wip-us.apache.org/repos/asf/metron/blob/813adf2d/metron-interface/metron-rest/README.md ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest/README.md b/metron-interface/metron-rest/README.md index 46a3fc0..b76712b 100644 --- a/metron-interface/metron-rest/README.md +++ b/metron-interface/metron-rest/README.md @@ -200,6 +200,7 @@ Request and Response objects are JSON formatted. The JSON schemas are available | [ `DELETE /api/v1/kafka/topic/{name}`](#delete-apiv1kafkatopicname)| | [ `GET /api/v1/kafka/topic/{name}/sample`](#get-apiv1kafkatopicnamesample)| | [ `GET /api/v1/search/search`](#get-apiv1searchsearch)| +| [ `GET /api/v1/search/findOne`](#get-apiv1searchfindone)| | [ `GET /api/v1/search/search`](#get-apiv1searchcolumnmetadata)| | [ `GET /api/v1/search/search`](#get-apiv1searchcolumnmetadatacommon)| | [ `GET /api/v1/sensor/enrichment/config`](#get-apiv1sensorenrichmentconfig)| @@ -242,6 +243,8 @@ Request and Response objects are JSON formatted. The JSON schemas are available | [ `GET /api/v1/storm/parser/stop/{name}`](#get-apiv1stormparserstopname)| | [ `GET /api/v1/storm/{name}`](#get-apiv1stormname)| | [ `GET /api/v1/storm/supervisors`](#get-apiv1stormsupervisors)| +| [ `PATCH /api/v1/update/patch`](#patch-apiv1updatepatch)| +| [ `PUT /api/v1/update/replace`](#patch-apiv1updatereplace)| | [ `GET /api/v1/user`](#get-apiv1user)| ### `GET /api/v1/global/config` @@ -350,6 +353,23 @@ Request and Response objects are JSON formatted. The JSON schemas are available * 200 - Returns sample message * 404 - Either Kafka topic is missing or contains no messages +### `GET /api/v1/search/findOne` + * Description: Returns latest document for a guid and sensor + * Input: + * getRequest - Get request + * guid - message UUID + * sensorType - Sensor Type + * Example: Return `bro` document with UUID of `000-000-0000` +``` +{ + "guid" : "000-000-0000", + "sensorType" : "bro" +} +``` + * Returns: + * 200 - Document representing the output + * 404 - Document with UUID and sensor type not found + ### `GET /api/v1/search/search` * Description: Searches the indexing store * Input: @@ -624,6 +644,56 @@ Request and Response objects are JSON formatted. The JSON schemas are available * Returns: * 200 - Returns a list of the status of all Storm Supervisors +### `PATCH /api/v1/update/patch` + * Description: Update a document with a patch + * Input: + * request - Patch Request + * guid - The Patch UUID + * sensorType - The sensor type + * patch - An array of [RFC 6902](https://tools.ietf.org/html/rfc6902) patches. + * Example adding a field called `project` with value `metron` to the `bro` message with UUID of `000-000-0000` : + ``` + { + "guid" : "000-000-0000", + "sensorType" : "bro", + "patch" : [ + { + "op": "add" + , "path": "/project" + , "value": "metron" + } + ] + } + ``` + * Returns: + * 200 - nothing + * 404 - document not found + +### `PUT /api/v1/update/replace` + * Description: Replace a document + * Input: + * request - Replacement request + * guid - The Patch UUID + * sensorType - The sensor type + * replacement - A Map representing the replaced document + * Example replacing a `bro` message with guid of `000-000-0000` +``` + { + "guid" : "000-000-0000", + "sensorType" : "bro", + "replacement" : { + "source:type": "bro", + "guid" : "bro_index_2017.01.01.01:1", + "ip_src_addr":"192.168.1.2", + "ip_src_port": 8009, + "timestamp":200, + "rejected":false + } + } +``` + * Returns: + * 200 - Current user + ### `GET /api/v1/user` * Description: Retrieves the current user * Returns: http://git-wip-us.apache.org/repos/asf/metron/blob/813adf2d/metron-interface/metron-rest/pom.xml ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest/pom.xml b/metron-interface/metron-rest/pom.xml index cb8752c..71caecc 100644 --- a/metron-interface/metron-rest/pom.xml +++ b/metron-interface/metron-rest/pom.xml @@ -147,10 +147,26 @@ </dependency> <dependency> <groupId>org.apache.metron</groupId> + <artifactId>metron-hbase-client</artifactId> + <version>${project.parent.version}</version> + </dependency> + <dependency> + <groupId>org.apache.metron</groupId> + <artifactId>metron-hbase</artifactId> + <version>${project.parent.version}</version> + <scope>test</scope> + <type>test-jar</type> + </dependency> + <dependency> + <groupId>org.apache.metron</groupId> <artifactId>metron-parsers</artifactId> <version>${project.parent.version}</version> <exclusions> <exclusion> + <groupId>org.apache.hbase</groupId> + <artifactId>hbase-client</artifactId> + </exclusion> + <exclusion> <groupId>com.fasterxml.jackson.core</groupId> <artifactId>jackson-databind</artifactId> </exclusion> http://git-wip-us.apache.org/repos/asf/metron/blob/813adf2d/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/MetronRestConstants.java ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/MetronRestConstants.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/MetronRestConstants.java index 9e32c7a..a080f77 100644 --- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/MetronRestConstants.java +++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/MetronRestConstants.java @@ -62,4 +62,5 @@ public class MetronRestConstants { public static final String SEARCH_MAX_RESULTS = "search.max.results"; public static final String INDEX_DAO_IMPL = "index.dao.impl"; + public static final String INDEX_HBASE_TABLE_PROVIDER_IMPL = "index.hbase.provider"; } http://git-wip-us.apache.org/repos/asf/metron/blob/813adf2d/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/IndexConfig.java ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/IndexConfig.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/IndexConfig.java index 2bfafea..6385116 100644 --- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/IndexConfig.java +++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/IndexConfig.java @@ -17,6 +17,8 @@ */ package org.apache.metron.rest.config; +import org.apache.metron.hbase.HTableProvider; +import org.apache.metron.hbase.TableProvider; import org.apache.metron.indexing.dao.AccessConfig; import org.apache.metron.indexing.dao.IndexDao; import org.apache.metron.indexing.dao.IndexDaoFactory; @@ -50,17 +52,35 @@ public class IndexConfig { @Bean public IndexDao indexDao() { - String indexDaoImpl = environment.getProperty(MetronRestConstants.INDEX_DAO_IMPL, String.class, null); - int searchMaxResults = environment.getProperty(MetronRestConstants.SEARCH_MAX_RESULTS, Integer.class, -1); - AccessConfig config = new AccessConfig(); - config.setMaxSearchResults(searchMaxResults); - if(indexDaoImpl == null) { - throw new IllegalStateException("You must provide an index DAO implementation via the " + INDEX_DAO_IMPL + " config"); - } try { - return IndexDaoFactory.create(indexDaoImpl, globalConfigService.get(), config); - } catch (Exception e) { - throw new IllegalStateException("Unable to instantiate " + indexDaoImpl + ": " + e.getMessage(), e); + String hbaseProviderImpl = environment.getProperty(MetronRestConstants.INDEX_HBASE_TABLE_PROVIDER_IMPL, String.class, null); + String indexDaoImpl = environment.getProperty(MetronRestConstants.INDEX_DAO_IMPL, String.class, null); + int searchMaxResults = environment.getProperty(MetronRestConstants.SEARCH_MAX_RESULTS, Integer.class, -1); + AccessConfig config = new AccessConfig(); + config.setMaxSearchResults(searchMaxResults); + config.setGlobalConfigSupplier(() -> { + try { + return globalConfigService.get(); + } catch (RestException e) { + throw new IllegalStateException("Unable to retrieve the global config.", e); + } + }); + config.setTableProvider(TableProvider.create(hbaseProviderImpl, () -> new HTableProvider())); + if (indexDaoImpl == null) { + throw new IllegalStateException("You must provide an index DAO implementation via the " + INDEX_DAO_IMPL + " config"); + } + IndexDao ret = IndexDaoFactory.combine(IndexDaoFactory.create(indexDaoImpl, config)); + if (ret == null) { + throw new IllegalStateException("IndexDao is unable to be created."); + } + return ret; + } + catch(RuntimeException re) { + throw re; + } + catch(Exception e) { + throw new IllegalStateException("Unable to create index DAO: " + e.getMessage(), e); } } + } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/metron/blob/813adf2d/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/SearchController.java ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/SearchController.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/SearchController.java index d3b805f..dea628c 100644 --- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/SearchController.java +++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/SearchController.java @@ -20,6 +20,8 @@ package org.apache.metron.rest.controller; import io.swagger.annotations.ApiOperation; import io.swagger.annotations.ApiParam; import io.swagger.annotations.ApiResponse; +import org.apache.metron.indexing.dao.search.GetRequest; +import org.apache.metron.indexing.dao.update.Document; import org.apache.metron.indexing.dao.search.FieldType; import org.apache.metron.rest.RestException; import org.apache.metron.indexing.dao.search.SearchRequest; @@ -33,6 +35,8 @@ import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestMethod; import org.springframework.web.bind.annotation.RestController; +import java.util.Map; +import java.util.Optional; import java.util.List; import java.util.Map; @@ -50,6 +54,24 @@ public class SearchController { return new ResponseEntity<>(searchService.search(searchRequest), HttpStatus.OK); } + @ApiOperation(value = "Returns latest document for a guid and sensor") + @ApiResponse(message = "Document representing the output", code = 200) + @RequestMapping(value = "/findOne", method = RequestMethod.POST) + ResponseEntity<Map<String, Object>> getLatest( + final @ApiParam(name = "getRequest", value = "Get Request", required = true) + @RequestBody + GetRequest request + ) throws RestException + { + Optional<Map<String, Object>> latest = searchService.getLatest(request); + if(latest.isPresent()) { + return new ResponseEntity<>(latest.get(), HttpStatus.OK); + } + else { + return new ResponseEntity<>(HttpStatus.NOT_FOUND); + } + } + @ApiOperation(value = "Get column metadata for each index in the list of indices") @ApiResponse(message = "Column Metadata", code = 200) @RequestMapping(value = "/column/metadata", method = RequestMethod.POST) http://git-wip-us.apache.org/repos/asf/metron/blob/813adf2d/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/UpdateController.java ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/UpdateController.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/UpdateController.java new file mode 100644 index 0000000..56b0b7b --- /dev/null +++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/UpdateController.java @@ -0,0 +1,70 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.rest.controller; + +import io.swagger.annotations.ApiOperation; +import io.swagger.annotations.ApiParam; +import io.swagger.annotations.ApiResponse; +import org.apache.metron.indexing.dao.update.OriginalNotFoundException; +import org.apache.metron.indexing.dao.update.PatchRequest; +import org.apache.metron.indexing.dao.update.ReplaceRequest; +import org.apache.metron.rest.RestException; +import org.apache.metron.rest.service.UpdateService; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RestController; + +@RestController +@RequestMapping("/api/v1/update") +public class UpdateController { + + @Autowired + private UpdateService service; + + @ApiOperation(value = "Update a document with a patch") + @ApiResponse(message = "Nothing", code = 200) + @RequestMapping(value = "/patch", method = RequestMethod.PATCH) + ResponseEntity<Void> patch( + final @ApiParam(name = "request", value = "Patch request", required = true) + @RequestBody + PatchRequest request + ) throws RestException { + try { + service.patch(request); + } catch (OriginalNotFoundException e) { + return new ResponseEntity<>(HttpStatus.NOT_FOUND); + } + return new ResponseEntity<>(HttpStatus.OK); + } + + @ApiOperation(value = "Replace a document with a full replacement") + @ApiResponse(message = "Nothing", code = 200) + @RequestMapping(value = "/replace", method = RequestMethod.POST) + ResponseEntity<Void> replace( + final @ApiParam(name = "request", value = "Replacement request", required = true) + @RequestBody + ReplaceRequest request + ) throws RestException { + service.replace(request); + return new ResponseEntity<>(HttpStatus.OK); + } +} http://git-wip-us.apache.org/repos/asf/metron/blob/813adf2d/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/SearchService.java ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/SearchService.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/SearchService.java index b2fb2e6..ea0ae81 100644 --- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/SearchService.java +++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/SearchService.java @@ -17,17 +17,22 @@ */ package org.apache.metron.rest.service; +import org.apache.metron.indexing.dao.search.GetRequest; +import org.apache.metron.indexing.dao.update.Document; import org.apache.metron.indexing.dao.search.FieldType; import org.apache.metron.rest.RestException; import org.apache.metron.indexing.dao.search.SearchRequest; import org.apache.metron.indexing.dao.search.SearchResponse; +import java.util.Map; +import java.util.Optional; import java.util.List; import java.util.Map; public interface SearchService { SearchResponse search(SearchRequest searchRequest) throws RestException; + Optional<Map<String, Object>> getLatest(GetRequest request) throws RestException; Map<String, Map<String, FieldType>> getColumnMetadata(List<String> indices) throws RestException; Map<String, FieldType> getCommonColumnMetadata(List<String> indices) throws RestException; http://git-wip-us.apache.org/repos/asf/metron/blob/813adf2d/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/UpdateService.java ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/UpdateService.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/UpdateService.java new file mode 100644 index 0000000..4cdf4b3 --- /dev/null +++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/UpdateService.java @@ -0,0 +1,29 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.rest.service; + +import org.apache.metron.indexing.dao.update.OriginalNotFoundException; +import org.apache.metron.indexing.dao.update.PatchRequest; +import org.apache.metron.indexing.dao.update.ReplaceRequest; +import org.apache.metron.rest.RestException; + +public interface UpdateService { + + void patch(PatchRequest request) throws RestException, OriginalNotFoundException; + void replace(ReplaceRequest request) throws RestException; +} http://git-wip-us.apache.org/repos/asf/metron/blob/813adf2d/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/IndexDaoSearchServiceImpl.java ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/IndexDaoSearchServiceImpl.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/IndexDaoSearchServiceImpl.java deleted file mode 100644 index b93a5fc..0000000 --- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/IndexDaoSearchServiceImpl.java +++ /dev/null @@ -1,75 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.metron.rest.service.impl; - -import org.apache.metron.indexing.dao.IndexDao; -import org.apache.metron.indexing.dao.search.InvalidSearchException; -import org.apache.metron.indexing.dao.search.SearchRequest; -import org.apache.metron.indexing.dao.search.SearchResponse; -import org.apache.metron.indexing.dao.search.FieldType; -import org.apache.metron.rest.RestException; -import org.apache.metron.rest.service.SearchService; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.core.env.Environment; -import org.springframework.stereotype.Service; - -import java.io.IOException; -import java.util.List; -import java.util.Map; - -@Service -public class IndexDaoSearchServiceImpl implements SearchService { - private IndexDao dao; - private Environment environment; - - @Autowired - public IndexDaoSearchServiceImpl(IndexDao dao, Environment environment) { - this.dao = dao; - this.environment = environment; - } - - @Override - public SearchResponse search(SearchRequest searchRequest) throws RestException { - try { - return dao.search(searchRequest); - } - catch(InvalidSearchException ise) { - throw new RestException(ise.getMessage(), ise); - } - } - - @Override - public Map<String, Map<String, FieldType>> getColumnMetadata(List<String> indices) throws RestException { - try { - return dao.getColumnMetadata(indices); - } - catch(IOException ioe) { - throw new RestException(ioe.getMessage(), ioe); - } - } - - @Override - public Map<String, FieldType> getCommonColumnMetadata(List<String> indices) throws RestException { - try { - return dao.getCommonColumnMetadata(indices); - } - catch(IOException ioe) { - throw new RestException(ioe.getMessage(), ioe); - } - } -} http://git-wip-us.apache.org/repos/asf/metron/blob/813adf2d/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SearchServiceImpl.java ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SearchServiceImpl.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SearchServiceImpl.java new file mode 100644 index 0000000..bdf6037 --- /dev/null +++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SearchServiceImpl.java @@ -0,0 +1,87 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.rest.service.impl; + +import org.apache.metron.indexing.dao.IndexDao; +import org.apache.metron.indexing.dao.search.GetRequest; +import org.apache.metron.indexing.dao.search.InvalidSearchException; +import org.apache.metron.indexing.dao.search.SearchRequest; +import org.apache.metron.indexing.dao.search.SearchResponse; +import org.apache.metron.indexing.dao.update.Document; +import org.apache.metron.indexing.dao.search.FieldType; +import org.apache.metron.rest.RestException; +import org.apache.metron.rest.service.SearchService; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.core.env.Environment; +import org.springframework.stereotype.Service; + +import java.io.IOException; +import java.util.Map; +import java.util.Optional; +import java.util.List; +import java.util.Map; + +@Service +public class SearchServiceImpl implements SearchService { + private IndexDao dao; + private Environment environment; + + @Autowired + public SearchServiceImpl(IndexDao dao, Environment environment) { + this.dao = dao; + this.environment = environment; + } + + @Override + public SearchResponse search(SearchRequest searchRequest) throws RestException { + try { + return dao.search(searchRequest); + } + catch(InvalidSearchException ise) { + throw new RestException(ise.getMessage(), ise); + } + } + + @Override + public Optional<Map<String, Object>> getLatest(GetRequest request) throws RestException { + try { + return dao.getLatestResult(request); + } catch (IOException e) { + throw new RestException(e.getMessage(), e); + } + } + + public Map<String, Map<String, FieldType>> getColumnMetadata(List<String> indices) throws RestException { + try { + return dao.getColumnMetadata(indices); + } + catch(IOException ioe) { + throw new RestException(ioe.getMessage(), ioe); + } + } + + @Override + public Map<String, FieldType> getCommonColumnMetadata(List<String> indices) throws RestException { + try { + return dao.getCommonColumnMetadata(indices); + } + catch(IOException ioe) { + throw new RestException(ioe.getMessage(), ioe); + } + } +} http://git-wip-us.apache.org/repos/asf/metron/blob/813adf2d/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/UpdateServiceImpl.java ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/UpdateServiceImpl.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/UpdateServiceImpl.java new file mode 100644 index 0000000..847173e --- /dev/null +++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/UpdateServiceImpl.java @@ -0,0 +1,58 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.rest.service.impl; + +import org.apache.metron.indexing.dao.IndexDao; +import org.apache.metron.indexing.dao.update.OriginalNotFoundException; +import org.apache.metron.indexing.dao.update.PatchRequest; +import org.apache.metron.indexing.dao.update.ReplaceRequest; +import org.apache.metron.rest.RestException; +import org.apache.metron.rest.service.UpdateService; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import java.util.Optional; + +@Service +public class UpdateServiceImpl implements UpdateService { + private IndexDao dao; + + @Autowired + public UpdateServiceImpl(IndexDao dao) { + this.dao = dao; + } + + + @Override + public void patch(PatchRequest request) throws RestException, OriginalNotFoundException { + try { + dao.patch(request, Optional.of(System.currentTimeMillis())); + } catch (Exception e) { + throw new RestException(e.getMessage(), e); + } + } + + @Override + public void replace(ReplaceRequest request) throws RestException { + try { + dao.replace(request, Optional.of(System.currentTimeMillis())); + } catch (Exception e) { + throw new RestException(e.getMessage(), e); + } + } +} http://git-wip-us.apache.org/repos/asf/metron/blob/813adf2d/metron-interface/metron-rest/src/main/resources/application-test.yml ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest/src/main/resources/application-test.yml b/metron-interface/metron-rest/src/main/resources/application-test.yml index 9dfcdf9..9793840 100644 --- a/metron-interface/metron-rest/src/main/resources/application-test.yml +++ b/metron-interface/metron-rest/src/main/resources/application-test.yml @@ -44,6 +44,11 @@ storm: search: max: results: 100 + index: dao: - impl: org.apache.metron.indexing.dao.InMemoryDao \ No newline at end of file + # By default, we use the InMemoryDao for our tests and HBaseDao for backing updates. + impl: org.apache.metron.indexing.dao.InMemoryDao,org.apache.metron.indexing.dao.HBaseDao + hbase: + # HBase is provided via a mock provider, so no actual HBase infrastructure is started. + provider: org.apache.metron.hbase.mock.MockHBaseTableProvider http://git-wip-us.apache.org/repos/asf/metron/blob/813adf2d/metron-interface/metron-rest/src/main/resources/application.yml ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest/src/main/resources/application.yml b/metron-interface/metron-rest/src/main/resources/application.yml index 473d29d..4aff769 100644 --- a/metron-interface/metron-rest/src/main/resources/application.yml +++ b/metron-interface/metron-rest/src/main/resources/application.yml @@ -47,4 +47,5 @@ search: index: dao: - impl: org.apache.metron.elasticsearch.dao.ElasticsearchDao \ No newline at end of file + # By default, we use the ElasticsearchDao and HBaseDao for backing updates. + impl: org.apache.metron.elasticsearch.dao.ElasticsearchDao,org.apache.metron.indexing.dao.HBaseDao \ No newline at end of file http://git-wip-us.apache.org/repos/asf/metron/blob/813adf2d/metron-interface/metron-rest/src/main/scripts/metron-rest ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest/src/main/scripts/metron-rest b/metron-interface/metron-rest/src/main/scripts/metron-rest index a84f496..08cdfd4 100644 --- a/metron-interface/metron-rest/src/main/scripts/metron-rest +++ b/metron-interface/metron-rest/src/main/scripts/metron-rest @@ -44,9 +44,10 @@ if [ -f "$METRON_SYSCONFIG" ]; then . "$METRON_SYSCONFIG" fi +HBASE_HOME=${HBASE_HOME:-/usr/hdp/current/hbase-client} PIDFILE="$METRON_PID_DIR/$NAME.pid" -METRON_REST_CLASSPATH="$METRON_HOME/lib/metron-rest-$METRON_VERSION.jar" +METRON_REST_CLASSPATH="${HBASE_HOME}/conf:$METRON_HOME/lib/metron-rest-$METRON_VERSION.jar" # the vagrant Spring profile provides configuration values, otherwise configuration is provided by rest_application.yml if [[ !($METRON_SPRING_PROFILES_ACTIVE == *"vagrant"*) ]]; then http://git-wip-us.apache.org/repos/asf/metron/blob/813adf2d/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/config/TestConfig.java ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/config/TestConfig.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/config/TestConfig.java index 8d0fe42..9c75f2f 100644 --- a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/config/TestConfig.java +++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/config/TestConfig.java @@ -21,10 +21,14 @@ import kafka.admin.AdminUtils$; import kafka.utils.ZKStringSerializer$; import kafka.utils.ZkUtils; import org.I0Itec.zkclient.ZkClient; +import org.apache.commons.io.IOUtils; +import org.apache.curator.CuratorZookeeperClient; import org.apache.curator.RetryPolicy; import org.apache.curator.framework.CuratorFramework; import org.apache.curator.framework.CuratorFrameworkFactory; import org.apache.curator.retry.ExponentialBackoffRetry; +import org.apache.metron.TestConstants; +import org.apache.metron.common.configuration.ConfigurationsUtils; import org.apache.metron.integration.ComponentRunner; import org.apache.metron.integration.UnableToStartException; import org.apache.metron.integration.components.KafkaComponent; @@ -32,6 +36,7 @@ import org.apache.metron.integration.components.ZKServerComponent; import org.apache.metron.rest.mock.MockStormCLIClientWrapper; import org.apache.metron.rest.mock.MockStormRestTemplate; import org.apache.metron.rest.service.impl.StormCLIWrapper; +import org.apache.zookeeper.KeeperException; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Profile; @@ -39,10 +44,16 @@ import org.springframework.kafka.core.ConsumerFactory; import org.springframework.kafka.core.DefaultKafkaConsumerFactory; import org.springframework.web.client.RestTemplate; +import java.io.BufferedReader; +import java.io.File; +import java.io.FileReader; +import java.io.IOException; import java.util.HashMap; import java.util.Map; import java.util.Properties; +import java.util.concurrent.atomic.AtomicBoolean; +import static org.apache.metron.common.configuration.ConfigurationsUtils.getClient; import static org.apache.metron.rest.MetronRestConstants.TEST_PROFILE; @Configuration @@ -65,7 +76,6 @@ public class TestConfig { return new KafkaComponent().withTopologyProperties(zkProperties); } - @Bean(destroyMethod = "stop") public ComponentRunner componentRunner(ZKServerComponent zkServerComponent, KafkaComponent kafkaWithZKComponent) { ComponentRunner runner = new ComponentRunner.Builder() @@ -74,6 +84,13 @@ public class TestConfig { .build(); try { runner.start(); + File globalConfigFile = new File("src/test/resources/zookeeper/global.json"); + try(BufferedReader r = new BufferedReader(new FileReader(globalConfigFile))){ + String globalConfig = IOUtils.toString(r); + ConfigurationsUtils.writeGlobalConfigToZookeeper(globalConfig.getBytes(), zkServerComponent.getConnectionString()); + } catch (Exception e) { + throw new IllegalStateException("Unable to upload global config", e); + } } catch (UnableToStartException e) { e.printStackTrace(); } http://git-wip-us.apache.org/repos/asf/metron/blob/813adf2d/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/DaoControllerTest.java ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/DaoControllerTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/DaoControllerTest.java new file mode 100644 index 0000000..096f1be --- /dev/null +++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/DaoControllerTest.java @@ -0,0 +1,58 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.rest.controller; + +import com.google.common.collect.ImmutableMap; +import org.apache.metron.common.Constants; +import org.apache.metron.indexing.dao.InMemoryDao; +import org.apache.metron.indexing.dao.SearchIntegrationTest; +import org.json.simple.JSONArray; +import org.json.simple.JSONObject; +import org.json.simple.parser.JSONParser; +import org.json.simple.parser.ParseException; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class DaoControllerTest { + public static final String TABLE = "updates"; + public static final String CF = "t"; + public void loadTestData() throws ParseException { + Map<String, List<String>> backingStore = new HashMap<>(); + for(Map.Entry<String, String> indices : + ImmutableMap.of( + "bro_index_2017.01.01.01", SearchIntegrationTest.broData, + "snort_index_2017.01.01.01", SearchIntegrationTest.snortData + ).entrySet() + ) + { + List<String> results = new ArrayList<>(); + backingStore.put(indices.getKey(), results); + JSONArray broArray = (JSONArray) new JSONParser().parse(indices.getValue()); + int i = 0; + for(Object o: broArray) { + JSONObject jsonObject = (JSONObject) o; + jsonObject.put(Constants.GUID, indices.getKey() + ":" + i++); + results.add(jsonObject.toJSONString()); + } + } + InMemoryDao.load(backingStore); + } +} http://git-wip-us.apache.org/repos/asf/metron/blob/813adf2d/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SearchControllerIntegrationTest.java ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SearchControllerIntegrationTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SearchControllerIntegrationTest.java index 44d9078..1c5310b 100644 --- a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SearchControllerIntegrationTest.java +++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SearchControllerIntegrationTest.java @@ -17,18 +17,15 @@ */ package org.apache.metron.rest.controller; -import com.google.common.collect.ImmutableMap; -import org.adrianwalker.multilinestring.Multiline; +import org.apache.metron.hbase.mock.MockHBaseTableProvider; import org.apache.metron.indexing.dao.InMemoryDao; -import org.apache.metron.indexing.dao.IndexingDaoIntegrationTest; +import org.apache.metron.indexing.dao.SearchIntegrationTest; import org.apache.metron.indexing.dao.search.FieldType; import org.apache.metron.rest.service.SearchService; -import org.json.simple.JSONArray; -import org.json.simple.JSONObject; -import org.json.simple.parser.JSONParser; import org.json.simple.parser.ParseException; import org.junit.After; import org.junit.Before; +import org.junit.BeforeClass; import org.junit.Test; import org.junit.runner.RunWith; import org.springframework.beans.factory.annotation.Autowired; @@ -40,9 +37,7 @@ import org.springframework.test.web.servlet.MockMvc; import org.springframework.test.web.servlet.setup.MockMvcBuilders; import org.springframework.web.context.WebApplicationContext; -import java.util.ArrayList; import java.util.HashMap; -import java.util.List; import java.util.Map; import static org.apache.metron.rest.MetronRestConstants.TEST_PROFILE; @@ -59,7 +54,7 @@ import static org.springframework.test.web.servlet.result.MockMvcResultMatchers. @RunWith(SpringRunner.class) @SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) @ActiveProfiles(TEST_PROFILE) -public class SearchControllerIntegrationTest { +public class SearchControllerIntegrationTest extends DaoControllerTest { @@ -75,6 +70,11 @@ public class SearchControllerIntegrationTest { private String user = "user"; private String password = "password"; + @BeforeClass + public static void setupHbase() { + MockHBaseTableProvider.addToCache("updates", "t"); + } + @Before public void setup() throws Exception { this.mockMvc = MockMvcBuilders.webAppContextSetup(this.wac).apply(springSecurity()).build(); @@ -89,14 +89,14 @@ public class SearchControllerIntegrationTest { @Test public void testSecurity() throws Exception { - this.mockMvc.perform(post(searchUrl + "/search").with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(IndexingDaoIntegrationTest.allQuery)) + this.mockMvc.perform(post(searchUrl + "/search").with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(SearchIntegrationTest.allQuery)) .andExpect(status().isUnauthorized()); } @Test public void test() throws Exception { - this.mockMvc.perform(post(searchUrl + "/search").with(httpBasic(user, password)).with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(IndexingDaoIntegrationTest.allQuery)) + this.mockMvc.perform(post(searchUrl + "/search").with(httpBasic(user, password)).with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(SearchIntegrationTest.allQuery)) .andExpect(status().isOk()) .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8"))) .andExpect(jsonPath("$.total").value(10)) @@ -121,7 +121,7 @@ public class SearchControllerIntegrationTest { .andExpect(jsonPath("$.results[9].source.source:type").value("bro")) .andExpect(jsonPath("$.results[9].source.timestamp").value(1)); - this.mockMvc.perform(post(searchUrl + "/search").with(httpBasic(user, password)).with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(IndexingDaoIntegrationTest.filterQuery)) + this.mockMvc.perform(post(searchUrl + "/search").with(httpBasic(user, password)).with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(SearchIntegrationTest.filterQuery)) .andExpect(status().isOk()) .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8"))) .andExpect(jsonPath("$.total").value(3)) @@ -133,7 +133,7 @@ public class SearchControllerIntegrationTest { .andExpect(jsonPath("$.results[2].source.timestamp").value(1)); - this.mockMvc.perform(post(searchUrl + "/search").with(httpBasic(user, password)).with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(IndexingDaoIntegrationTest.sortQuery)) + this.mockMvc.perform(post(searchUrl + "/search").with(httpBasic(user, password)).with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(SearchIntegrationTest.sortQuery)) .andExpect(status().isOk()) .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8"))) .andExpect(jsonPath("$.total").value(10)) @@ -148,7 +148,7 @@ public class SearchControllerIntegrationTest { .andExpect(jsonPath("$.results[8].source.ip_src_port").value(8009)) .andExpect(jsonPath("$.results[9].source.ip_src_port").value(8010)); - this.mockMvc.perform(post(searchUrl + "/search").with(httpBasic(user, password)).with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(IndexingDaoIntegrationTest.paginationQuery)) + this.mockMvc.perform(post(searchUrl + "/search").with(httpBasic(user, password)).with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(SearchIntegrationTest.paginationQuery)) .andExpect(status().isOk()) .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8"))) .andExpect(jsonPath("$.total").value(10)) @@ -159,7 +159,7 @@ public class SearchControllerIntegrationTest { .andExpect(jsonPath("$.results[2].source.source:type").value("bro")) .andExpect(jsonPath("$.results[2].source.timestamp").value(4)); - this.mockMvc.perform(post(searchUrl + "/search").with(httpBasic(user, password)).with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(IndexingDaoIntegrationTest.indexQuery)) + this.mockMvc.perform(post(searchUrl + "/search").with(httpBasic(user, password)).with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(SearchIntegrationTest.indexQuery)) .andExpect(status().isOk()) .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8"))) .andExpect(jsonPath("$.total").value(5)) @@ -174,7 +174,7 @@ public class SearchControllerIntegrationTest { .andExpect(jsonPath("$.results[4].source.source:type").value("bro")) .andExpect(jsonPath("$.results[4].source.timestamp").value(1)); - this.mockMvc.perform(post(searchUrl + "/search").with(httpBasic(user, password)).with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(IndexingDaoIntegrationTest.exceededMaxResultsQuery)) + this.mockMvc.perform(post(searchUrl + "/search").with(httpBasic(user, password)).with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(SearchIntegrationTest.exceededMaxResultsQuery)) .andExpect(status().isInternalServerError()) .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8"))) .andExpect(jsonPath("$.responseCode").value(500)) @@ -239,25 +239,6 @@ public class SearchControllerIntegrationTest { - private void loadTestData() throws ParseException { - Map<String, List<String>> backingStore = new HashMap<>(); - for(Map.Entry<String, String> indices : - ImmutableMap.of( - "bro_index_2017.01.01.01", IndexingDaoIntegrationTest.broData, - "snort_index_2017.01.01.01", IndexingDaoIntegrationTest.snortData - ).entrySet() - ) - { - List<String> results = new ArrayList<>(); - backingStore.put(indices.getKey(), results); - JSONArray broArray = (JSONArray) new JSONParser().parse(indices.getValue()); - for(Object o: broArray) { - JSONObject jsonObject = (JSONObject) o; - results.add(jsonObject.toJSONString()); - } - } - InMemoryDao.load(backingStore); - } private void loadColumnTypes() throws ParseException { Map<String, Map<String, FieldType>> columnTypes = new HashMap<>(); http://git-wip-us.apache.org/repos/asf/metron/blob/813adf2d/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/UpdateControllerIntegrationTest.java ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/UpdateControllerIntegrationTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/UpdateControllerIntegrationTest.java new file mode 100644 index 0000000..0d7fde7 --- /dev/null +++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/UpdateControllerIntegrationTest.java @@ -0,0 +1,192 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.rest.controller; + +import org.adrianwalker.multilinestring.Multiline; +import org.apache.curator.framework.CuratorFramework; +import org.apache.hadoop.hbase.client.Get; +import org.apache.hadoop.hbase.client.Result; +import org.apache.metron.hbase.mock.MockHTable; +import org.apache.metron.hbase.mock.MockHBaseTableProvider; +import org.apache.metron.rest.service.UpdateService; +import org.junit.Assert; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.http.MediaType; +import org.springframework.test.context.ActiveProfiles; +import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.test.web.servlet.MockMvc; +import org.springframework.test.web.servlet.ResultActions; +import org.springframework.test.web.servlet.setup.MockMvcBuilders; +import org.springframework.web.context.WebApplicationContext; + +import java.util.NavigableMap; + +import static org.apache.metron.rest.MetronRestConstants.TEST_PROFILE; +import static org.springframework.security.test.web.servlet.setup.SecurityMockMvcConfigurers.springSecurity; +import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.csrf; +import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.httpBasic; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.patch; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; + +@RunWith(SpringRunner.class) +@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) +@ActiveProfiles(TEST_PROFILE) +public class UpdateControllerIntegrationTest extends DaoControllerTest { + @Autowired + private UpdateService searchService; + @Autowired + public CuratorFramework client; + + @Autowired + private WebApplicationContext wac; + + private MockMvc mockMvc; + + private String updateUrl = "/api/v1/update"; + private String searchUrl = "/api/v1/search"; + private String user = "user"; + private String password = "password"; + + /** + { + "guid" : "bro_index_2017.01.01.01:1", + "sensorType" : "bro" + } + */ + @Multiline + public static String findMessage0; + + /** + { + "guid" : "bro_index_2017.01.01.01:1", + "sensorType" : "bro", + "patch" : [ + { + "op": "add" + , "path": "/project" + , "value": "metron" + } + ] + } + */ + @Multiline + public static String patch; + + /** + { + "guid" : "bro_index_2017.01.01.01:1", + "sensorType" : "bro", + "replacement" : { + "source:type": "bro", + "guid" : "bro_index_2017.01.01.01:1", + "ip_src_addr":"192.168.1.2", + "ip_src_port": 8009, + "timestamp":200, + "rejected":false + } + } + */ + @Multiline + public static String replace; + + + @BeforeClass + public static void setupHbase() { + MockHBaseTableProvider.addToCache(TABLE, CF); + } + + @Before + public void setup() throws Exception { + this.mockMvc = MockMvcBuilders.webAppContextSetup(this.wac).apply(springSecurity()).build(); + loadTestData(); + } + + @Test + public void test() throws Exception { + String guid = "bro_index_2017.01.01.01:1"; + ResultActions result = this.mockMvc.perform(post(searchUrl + "/findOne").with(httpBasic(user, password)).with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(findMessage0)); + try { + result.andExpect(status().isOk()) + .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8"))) + .andExpect(jsonPath("$.source:type").value("bro")) + .andExpect(jsonPath("$.guid").value(guid)) + .andExpect(jsonPath("$.project").doesNotExist()) + .andExpect(jsonPath("$.timestamp").value(2)) + ; + } + catch(Throwable t) { + System.err.println(result.andReturn().getResponse().getContentAsString()); + throw t; + } + MockHTable table = (MockHTable) MockHBaseTableProvider.getFromCache(TABLE); + Assert.assertEquals(0,table.size()); + this.mockMvc.perform(patch(updateUrl+ "/patch").with(httpBasic(user, password)) + .with(csrf()) + .contentType(MediaType.parseMediaType("application/json;charset=UTF-8")) + .content(patch) + ) + .andExpect(status().isOk()); + this.mockMvc.perform(post(searchUrl + "/findOne").with(httpBasic(user, password)).with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(findMessage0)) + .andExpect(status().isOk()) + .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8"))) + .andExpect(jsonPath("$.source:type").value("bro")) + .andExpect(jsonPath("$.guid").value(guid)) + .andExpect(jsonPath("$.project").value("metron")) + .andExpect(jsonPath("$.timestamp").value(2)) + ; + Assert.assertEquals(1,table.size()); + { + //ensure hbase is up to date + Get g = new Get(guid.getBytes()); + Result r = table.get(g); + NavigableMap<byte[], byte[]> columns = r.getFamilyMap(CF.getBytes()); + Assert.assertEquals(1, columns.size()); + } + this.mockMvc.perform(post(updateUrl+ "/replace").with(httpBasic(user, password)) + .with(csrf()) + .contentType(MediaType.parseMediaType("application/json;charset=UTF-8")) + .content(replace) + ) + .andExpect(status().isOk()); + this.mockMvc.perform(post(searchUrl + "/findOne").with(httpBasic(user, password)).with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(findMessage0)) + .andExpect(status().isOk()) + .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8"))) + .andExpect(jsonPath("$.source:type").value("bro")) + .andExpect(jsonPath("$.guid").value(guid)) + .andExpect(jsonPath("$.project").doesNotExist()) + .andExpect(jsonPath("$.timestamp").value(200)) + ; + Assert.assertEquals(1,table.size()); + { + //ensure hbase is up to date + Get g = new Get(guid.getBytes()); + Result r = table.get(g); + NavigableMap<byte[], byte[]> columns = r.getFamilyMap(CF.getBytes()); + Assert.assertEquals(2, columns.size()); + } + } + +} http://git-wip-us.apache.org/repos/asf/metron/blob/813adf2d/metron-interface/metron-rest/src/test/resources/zookeeper/global.json ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest/src/test/resources/zookeeper/global.json b/metron-interface/metron-rest/src/test/resources/zookeeper/global.json new file mode 100644 index 0000000..396896f --- /dev/null +++ b/metron-interface/metron-rest/src/test/resources/zookeeper/global.json @@ -0,0 +1,4 @@ +{ + "update.hbase.table" : "updates", + "update.hbase.cf" : "t" +} \ No newline at end of file
