METRON-1004 Travis CI - Job Exceeded Maximum Time Limit (justinleet) closes apache/metron#624
Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/df94ed40 Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/df94ed40 Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/df94ed40 Branch: refs/heads/master Commit: df94ed40523db375b0bac4601a7efffa5a1fede5 Parents: 095be23 Author: justinleet <[email protected]> Authored: Fri Jun 30 09:21:17 2017 -0400 Committer: leet <[email protected]> Committed: Fri Jun 30 09:21:17 2017 -0400 ---------------------------------------------------------------------- .travis.yml | 12 +- .../maas/service/MaasIntegrationTest.java | 30 +- .../client/HBaseProfilerClientTest.java | 28 +- .../integration/ConfigUploadComponent.java | 16 +- .../integration/ProfilerIntegrationTest.java | 73 +- metron-interface/metron-rest/pom.xml | 1 + .../apache/metron/rest/config/TestConfig.java | 6 +- .../KafkaControllerIntegrationTest.java | 78 +- .../metron/common/bolt/ConfiguredBolt.java | 4 + .../extractor/stix/StixExtractorTest.java | 70 +- .../test/resources/taxii-messages/messages.poll | 2489 ------------------ .../components/ConfigUploadComponent.java | 5 + .../metron/hbase/client/HBaseClientTest.java | 49 +- .../integration/IndexingIntegrationTest.java | 2 + .../metron/integration/ComponentRunner.java | 23 +- .../metron/integration/InMemoryComponent.java | 5 +- .../components/FluxTopologyComponent.java | 43 +- .../integration/components/KafkaComponent.java | 37 +- .../components/ZKServerComponent.java | 18 +- metron-platform/metron-management/pom.xml | 4 + .../management/ConfigurationFunctionsTest.java | 13 +- .../management/FileSystemFunctionsTest.java | 55 +- .../KafkaFunctionsIntegrationTest.java | 8 +- .../metron-parsers/parser-testing.md | 70 + metron-platform/metron-parsers/pom.xml | 4 + .../apache/metron/parsers/bolt/ParserBolt.java | 22 +- .../parsers/integration/ParserDriver.java | 170 ++ .../integration/ParserIntegrationTest.java | 112 +- .../components/ParserTopologyComponent.java | 32 +- .../PcapTopologyIntegrationTest.java | 4 + .../integration/components/SolrComponent.java | 9 + .../storm/kafka/flux/StormKafkaSpout.java | 14 +- .../org/apache/metron/test/mock/MockHTable.java | 8 +- pom.xml | 1 + 34 files changed, 762 insertions(+), 2753 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/metron/blob/df94ed40/.travis.yml ---------------------------------------------------------------------- diff --git a/.travis.yml b/.travis.yml index 8fb218a..97a816c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,6 +6,7 @@ addons: - ubuntu-toolchain-r-test packages: - g++-4.8 +sudo: required install: true language: java jdk: @@ -15,9 +16,18 @@ before_install: - unzip -qq apache-maven-3.3.9-bin.zip - export M2_HOME=$PWD/apache-maven-3.3.9 - export PATH=$M2_HOME/bin:$PATH + - npm config set cache $HOME/.npm-cache --global + - npm config set prefix $HOME/.npm-prefix --global script: - | - time mvn -q -T 2C -DskipTests install && time mvn -q -T 2C jacoco:prepare-agent surefire:test@unit-tests && mvn -q jacoco:prepare-agent surefire:test@integration-tests && time mvn -q jacoco:prepare-agent test --projects metron-interface/metron-config && time build_utils/verify_licenses.sh + time mvn -q -T 2C -DskipTests install && time mvn -q -T 2C surefire:test@unit-tests && time mvn -q surefire:test@integration-tests && time mvn -q test --projects metron-interface/metron-config && time build_utils/verify_licenses.sh +before_cache: + - rm -rf $HOME/.m2/repository/org/apache/metron + cache: + timeout: 1000 directories: + - $HOME/.npm-cache + - $HOME/.npm-prefix + - metron-interface/metron-config/node_modules - $HOME/.m2 http://git-wip-us.apache.org/repos/asf/metron/blob/df94ed40/metron-analytics/metron-maas-service/src/test/java/org/apache/metron/maas/service/MaasIntegrationTest.java ---------------------------------------------------------------------- diff --git a/metron-analytics/metron-maas-service/src/test/java/org/apache/metron/maas/service/MaasIntegrationTest.java b/metron-analytics/metron-maas-service/src/test/java/org/apache/metron/maas/service/MaasIntegrationTest.java index 221a840..a75f2a3 100644 --- a/metron-analytics/metron-maas-service/src/test/java/org/apache/metron/maas/service/MaasIntegrationTest.java +++ b/metron-analytics/metron-maas-service/src/test/java/org/apache/metron/maas/service/MaasIntegrationTest.java @@ -33,17 +33,14 @@ import org.apache.curator.RetryPolicy; import org.apache.curator.framework.CuratorFramework; import org.apache.curator.framework.CuratorFrameworkFactory; import org.apache.curator.retry.ExponentialBackoffRetry; -import org.apache.curator.test.TestingServer; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.net.NetUtils; -import org.apache.hadoop.util.JarFinder; import org.apache.hadoop.util.Shell; import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.client.api.YarnClient; import org.apache.hadoop.yarn.conf.YarnConfiguration; -import org.apache.hadoop.yarn.server.MiniYARNCluster; import org.apache.metron.integration.ComponentRunner; import org.apache.metron.integration.components.YarnComponent; import org.apache.metron.integration.components.ZKServerComponent; @@ -57,25 +54,25 @@ import org.apache.metron.maas.util.ConfigUtil; import org.apache.metron.test.utils.UnitTestHelper; import org.apache.zookeeper.KeeperException; import org.junit.After; +import org.junit.AfterClass; import org.junit.Assert; -import org.junit.Before; +import org.junit.BeforeClass; import org.junit.Test; public class MaasIntegrationTest { private static final Log LOG = LogFactory.getLog(MaasIntegrationTest.class); - private CuratorFramework client; - private ComponentRunner runner; - private YarnComponent yarnComponent; - private ZKServerComponent zkServerComponent; - @Before - public void setup() throws Exception { + private static CuratorFramework client; + private static ComponentRunner runner; + private static YarnComponent yarnComponent; + private static ZKServerComponent zkServerComponent; + + @BeforeClass + public static void setupBeforeClass() throws Exception { UnitTestHelper.setJavaLoggingLevel(Level.SEVERE); LOG.info("Starting up YARN cluster"); - Map<String, String> properties = new HashMap<>(); zkServerComponent = new ZKServerComponent(); - yarnComponent = new YarnComponent().withApplicationMasterClass(ApplicationMaster.class).withTestName(MaasIntegrationTest.class.getSimpleName()); runner = new ComponentRunner.Builder() @@ -92,14 +89,19 @@ public class MaasIntegrationTest { client.start(); } - @After - public void tearDown(){ + @AfterClass + public static void tearDownAfterClass(){ if(client != null){ client.close(); } runner.stop(); } + @After + public void tearDown() { + runner.reset(); + } + @Test(timeout=900000) public void testMaaSWithDomain() throws Exception { testDSShell(true); http://git-wip-us.apache.org/repos/asf/metron/blob/df94ed40/metron-analytics/metron-profiler-client/src/test/java/org/apache/metron/profiler/client/HBaseProfilerClientTest.java ---------------------------------------------------------------------- diff --git a/metron-analytics/metron-profiler-client/src/test/java/org/apache/metron/profiler/client/HBaseProfilerClientTest.java b/metron-analytics/metron-profiler-client/src/test/java/org/apache/metron/profiler/client/HBaseProfilerClientTest.java index ed75a65..d3a0fe5 100644 --- a/metron-analytics/metron-profiler-client/src/test/java/org/apache/metron/profiler/client/HBaseProfilerClientTest.java +++ b/metron-analytics/metron-profiler-client/src/test/java/org/apache/metron/profiler/client/HBaseProfilerClientTest.java @@ -20,11 +20,6 @@ package org.apache.metron.profiler.client; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.HBaseConfiguration; -import org.apache.hadoop.hbase.HBaseTestingUtility; -import org.apache.hadoop.hbase.client.HTableInterface; -import org.apache.hadoop.hbase.util.Bytes; import org.apache.metron.profiler.ProfileMeasurement; import org.apache.metron.profiler.hbase.ColumnBuilder; import org.apache.metron.profiler.hbase.RowKeyBuilder; @@ -32,6 +27,7 @@ import org.apache.metron.profiler.hbase.SaltyRowKeyBuilder; import org.apache.metron.profiler.hbase.ValueOnlyColumnBuilder; import org.apache.metron.profiler.stellar.DefaultStellarExecutor; import org.apache.metron.profiler.stellar.StellarExecutor; +import org.apache.metron.test.mock.MockHTable; import org.junit.After; import org.junit.AfterClass; import org.junit.Before; @@ -62,30 +58,14 @@ public class HBaseProfilerClientTest { private static final int periodsPerHour = 4; private HBaseProfilerClient client; - private HTableInterface table; + private MockHTable table; private StellarExecutor executor; - private static HBaseTestingUtility util; private ProfileWriter profileWriter; - @BeforeClass - public static void startHBase() throws Exception { - Configuration config = HBaseConfiguration.create(); - config.set("hbase.master.hostname", "localhost"); - config.set("hbase.regionserver.hostname", "localhost"); - util = new HBaseTestingUtility(config); - util.startMiniCluster(); - } - - @AfterClass - public static void stopHBase() throws Exception { - util.shutdownMiniCluster(); - util.cleanupTestDir(); - } - @Before public void setup() throws Exception { - table = util.createTable(Bytes.toBytes(tableName), Bytes.toBytes(columnFamily)); + table = new MockHTable(tableName, columnFamily); executor = new DefaultStellarExecutor(); // used to write values to be read during testing @@ -99,7 +79,7 @@ public class HBaseProfilerClientTest { @After public void tearDown() throws Exception { - util.deleteTable(tableName); + table.clear(); } /** http://git-wip-us.apache.org/repos/asf/metron/blob/df94ed40/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/integration/ConfigUploadComponent.java ---------------------------------------------------------------------- diff --git a/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/integration/ConfigUploadComponent.java b/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/integration/ConfigUploadComponent.java index b3fc6c7..b59d0b5 100644 --- a/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/integration/ConfigUploadComponent.java +++ b/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/integration/ConfigUploadComponent.java @@ -20,6 +20,7 @@ package org.apache.metron.profiler.integration; import org.apache.curator.framework.CuratorFramework; +import org.apache.curator.framework.imps.CuratorFrameworkState; import org.apache.metron.integration.InMemoryComponent; import org.apache.metron.integration.UnableToStartException; import org.apache.metron.integration.components.ZKServerComponent; @@ -56,6 +57,15 @@ public class ConfigUploadComponent implements InMemoryComponent { // nothing to do } + public void update() + throws UnableToStartException { + try { + upload(); + } catch (Exception e) { + throw new UnableToStartException(e.getMessage(), e); + } + } + /** * Uploads configuration to Zookeeper. * @throws Exception @@ -63,7 +73,9 @@ public class ConfigUploadComponent implements InMemoryComponent { private void upload() throws Exception { final String zookeeperUrl = topologyProperties.getProperty(ZKServerComponent.ZOOKEEPER_PROPERTY); try(CuratorFramework client = getClient(zookeeperUrl)) { - client.start(); + if(client.getState() != CuratorFrameworkState.STARTED) { + client.start(); + } uploadGlobalConfig(client); uploadProfilerConfig(client); } @@ -87,7 +99,7 @@ public class ConfigUploadComponent implements InMemoryComponent { * @param client The zookeeper client. */ private void uploadGlobalConfig(CuratorFramework client) throws Exception { - if (globalConfiguration == null) { + if (globalConfiguration != null) { byte[] globalConfig = readGlobalConfigFromFile(globalConfiguration); if (globalConfig.length > 0) { writeGlobalConfigToZookeeper(readGlobalConfigFromFile(globalConfiguration), client); http://git-wip-us.apache.org/repos/asf/metron/blob/df94ed40/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/integration/ProfilerIntegrationTest.java ---------------------------------------------------------------------- diff --git a/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/integration/ProfilerIntegrationTest.java b/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/integration/ProfilerIntegrationTest.java index 7591300..b863ebc 100644 --- a/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/integration/ProfilerIntegrationTest.java +++ b/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/integration/ProfilerIntegrationTest.java @@ -33,6 +33,7 @@ import org.apache.metron.common.utils.SerDeUtils; import org.apache.metron.hbase.TableProvider; import org.apache.metron.integration.BaseIntegrationTest; import org.apache.metron.integration.ComponentRunner; +import org.apache.metron.integration.UnableToStartException; import org.apache.metron.integration.components.FluxTopologyComponent; import org.apache.metron.integration.components.KafkaComponent; import org.apache.metron.integration.components.ZKServerComponent; @@ -41,7 +42,10 @@ import org.apache.metron.profiler.hbase.ValueOnlyColumnBuilder; import org.apache.metron.statistics.OnlineStatisticsProvider; import org.apache.metron.test.mock.MockHTable; import org.junit.After; +import org.junit.AfterClass; import org.junit.Assert; +import org.junit.Before; +import org.junit.BeforeClass; import org.junit.Test; import java.io.File; @@ -76,7 +80,7 @@ public class ProfilerIntegrationTest extends BaseIntegrationTest { * } */ @Multiline - private String message1; + private static String message1; /** * { @@ -87,7 +91,7 @@ public class ProfilerIntegrationTest extends BaseIntegrationTest { * } */ @Multiline - private String message2; + private static String message2; /** * { @@ -98,15 +102,16 @@ public class ProfilerIntegrationTest extends BaseIntegrationTest { * } */ @Multiline - private String message3; + private static String message3; - private ColumnBuilder columnBuilder; - private ZKServerComponent zkComponent; - private FluxTopologyComponent fluxComponent; - private KafkaComponent kafkaComponent; - private List<byte[]> input; - private ComponentRunner runner; - private MockHTable profilerTable; + private static ColumnBuilder columnBuilder; + private static ZKServerComponent zkComponent; + private static FluxTopologyComponent fluxComponent; + private static KafkaComponent kafkaComponent; + private static ConfigUploadComponent configUploadComponent; + private static List<byte[]> input; + private static ComponentRunner runner; + private static MockHTable profilerTable; private static final String tableName = "profiler"; private static final String columnFamily = "P"; @@ -133,7 +138,7 @@ public class ProfilerIntegrationTest extends BaseIntegrationTest { @Test public void testExample1() throws Exception { - setup(TEST_RESOURCES + "/config/zookeeper/readme-example-1"); + update(TEST_RESOURCES + "/config/zookeeper/readme-example-1"); // start the topology and write test messages to kafka fluxComponent.submitTopology(); @@ -158,7 +163,7 @@ public class ProfilerIntegrationTest extends BaseIntegrationTest { @Test public void testExample2() throws Exception { - setup(TEST_RESOURCES + "/config/zookeeper/readme-example-2"); + update(TEST_RESOURCES + "/config/zookeeper/readme-example-2"); // start the topology and write test messages to kafka fluxComponent.submitTopology(); @@ -191,7 +196,7 @@ public class ProfilerIntegrationTest extends BaseIntegrationTest { @Test public void testExample3() throws Exception { - setup(TEST_RESOURCES + "/config/zookeeper/readme-example-3"); + update(TEST_RESOURCES + "/config/zookeeper/readme-example-3"); // start the topology and write test messages to kafka fluxComponent.submitTopology(); @@ -216,7 +221,7 @@ public class ProfilerIntegrationTest extends BaseIntegrationTest { @Test public void testExample4() throws Exception { - setup(TEST_RESOURCES + "/config/zookeeper/readme-example-4"); + update(TEST_RESOURCES + "/config/zookeeper/readme-example-4"); // start the topology and write test messages to kafka fluxComponent.submitTopology(); @@ -239,7 +244,8 @@ public class ProfilerIntegrationTest extends BaseIntegrationTest { @Test public void testPercentiles() throws Exception { - setup(TEST_RESOURCES + "/config/zookeeper/percentiles"); + update(TEST_RESOURCES + "/config/zookeeper/percentiles"); + // start the topology and write test messages to kafka fluxComponent.submitTopology(); @@ -277,9 +283,15 @@ public class ProfilerIntegrationTest extends BaseIntegrationTest { return results; } - public void setup(String pathToConfig) throws Exception { + @BeforeClass + public static void setupBeforeClass() throws UnableToStartException { columnBuilder = new ValueOnlyColumnBuilder(columnFamily); + List<String> inputNew = Stream.of(message1, message2, message3) + .map(m -> Collections.nCopies(5, m)) + .flatMap(l -> l.stream()) + .collect(Collectors.toList()); + // create input messages for the profiler to consume input = Stream.of(message1, message2, message3) .map(Bytes::toBytes) @@ -320,10 +332,8 @@ public class ProfilerIntegrationTest extends BaseIntegrationTest { new KafkaComponent.Topic(outputTopic, 1))); // upload profiler configuration to zookeeper - ConfigUploadComponent configUploadComponent = new ConfigUploadComponent() - .withTopologyProperties(topologyProperties) - .withGlobalConfiguration(pathToConfig) - .withProfilerConfiguration(pathToConfig); + configUploadComponent = new ConfigUploadComponent() + .withTopologyProperties(topologyProperties); // load flux definition for the profiler topology fluxComponent = new FluxTopologyComponent.Builder() @@ -345,11 +355,32 @@ public class ProfilerIntegrationTest extends BaseIntegrationTest { runner.start(); } + public void update(String path) throws Exception { + configUploadComponent.withGlobalConfiguration(path) + .withProfilerConfiguration(path); + configUploadComponent.update(); + } + + @AfterClass + public static void tearDownAfterClass() throws Exception { + MockHTable.Provider.clear(); + if (runner != null) { + runner.stop(); + } + } + + @Before + public void setup() { + // create the mock table + profilerTable = (MockHTable) MockHTable.Provider.addToCache(tableName, columnFamily); + } + @After public void tearDown() throws Exception { MockHTable.Provider.clear(); + profilerTable.clear(); if (runner != null) { - runner.stop(); + runner.reset(); } } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/metron/blob/df94ed40/metron-interface/metron-rest/pom.xml ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest/pom.xml b/metron-interface/metron-rest/pom.xml index 1c3ff92..94ed64b 100644 --- a/metron-interface/metron-rest/pom.xml +++ b/metron-interface/metron-rest/pom.xml @@ -257,6 +257,7 @@ <version>1.7</version> <scope>test</scope> </dependency> + </dependencies> <dependencyManagement> http://git-wip-us.apache.org/repos/asf/metron/blob/df94ed40/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/config/TestConfig.java ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/config/TestConfig.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/config/TestConfig.java index adfe056..5c2acf7 100644 --- a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/config/TestConfig.java +++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/config/TestConfig.java @@ -65,13 +65,11 @@ public class TestConfig { return new KafkaComponent().withTopologyProperties(zkProperties); } - //@Bean(destroyMethod = "stop") - @Bean + @Bean(destroyMethod = "stop") public ComponentRunner componentRunner(ZKServerComponent zkServerComponent, KafkaComponent kafkaWithZKComponent) { ComponentRunner runner = new ComponentRunner.Builder() .withComponent("zk", zkServerComponent) - .withComponent("kafka", kafkaWithZKComponent) - .withCustomShutdownOrder(new String[]{"kafka", "zk"}) + .withCustomShutdownOrder(new String[]{"zk"}) .build(); try { runner.start(); http://git-wip-us.apache.org/repos/asf/metron/blob/df94ed40/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/KafkaControllerIntegrationTest.java ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/KafkaControllerIntegrationTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/KafkaControllerIntegrationTest.java index 745bc56..9e6d408 100644 --- a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/KafkaControllerIntegrationTest.java +++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/KafkaControllerIntegrationTest.java @@ -19,6 +19,8 @@ package org.apache.metron.rest.controller; import kafka.common.TopicAlreadyMarkedForDeletionException; import org.adrianwalker.multilinestring.Multiline; +import org.apache.metron.integration.ComponentRunner; +import org.apache.metron.integration.UnableToStartException; import org.apache.metron.integration.components.KafkaComponent; import org.apache.metron.rest.generator.SampleDataGenerator; import org.apache.metron.rest.service.KafkaService; @@ -35,7 +37,6 @@ import org.springframework.test.context.ActiveProfiles; import org.springframework.test.context.junit4.SpringRunner; import org.springframework.test.web.servlet.MockMvc; import org.springframework.test.web.servlet.MvcResult; -import org.springframework.test.web.servlet.ResultActions; import org.springframework.test.web.servlet.setup.MockMvcBuilders; import org.springframework.web.context.WebApplicationContext; import org.springframework.web.util.NestedServletException; @@ -59,8 +60,44 @@ import static org.springframework.test.web.servlet.result.MockMvcResultMatchers. public class KafkaControllerIntegrationTest { private static final int KAFKA_RETRY = 10; + // A bug in Spring and/or Kafka forced us to move into a component that is spun up and down per test-case + // Given the large spinup time of components, please avoid this pattern until we upgrade Spring. + // See: https://issues.apache.org/jira/browse/METRON-1009 @Autowired private KafkaComponent kafkaWithZKComponent; + private ComponentRunner runner; + + + interface Evaluation { + void tryTest() throws Exception; + } + + private void testAndRetry(Evaluation evaluation) throws Exception{ + testAndRetry(KAFKA_RETRY, evaluation); + } + + private void testAndRetry(int numRetries, Evaluation evaluation) throws Exception { + AssertionError lastError = null; + for(int i = 0;i < numRetries;++i) { + try { + evaluation.tryTest(); + return; + } + catch(AssertionError error) { + if(error.getMessage().contains("but was:<404>")) { + lastError = error; + Thread.sleep(1000); + continue; + } + else { + throw error; + } + } + } + if(lastError != null) { + throw lastError; + } + } class SampleDataRunner implements Runnable { @@ -79,7 +116,7 @@ public class KafkaControllerIntegrationTest { broSampleDataGenerator.generateSampleData(path); } } catch (ParseException|IOException e) { - e.printStackTrace(); + throw new IllegalStateException("Caught an error generating sample data", e); } } @@ -116,6 +153,15 @@ public class KafkaControllerIntegrationTest { @Before public void setup() throws Exception { + runner = new ComponentRunner.Builder() + .withComponent("kafka", kafkaWithZKComponent) + .withCustomShutdownOrder(new String[]{"kafka"}) + .build(); + try { + runner.start(); + } catch (UnableToStartException e) { + e.printStackTrace(); + } this.mockMvc = MockMvcBuilders.webAppContextSetup(this.wac).apply(springSecurity()).build(); } @@ -142,34 +188,40 @@ public class KafkaControllerIntegrationTest { this.kafkaService.deleteTopic("bro"); this.kafkaService.deleteTopic("someTopic"); Thread.sleep(1000); + testAndRetry(() -> this.mockMvc.perform(delete(kafkaUrl + "/topic/bro").with(httpBasic(user,password)).with(csrf())) + .andExpect(status().isNotFound()) + ); - this.mockMvc.perform(delete(kafkaUrl + "/topic/bro").with(httpBasic(user,password)).with(csrf())) - .andExpect(status().isNotFound()); - + testAndRetry(() -> this.mockMvc.perform(post(kafkaUrl + "/topic").with(httpBasic(user,password)).with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(broTopic)) .andExpect(status().isCreated()) .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8"))) .andExpect(jsonPath("$.name").value("bro")) .andExpect(jsonPath("$.numPartitions").value(1)) - .andExpect(jsonPath("$.replicationFactor").value(1)); - + .andExpect(jsonPath("$.replicationFactor").value(1)) + ); sampleDataThread.start(); Thread.sleep(1000); - + testAndRetry(() -> this.mockMvc.perform(get(kafkaUrl + "/topic/bro").with(httpBasic(user,password))) .andExpect(status().isOk()) .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8"))) .andExpect(jsonPath("$.name").value("bro")) .andExpect(jsonPath("$.numPartitions").value(1)) - .andExpect(jsonPath("$.replicationFactor").value(1)); + .andExpect(jsonPath("$.replicationFactor").value(1)) + ); + this.mockMvc.perform(get(kafkaUrl + "/topic/someTopic").with(httpBasic(user,password))) .andExpect(status().isNotFound()); + testAndRetry(() -> this.mockMvc.perform(get(kafkaUrl + "/topic").with(httpBasic(user,password))) .andExpect(status().isOk()) .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8"))) - .andExpect(jsonPath("$", Matchers.hasItem("bro"))); + .andExpect(jsonPath("$", Matchers.hasItem("bro"))) + ); + for(int i = 0;i < KAFKA_RETRY;++i) { MvcResult result = this.mockMvc.perform(get(kafkaUrl + "/topic/bro/sample").with(httpBasic(user, password))) .andReturn(); @@ -178,10 +230,13 @@ public class KafkaControllerIntegrationTest { } Thread.sleep(1000); } + + testAndRetry(() -> this.mockMvc.perform(get(kafkaUrl + "/topic/bro/sample").with(httpBasic(user,password))) .andExpect(status().isOk()) .andExpect(content().contentType(MediaType.parseMediaType("text/plain;charset=UTF-8"))) - .andExpect(jsonPath("$").isNotEmpty()); + .andExpect(jsonPath("$").isNotEmpty()) + ); this.mockMvc.perform(get(kafkaUrl + "/topic/someTopic/sample").with(httpBasic(user,password))) .andExpect(status().isNotFound()); @@ -216,5 +271,6 @@ public class KafkaControllerIntegrationTest { @After public void tearDown() { sampleDataRunner.stop(); + runner.stop(); } } http://git-wip-us.apache.org/repos/asf/metron/blob/df94ed40/metron-platform/metron-common/src/main/java/org/apache/metron/common/bolt/ConfiguredBolt.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/bolt/ConfiguredBolt.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/bolt/ConfiguredBolt.java index 45463bf..8163981 100644 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/bolt/ConfiguredBolt.java +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/bolt/ConfiguredBolt.java @@ -69,6 +69,10 @@ public abstract class ConfiguredBolt<CONFIG_T extends Configurations> extends Ba @Override public void prepare(Map stormConf, TopologyContext context, OutputCollector collector) { + prepCache(); + } + + protected void prepCache() { try { if (client == null) { RetryPolicy retryPolicy = new ExponentialBackoffRetry(1000, 3); http://git-wip-us.apache.org/repos/asf/metron/blob/df94ed40/metron-platform/metron-data-management/src/test/java/org/apache/metron/dataloads/extractor/stix/StixExtractorTest.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-data-management/src/test/java/org/apache/metron/dataloads/extractor/stix/StixExtractorTest.java b/metron-platform/metron-data-management/src/test/java/org/apache/metron/dataloads/extractor/stix/StixExtractorTest.java index 597a3a5..dc078ba 100644 --- a/metron-platform/metron-data-management/src/test/java/org/apache/metron/dataloads/extractor/stix/StixExtractorTest.java +++ b/metron-platform/metron-data-management/src/test/java/org/apache/metron/dataloads/extractor/stix/StixExtractorTest.java @@ -83,33 +83,55 @@ public class StixExtractorTest { testStixAddresses(stixDocWithoutCondition); } - public void testStixAddresses(String stixDoc) throws Exception { + public void testStixAddresses(final String stixDoc) throws Exception { + Thread t1 = new Thread( () -> { - ExtractorHandler handler = ExtractorHandler.load(stixConfigOnlyIPV4); - Extractor extractor = handler.getExtractor(); - Iterable<LookupKV> results = extractor.extract(stixDoc); + try { + ExtractorHandler handler = ExtractorHandler.load(stixConfigOnlyIPV4); + Extractor extractor = handler.getExtractor(); + Iterable<LookupKV> results = extractor.extract(stixDoc); - Assert.assertEquals(3, Iterables.size(results)); - Assert.assertEquals("10.0.0.0", ((EnrichmentKey)(Iterables.get(results, 0).getKey())).indicator); - Assert.assertEquals("10.0.0.1", ((EnrichmentKey)(Iterables.get(results, 1).getKey())).indicator); - Assert.assertEquals("10.0.0.2", ((EnrichmentKey)(Iterables.get(results, 2).getKey())).indicator); - } + Assert.assertEquals(3, Iterables.size(results)); + Assert.assertEquals("10.0.0.0", ((EnrichmentKey) (Iterables.get(results, 0).getKey())).indicator); + Assert.assertEquals("10.0.0.1", ((EnrichmentKey) (Iterables.get(results, 1).getKey())).indicator); + Assert.assertEquals("10.0.0.2", ((EnrichmentKey) (Iterables.get(results, 2).getKey())).indicator); + } + catch(Exception ex) { + throw new RuntimeException(ex.getMessage(), ex); + } + }); + Thread t2 = new Thread( () -> { - - ExtractorHandler handler = ExtractorHandler.load(stixConfig); - Extractor extractor = handler.getExtractor(); - Iterable<LookupKV> results = extractor.extract(stixDoc); - Assert.assertEquals(3, Iterables.size(results)); - Assert.assertEquals("10.0.0.0", ((EnrichmentKey)(Iterables.get(results, 0).getKey())).indicator); - Assert.assertEquals("10.0.0.1", ((EnrichmentKey)(Iterables.get(results, 1).getKey())).indicator); - Assert.assertEquals("10.0.0.2", ((EnrichmentKey)(Iterables.get(results, 2).getKey())).indicator); - } + try { + ExtractorHandler handler = ExtractorHandler.load(stixConfig); + Extractor extractor = handler.getExtractor(); + Iterable<LookupKV> results = extractor.extract(stixDoc); + Assert.assertEquals(3, Iterables.size(results)); + Assert.assertEquals("10.0.0.0", ((EnrichmentKey) (Iterables.get(results, 0).getKey())).indicator); + Assert.assertEquals("10.0.0.1", ((EnrichmentKey) (Iterables.get(results, 1).getKey())).indicator); + Assert.assertEquals("10.0.0.2", ((EnrichmentKey) (Iterables.get(results, 2).getKey())).indicator); + } + catch(Exception ex) { + throw new RuntimeException(ex.getMessage(), ex); + } + }); + Thread t3 = new Thread( () -> { - - ExtractorHandler handler = ExtractorHandler.load(stixConfigOnlyIPV6); - Extractor extractor = handler.getExtractor(); - Iterable<LookupKV> results = extractor.extract(stixDoc); - Assert.assertEquals(0, Iterables.size(results)); - } + try { + ExtractorHandler handler = ExtractorHandler.load(stixConfigOnlyIPV6); + Extractor extractor = handler.getExtractor(); + Iterable<LookupKV> results = extractor.extract(stixDoc); + Assert.assertEquals(0, Iterables.size(results)); + } + catch(Exception ex) { + throw new RuntimeException(ex.getMessage(), ex); + } + }); + t1.run(); + t2.run(); + t3.run(); + t1.join(); + t2.join(); + t3.join(); } }
