Github user nickwallen commented on a diff in the pull request:
https://github.com/apache/metron/pull/1247#discussion_r236461564
--- Diff:
metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java
---
@@ -97,45 +118,63 @@ protected static InMemoryComponent startIndex() throws
Exception {
return es;
}
- protected static void loadTestData() throws ParseException, IOException {
+ protected static void loadTestData() throws Exception {
ElasticSearchComponent es = (ElasticSearchComponent) indexComponent;
+ // define the bro index template
+ String broIndex = "bro_index_2017.01.01.01";
JSONObject broTemplate = JSONUtils.INSTANCE.load(new
File(broTemplatePath), JSONObject.class);
addTestFieldMappings(broTemplate, "bro_doc");
-
es.getClient().admin().indices().prepareCreate("bro_index_2017.01.01.01")
- .addMapping("bro_doc",
JSONUtils.INSTANCE.toJSON(broTemplate.get("mappings"), false)).get();
+ es.getClient().admin().indices().prepareCreate(broIndex)
+ .addMapping("bro_doc",
JSONUtils.INSTANCE.toJSON(broTemplate.get("mappings"), false)).get();
+
+ // define the snort index template
+ String snortIndex = "snort_index_2017.01.01.02";
JSONObject snortTemplate = JSONUtils.INSTANCE.load(new
File(snortTemplatePath), JSONObject.class);
addTestFieldMappings(snortTemplate, "snort_doc");
-
es.getClient().admin().indices().prepareCreate("snort_index_2017.01.01.02")
- .addMapping("snort_doc",
JSONUtils.INSTANCE.toJSON(snortTemplate.get("mappings"), false)).get();
-
- BulkRequestBuilder bulkRequest = es.getClient().prepareBulk()
- .setRefreshPolicy(WriteRequest.RefreshPolicy.WAIT_UNTIL);
- JSONArray broArray = (JSONArray) new JSONParser().parse(broData);
- for (Object o : broArray) {
- JSONObject jsonObject = (JSONObject) o;
- IndexRequestBuilder indexRequestBuilder = es.getClient()
- .prepareIndex("bro_index_2017.01.01.01", "bro_doc");
- indexRequestBuilder = indexRequestBuilder.setId((String)
jsonObject.get("guid"));
- indexRequestBuilder =
indexRequestBuilder.setSource(jsonObject.toJSONString());
- indexRequestBuilder = indexRequestBuilder
- .setTimestamp(jsonObject.get("timestamp").toString());
- bulkRequest.add(indexRequestBuilder);
+ es.getClient().admin().indices().prepareCreate(snortIndex)
+ .addMapping("snort_doc",
JSONUtils.INSTANCE.toJSON(snortTemplate.get("mappings"), false)).get();
+
+ // setup the classes required to write the test data
+ AccessConfig accessConfig = createAccessConfig();
+ ElasticsearchClient client =
ElasticsearchUtils.getClient(createGlobalConfig());
+ ElasticsearchRetrieveLatestDao retrieveLatestDao = new
ElasticsearchRetrieveLatestDao(client);
+ ElasticsearchColumnMetadataDao columnMetadataDao = new
ElasticsearchColumnMetadataDao(client);
+ ElasticsearchRequestSubmitter requestSubmitter = new
ElasticsearchRequestSubmitter(client);
+ ElasticsearchUpdateDao updateDao = new ElasticsearchUpdateDao(client,
accessConfig, retrieveLatestDao);
+ ElasticsearchSearchDao searchDao = new ElasticsearchSearchDao(client,
accessConfig, columnMetadataDao, requestSubmitter);
+
+ // write the test documents for Bro
+ List<String> broDocuments = new ArrayList<>();
+ for (Object broObject: (JSONArray) new JSONParser().parse(broData)) {
+ broDocuments.add(((JSONObject) broObject).toJSONString());
}
- JSONArray snortArray = (JSONArray) new JSONParser().parse(snortData);
- for (Object o : snortArray) {
- JSONObject jsonObject = (JSONObject) o;
- IndexRequestBuilder indexRequestBuilder = es.getClient()
- .prepareIndex("snort_index_2017.01.01.02", "snort_doc");
- indexRequestBuilder = indexRequestBuilder.setId((String)
jsonObject.get("guid"));
- indexRequestBuilder =
indexRequestBuilder.setSource(jsonObject.toJSONString());
- indexRequestBuilder = indexRequestBuilder
- .setTimestamp(jsonObject.get("timestamp").toString());
- bulkRequest.add(indexRequestBuilder);
+ es.add(updateDao, broIndex, "bro", broDocuments);
+
+ // write the test documents for Snort
+ List<String> snortDocuments = new ArrayList<>();
+ for (Object snortObject: (JSONArray) new
JSONParser().parse(snortData)) {
+ snortDocuments.add(((JSONObject) snortObject).toJSONString());
}
- BulkResponse bulkResponse = bulkRequest.execute().actionGet();
- if (bulkResponse.hasFailures()) {
- throw new RuntimeException("Failed to index test data");
+ es.add(updateDao, snortIndex, "snort", snortDocuments);
+
+ // wait until the test documents are visible
+ assertEventually(() -> Assert.assertEquals(10,
findAll(searchDao).getTotal()));
--- End diff --
I have tried to do exactly this on a private branch that is closer to the
code base in #1269. I made the integration tests supply a RefreshPolicy that
was supposed to hold them up until the loaded data is discoverable/searchable.
I was never able to get that to work consistently though despite what the ES
docs say. I still needed the `assertEventually` for the integration tests to
pass consistently.
---
