This is an automated email from the ASF dual-hosted git repository. oleewere pushed a commit to branch branch-2.7 in repository https://gitbox.apache.org/repos/asf/ambari.git
The following commit(s) were added to refs/heads/branch-2.7 by this push: new f82827b AMBARI-24353. Log Feeder: generate solr id from specific fields. (#1882) f82827b is described below commit f82827b98faa9746abb9ea5feca8e390778843e4 Author: Olivér Szabó <oleew...@gmail.com> AuthorDate: Wed Jul 25 15:57:13 2018 +0200 AMBARI-24353. Log Feeder: generate solr id from specific fields. (#1882) --- .../ambari/logfeeder/plugin/output/Output.java | 8 ++ .../ambari/logfeeder/common/IdGeneratorHelper.java | 62 ++++++++++++++ .../ambari/logfeeder/output/OutputManagerImpl.java | 9 +- .../apache/ambari/logfeeder/output/OutputSolr.java | 16 ++-- .../logfeeder/common/IdGeneratorHelperTest.java | 98 ++++++++++++++++++++++ 5 files changed, 183 insertions(+), 10 deletions(-) diff --git a/ambari-logsearch/ambari-logsearch-logfeeder-plugin-api/src/main/java/org/apache/ambari/logfeeder/plugin/output/Output.java b/ambari-logsearch/ambari-logsearch-logfeeder-plugin-api/src/main/java/org/apache/ambari/logfeeder/plugin/output/Output.java index f369680..13e5ad8 100644 --- a/ambari-logsearch/ambari-logsearch-logfeeder-plugin-api/src/main/java/org/apache/ambari/logfeeder/plugin/output/Output.java +++ b/ambari-logsearch/ambari-logsearch-logfeeder-plugin-api/src/main/java/org/apache/ambari/logfeeder/plugin/output/Output.java @@ -30,6 +30,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.File; +import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -79,6 +80,13 @@ public abstract class Output<PROP_TYPE extends LogFeederProperties, INPUT_MARKER this.destination = destination; } + /** + * Get the list of fields that will be used for ID generation of log entries. + */ + public List<String> getIdFields() { + return new ArrayList<>(); + } + public boolean isClosed() { return isClosed; } diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/common/IdGeneratorHelper.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/common/IdGeneratorHelper.java new file mode 100644 index 0000000..7430573 --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/common/IdGeneratorHelper.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.ambari.logfeeder.common; + +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang3.StringUtils; + +import java.util.List; +import java.util.Map; +import java.util.UUID; + +/** + * Helper class to generete UUID (random or based on specific fields) + */ +public class IdGeneratorHelper { + + private IdGeneratorHelper() { + } + + /** + * Generate UUID based on fields (or just randomly) + * @param data object map which can contain the field key-value pairs + * @param idFields field names that used for generating uuid + * @return generated UUID string + */ + public static String generateUUID(Map<String, Object> data, List<String> idFields) { + String uuid = null; + if (CollectionUtils.isNotEmpty(idFields)) { + final StringBuilder sb = new StringBuilder(); + for (String idField : idFields) { + if (data.containsKey(idField)) { + sb.append(data.get(idField).toString()); + } + } + String concatId = sb.toString(); + if (StringUtils.isNotEmpty(concatId)) { + uuid = UUID.nameUUIDFromBytes(concatId.getBytes()).toString(); + } else { + uuid = UUID.randomUUID().toString(); + } + } else { + uuid = UUID.randomUUID().toString(); + } + return uuid; + } +} diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputManagerImpl.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputManagerImpl.java index 09951cc..390a770 100644 --- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputManagerImpl.java +++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputManagerImpl.java @@ -20,10 +20,10 @@ package org.apache.ambari.logfeeder.output; import com.google.common.annotations.VisibleForTesting; import com.google.common.hash.Hashing; +import org.apache.ambari.logfeeder.common.IdGeneratorHelper; import org.apache.ambari.logfeeder.common.LogFeederConstants; import org.apache.ambari.logfeeder.conf.LogFeederProps; import org.apache.ambari.logfeeder.loglevelfilter.LogLevelFilterHandler; -import org.apache.ambari.logfeeder.input.InputFile; import org.apache.ambari.logfeeder.plugin.common.MetricData; import org.apache.ambari.logfeeder.plugin.input.Input; import org.apache.ambari.logfeeder.plugin.input.InputMarker; @@ -42,7 +42,6 @@ import java.util.ArrayList; import java.util.Date; import java.util.List; import java.util.Map; -import java.util.UUID; public class OutputManagerImpl extends OutputManager { private static final Logger LOG = Logger.getLogger(OutputManagerImpl.class); @@ -127,9 +126,6 @@ public class OutputManagerImpl extends OutputManager { } jsonObj.put("seq_num", new Long(docCounter++)); - if (jsonObj.get("id") == null) { - jsonObj.put("id", UUID.randomUUID().toString()); - } if (jsonObj.get("event_count") == null) { jsonObj.put("event_count", new Integer(1)); } @@ -154,6 +150,9 @@ public class OutputManagerImpl extends OutputManager { List<? extends Output> outputList = input.getOutputList(); for (Output output : outputList) { try { + if (jsonObj.get("id") == null) { + jsonObj.put("id", IdGeneratorHelper.generateUUID(jsonObj, output.getIdFields())); + } output.write(jsonObj, inputMarker); } catch (Exception e) { LOG.error("Error writing. to " + output.getShortDescription(), e); diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputSolr.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputSolr.java index 6b27553..278a7f5 100644 --- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputSolr.java +++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputSolr.java @@ -19,6 +19,7 @@ package org.apache.ambari.logfeeder.output; +import org.apache.ambari.logfeeder.common.IdGeneratorHelper; import org.apache.ambari.logfeeder.common.LogFeederSolrClientFactory; import org.apache.ambari.logfeeder.conf.LogFeederProps; import org.apache.ambari.logfeeder.plugin.input.InputMarker; @@ -45,13 +46,10 @@ import java.io.IOException; import java.net.MalformedURLException; import java.util.ArrayList; import java.util.Calendar; -import java.util.Collection; -import java.util.Collections; +import java.util.Collection;; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Set; -import java.util.UUID; import java.util.concurrent.BlockingQueue; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.TimeUnit; @@ -85,6 +83,7 @@ public class OutputSolr extends Output<LogFeederProps, InputMarker> { private boolean implicitRouting = false; private int lastSlotByMin = -1; private boolean skipLogtime = false; + private List<String> idFields = new ArrayList<>(); private BlockingQueue<OutputData> outgoingBuffer = null; private List<SolrWorkerThread> workerThreadList = new ArrayList<>(); @@ -134,6 +133,8 @@ public class OutputSolr extends Output<LogFeederProps, InputMarker> { solrUrls = solrUrlsList.toArray(new String[0]); } + idFields = getListValue("id_fields", new ArrayList<>()); + skipLogtime = getBooleanValue("skip_logtime", DEFAULT_SKIP_LOGTIME); maxIntervalMS = getIntValue("idle_flush_time_ms", DEFAULT_MAX_INTERVAL_MS); @@ -409,7 +410,7 @@ public class OutputSolr extends Output<LogFeederProps, InputMarker> { outputData = outgoingBuffer.poll(nextDispatchDuration, TimeUnit.MILLISECONDS); } if (outputData != null && outputData.jsonObj.get("id") == null) { - outputData.jsonObj.put("id", UUID.randomUUID().toString()); + outputData.jsonObj.put("id", IdGeneratorHelper.generateUUID(outputData.jsonObj, idFields)); } return outputData; } @@ -501,4 +502,9 @@ public class OutputSolr extends Output<LogFeederProps, InputMarker> { public void copyFile(File inputFile, InputMarker inputMarker) throws UnsupportedOperationException { throw new UnsupportedOperationException("copyFile method is not yet supported for output=solr"); } + + @Override + public List<String> getIdFields() { + return idFields; + } } diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/common/IdGeneratorHelperTest.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/common/IdGeneratorHelperTest.java new file mode 100644 index 0000000..ae43ac0 --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/common/IdGeneratorHelperTest.java @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.ambari.logfeeder.common; + +import org.junit.Test; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +public class IdGeneratorHelperTest { + + @Test + public void testGenerateRandomUUID() { + // GIVEN + Map<String, Object> fieldKeyMap = new HashMap<>(); + List<String> fields = new ArrayList<>(); + // WHEN + String uuid1 = IdGeneratorHelper.generateUUID(fieldKeyMap, fields); + String uuid2 = IdGeneratorHelper.generateUUID(fieldKeyMap, fields); + // THEN + assertFalse(uuid1.equals(uuid2)); + } + + @Test + public void testUUIDFromFields() { + // GIVEN + Map<String, Object> fieldKeyMap1 = new HashMap<>(); + fieldKeyMap1.put("one-field", "1"); + Map<String, Object> fieldKeyMap2 = new HashMap<>(); + fieldKeyMap2.put("one-field", "1"); + List<String> fields = new ArrayList<>(); + fields.add("one-field"); + // WHEN + String uuid1 = IdGeneratorHelper.generateUUID(fieldKeyMap1, fields); + String uuid2 = IdGeneratorHelper.generateUUID(fieldKeyMap2, fields); + // THEN + assertTrue(uuid1.equals(uuid2)); + } + + @Test + public void testUUIDFromFieldsWithMultipleFields() { + // GIVEN + Map<String, Object> fieldKeyMap1 = new HashMap<>(); + fieldKeyMap1.put("one-field", "1"); + fieldKeyMap1.put("two-field", "2"); + Map<String, Object> fieldKeyMap2 = new HashMap<>(); + fieldKeyMap2.put("one-field", "1"); + fieldKeyMap2.put("two-field", "2"); + List<String> fields = new ArrayList<>(); + fields.add("one-field"); + fields.add("two-field"); + // WHEN + String uuid1 = IdGeneratorHelper.generateUUID(fieldKeyMap1, fields); + String uuid2 = IdGeneratorHelper.generateUUID(fieldKeyMap2, fields); + // THEN + assertTrue(uuid1.equals(uuid2)); + } + + @Test + public void testUUIDFromFieldsDifferentNumberOfFields() { + // GIVEN + Map<String, Object> fieldKeyMap1 = new HashMap<>(); + fieldKeyMap1.put("one-field", "1"); + Map<String, Object> fieldKeyMap2 = new HashMap<>(); + fieldKeyMap2.put("one-field", "1"); + fieldKeyMap2.put("two-field", "2"); + List<String> fields = new ArrayList<>(); + fields.add("one-field"); + fields.add("two-field"); + // WHEN + String uuid1 = IdGeneratorHelper.generateUUID(fieldKeyMap1, fields); + String uuid2 = IdGeneratorHelper.generateUUID(fieldKeyMap2, fields); + // THEN + assertFalse(uuid1.equals(uuid2)); + } + +}