[
https://issues.apache.org/jira/browse/METRON-694?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15864133#comment-15864133
]
ASF GitHub Bot commented on METRON-694:
---------------------------------------
Github user JonZeolla commented on a diff in the pull request:
https://github.com/apache/incubator-metron/pull/453#discussion_r100859285
--- Diff:
metron-platform/metron-common/src/main/java/org/apache/metron/common/error/MetronError.java
---
@@ -0,0 +1,200 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.common.error;
+
+import org.apache.commons.codec.digest.DigestUtils;
+import org.apache.commons.lang.exception.ExceptionUtils;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.metron.common.Constants;
+import org.apache.metron.common.Constants.ErrorType;
+import org.json.simple.JSONObject;
+
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Optional;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.apache.metron.common.Constants.ErrorFields;
+
+public class MetronError {
+
+ private String message;
+ private Throwable throwable;
+ private String sensorType = "error";
+ private ErrorType errorType = ErrorType.DEFAULT_ERROR;
+ private Set<String> errorFields;
+ private List<Object> rawMessages;
+
+ public MetronError withMessage(String message) {
+ this.message = message;
+ return this;
+ }
+
+ public MetronError withThrowable(Throwable throwable) {
+ this.throwable = throwable;
+ return this;
+ }
+
+ public MetronError withSensorType(String sensorType) {
+ this.sensorType = sensorType;
+ return this;
+ }
+
+ public MetronError withErrorType(ErrorType errorType) {
+ this.errorType = errorType;
+ return this;
+ }
+
+ public MetronError withErrorFields(Set<String> errorFields) {
+ this.errorFields = errorFields;
+ return this;
+ }
+
+
+ public MetronError addRawMessage(Object rawMessage) {
+ if (rawMessage != null) {
+ if (this.rawMessages == null) {
+ this.rawMessages = new ArrayList<>();
+ }
+ this.rawMessages.add(rawMessage);
+ }
+ return this;
+ }
+
+ public MetronError withRawMessages(List<Object> rawMessages) {
+ this.rawMessages = rawMessages;
+ return this;
+ }
+
+ public Optional<Throwable> getThrowable() {
+ return throwable != null ? Optional.of(throwable) : Optional.empty();
+ }
+
+ @SuppressWarnings({"unchecked"})
+ public JSONObject getJSONObject() {
+ JSONObject errorMessage = new JSONObject();
+ errorMessage.put(Constants.SENSOR_TYPE, "error");
+
+ /*
+ * Save full stack trace in object.
+ */
+ if (throwable != null) {
+ String stackTrace = ExceptionUtils.getStackTrace(throwable);
+ String exception = throwable.toString();
+ errorMessage.put(ErrorFields.EXCEPTION.getName(), exception);
+ errorMessage.put(ErrorFields.STACK.getName(), stackTrace);
+ }
+
+ errorMessage.put(ErrorFields.TIMESTAMP.getName(),
System.currentTimeMillis());
+ try {
+ errorMessage.put(ErrorFields.HOSTNAME.getName(),
InetAddress.getLocalHost().getHostName());
+ } catch (UnknownHostException ex) {
+
+ }
+ if(rawMessages != null) {
+ for(int i = 0; i < rawMessages.size(); i++) {
+ Object rawMessage = rawMessages.get(i);
+ // If multiple messages are included add an index to the field
name, otherwise leave it off
+ String rawMessageField = rawMessages.size() == 1 ?
ErrorFields.RAW_MESSAGE.getName() : ErrorFields.RAW_MESSAGE.getName() + "_" + i;
+ String rawMessageBytesField = rawMessages.size() == 1 ?
ErrorFields.RAW_MESSAGE_BYTES.getName() :
ErrorFields.RAW_MESSAGE_BYTES.getName() + "_" + i;
+ if(rawMessage instanceof byte[]) {
+ errorMessage.put(rawMessageField,
Bytes.toString((byte[])rawMessage));
+ errorMessage.put(rawMessageBytesField,
toByteArrayList((byte[])rawMessage));
+ } else {
+ errorMessage.put(rawMessageField, rawMessage);
+ }
+ }
+ }
+
+ if (rawMessages != null && rawMessages.size() == 1) {
+ Object rawMessage = rawMessages.get(0);
+ if (rawMessage instanceof JSONObject) {
+ JSONObject rawJSON = (JSONObject) rawMessage;
+ if (errorFields != null) {
+ String errorFieldString = String.join(",", errorFields);
+ errorMessage.put(ErrorFields.ERROR_FIELDS.getName(),
errorFieldString);
+ List<String> hashElements = errorFields.stream().map(errorField
->
+ String.format("%s-%s", errorField,
rawJSON.get(errorField))).collect(Collectors.toList());
+ errorMessage.put(ErrorFields.ERROR_HASH.getName(),
DigestUtils.sha256Hex(String.join("|", hashElements).getBytes(UTF_8)));
--- End diff --
Hashing was suggested to provide a unique identifier of the original
information, and thus giving you something to key in on and create a count of
unique events/prioritize more prevalent issues without the concern of cyclical
issues (for instance, if the issue is with indexing a specific message, and you
try to index it again, it will just fail in a loop).
> Index Errors from Topologies
> ----------------------------
>
> Key: METRON-694
> URL: https://issues.apache.org/jira/browse/METRON-694
> Project: Metron
> Issue Type: Bug
> Reporter: Ryan Merriman
>
> Need to make sure (and review) that all the bolts write into the error queue.
> Errors should then be consumed from the error queue and indexed.
--
This message was sent by Atlassian JIRA
(v6.3.15#6346)
