[
https://issues.apache.org/jira/browse/METRON-295?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15651390#comment-15651390
]
ASF GitHub Bot commented on METRON-295:
---------------------------------------
Github user kylerichardson commented on a diff in the pull request:
https://github.com/apache/incubator-metron/pull/338#discussion_r87228757
--- Diff:
metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/ScriptParser.java
---
@@ -0,0 +1,189 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.metron.parsers;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.Serializable;
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.List;
+import java.util.Map;
+
+import javax.script.Invocable;
+import javax.script.ScriptEngine;
+import javax.script.ScriptEngineFactory;
+import javax.script.ScriptEngineManager;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.metron.common.Constants;
+import org.apache.metron.parsers.interfaces.MessageParser;
+import org.json.simple.JSONObject;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.base.Joiner;
+import com.google.common.base.Splitter;
+
+public class ScriptParser implements
MessageParser<JSONObject>,Serializable{
+
+ protected static final Logger LOG =
LoggerFactory.getLogger(ScriptParser.class);
+ protected String scriptPath;
+ protected ScriptEngine engine;
+ protected String parseFunction;
+ protected String language;
+ protected String commonScript="/scripts/";
+ protected List<String> timeFields = new ArrayList<>();
+ protected String timestampField;
+ protected SimpleDateFormat dateFormat = new
SimpleDateFormat("yyyy-MM-dd HH:mm:ss.S z");
+
+ @Override
+ public void configure(Map<String, Object> config) {
+ this.scriptPath=(String) config.get("path");
+ this.parseFunction=(String)config.get("function");
+ this.language=(String)config.get("language");
+ this.commonScript=this.commonScript+language+"/common";
+ if(this.parseFunction==null)
+ this.parseFunction="parse";
+ }
+ //Should this be sent to the interface as a default method?
+ public InputStream openInputStream(String streamName) throws
IOException {
+ FileSystem fs = FileSystem.get(new Configuration());
+ Path path = new Path(streamName);
+ if(fs.exists(path)) {
+ return fs.open(path);
+ } else {
+ return getClass().getResourceAsStream(streamName);
+ }
+ }
+
+ @Override
+ public void init() {
+ engine = new
ScriptEngineManager().getEngineByName(this.language);
+ try{
+ InputStream commonStream =
openInputStream(this.commonScript);
+ if (commonStream == null) {
+ throw new RuntimeException(
+ "Unable to initialize "+this.language+" Parser:
Unable to load " + this.commonScript + " from either classpath or HDFS");
+ }
+
+ engine.eval(new InputStreamReader(commonStream));
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Loading parser-specific functions from: " +
this.scriptPath);
+ }
+
+ InputStream patterInputStream =
openInputStream(this.scriptPath);
+ if (patterInputStream == null) {
+ throw new RuntimeException("Script parser unable to
initialize "+this.language+" parser: Unable to load " + this.scriptPath
+ + " from either classpath or HDFS");
+ }
+ engine.eval(new InputStreamReader(patterInputStream));
+ }catch(Throwable e){
+ LOG.error(e.getMessage(), e);
+ throw new RuntimeException(this.language+" Script parser
Error: " + e.getMessage(), e);
+ }
+
+ Invocable invocable = (Invocable) engine;
+
+ }
+
+ @Override
+ public List<JSONObject> parse(byte[] rawMessage) {
+ List<JSONObject> messages = new ArrayList<JSONObject>();
+ if(engine==null)
+ init();
+ String originalMessage = null;
+ try{
+ originalMessage = new String(rawMessage, "UTF-8");
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Script parser parsing message: " +
originalMessage);
+ }
+ Invocable invocable = (Invocable) engine;
+
+ Object result =
invocable.invokeFunction(this.parseFunction,originalMessage);
+ JSONObject message = new JSONObject((Map)result);
+
+ if (message.size() == 0)
+ throw new RuntimeException("Script produced a null
message. Original message was: "
+ + originalMessage + " and the parsed message
was: " + message + " . Check the function at: "
+ + this.scriptPath);
+
+ message.put("original_string", originalMessage);
--- End diff --
To move toward consistency, can this "original_string" field be updated to
use the new `Constants.Fields.ORIGINAL.getName()`?
> Script parsing bolt
> --------------------
>
> Key: METRON-295
> URL: https://issues.apache.org/jira/browse/METRON-295
> Project: Metron
> Issue Type: New Feature
> Affects Versions: 0.2.1BETA
> Reporter: James Sirota
> Assignee: Karthik Narayanan
> Priority: Minor
> Labels: newbie, platform
> Fix For: 0.2.2BETA
>
>
> In addition to having a Grok parsing bolt we need a bolt that can execute a
> script in order to parse a telemetry. This way you can still script the
> parsing for telemetries for which Grok expressions are too complex, but still
> don't have to define a java parser
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
