This is an automated email from the ASF dual-hosted git repository.
sgoeschl pushed a commit to branch FREEMARKER-181
in repository https://gitbox.apache.org/repos/asf/freemarker-generator.git
The following commit(s) were added to refs/heads/FREEMARKER-181 by this push:
new 3b4077b FREEMARKER-181 Support custom pattern definitions for Grok
tool
3b4077b is described below
commit 3b4077bf8bd074d9e705b1966dcb2bf6862af1b5
Author: Siegfried Goeschl <[email protected]>
AuthorDate: Mon May 24 12:54:08 2021 +0200
FREEMARKER-181 Support custom pattern definitions for Grok tool
---
.../templates/accesslog/combined-access.ftl | 4 +-
.../templates/logs/csv/serverlog-to-csv.ftl | 4 +-
.../site/markdown/cli/usage/parsing-with-grok.md | 8 +--
.../freemarker/generator/cli/ExamplesTest.java | 1 +
.../freemarker/generator/tools/grok/GrokTool.java | 57 +++++++++-------------
.../generator/tools/grok/impl/GrokWrapper.java | 45 -----------------
.../generator/tools/grok/GrokToolTest.java | 12 +++--
pom.xml | 1 +
8 files changed, 40 insertions(+), 92 deletions(-)
diff --git
a/freemarker-generator-cli/src/app/examples/templates/accesslog/combined-access.ftl
b/freemarker-generator-cli/src/app/examples/templates/accesslog/combined-access.ftl
index af78768..944a826 100644
---
a/freemarker-generator-cli/src/app/examples/templates/accesslog/combined-access.ftl
+++
b/freemarker-generator-cli/src/app/examples/templates/accesslog/combined-access.ftl
@@ -15,14 +15,14 @@
specific language governing permissions and limitations
under the License.
-->
-<#assign grok = tools.grok.compile("%{COMBINEDAPACHELOG}")>
+<#assign grok = tools.grok.create("%{COMBINEDAPACHELOG}")>
<#assign dataSource = dataSources?values[0]>
<#assign lines = dataSource.getLineIterator()>
<#compress>
TIMESTAMP;VERB;REQUEST;HTTPVERSION
<#list lines as line>
- <#assign parts = grok.match(line)>
+ <#assign parts = grok.match(line).capture()>
<#assign timestamp = parts.timestamp>
<#assign verb = parts.verb>
<#assign request = parts.request>
diff --git
a/freemarker-generator-cli/src/app/examples/templates/logs/csv/serverlog-to-csv.ftl
b/freemarker-generator-cli/src/app/examples/templates/logs/csv/serverlog-to-csv.ftl
index bccad9a..544975f 100644
---
a/freemarker-generator-cli/src/app/examples/templates/logs/csv/serverlog-to-csv.ftl
+++
b/freemarker-generator-cli/src/app/examples/templates/logs/csv/serverlog-to-csv.ftl
@@ -29,7 +29,7 @@
}>
<#-- Instantiante the grok tool -->
-<#assign grok = tools.grok.compile("%{MY_SERVERLOG}", patternDefinitions)>
+<#assign grok = tools.grok.create("%{MY_SERVERLOG}", patternDefinitions)>
<#-- Iterate over all data sources and convert matching lines to CSV output -->
<#compress>
@@ -37,7 +37,7 @@
<#if dataSources?has_content>
<#list dataSources?values as dataSource>
<#list dataSource.getLineIterator() as line>
- <#assign parts = grok.match(line)>
+ <#assign parts = grok.match(line).capture()>
<#if parts?has_content>
<#-- Skip all response times less than 5 ms because these
are boring pings -->
<#if parts.response_time?number gt 5>
diff --git
a/freemarker-generator-cli/src/site/markdown/cli/usage/parsing-with-grok.md
b/freemarker-generator-cli/src/site/markdown/cli/usage/parsing-with-grok.md
index 7e974b5..5cbadda 100644
--- a/freemarker-generator-cli/src/site/markdown/cli/usage/parsing-with-grok.md
+++ b/freemarker-generator-cli/src/site/markdown/cli/usage/parsing-with-grok.md
@@ -32,14 +32,14 @@ using the following FreeMarker template
```text
<#ftl output_format="plainText" strip_whitespace=true>
-<#assign grok = tools.grok.compile("%{COMBINEDAPACHELOG}")>
+<#assign grok = tools.grok.create("%{COMBINEDAPACHELOG}")>
<#assign dataSource = dataSources?values[0]>
<#assign lines = dataSource.getLineIterator()>
<#compress>
TIMESTAMP;VERB;REQUEST;HTTPVERSION
<#list lines as line>
- <#assign parts = grok.match(line)>
+ <#assign parts = grok.match(line).capture()>
<#assign timestamp = parts["timestamp"]>
<#assign verb = parts["verb"]>
<#assign request = parts["request"]>
@@ -87,7 +87,7 @@ In technical terms the FTL
}>
<#-- Instantiante the grok tool -->
-<#assign grok = tools.grok.compile("%{MY_SERVERLOG}", patternDefinitions)>
+<#assign grok = tools.grok.create("%{MY_SERVERLOG}", patternDefinitions)>
<#-- Iterate over all data sources and convert matching lines to CSV output -->
<#compress>
@@ -95,7 +95,7 @@ In technical terms the FTL
<#if dataSources?has_content>
<#list dataSources?values as dataSource>
<#list dataSource.getLineIterator() as line>
- <#assign parts = grok.match(line)>
+ <#assign parts = grok.match(line).capture()>
<#if parts?has_content>
<#-- Skip all response times less than 5 ms because these
are boring pings -->
<#if parts.response_time?number gt 5>
diff --git
a/freemarker-generator-cli/src/test/java/org/apache/freemarker/generator/cli/ExamplesTest.java
b/freemarker-generator-cli/src/test/java/org/apache/freemarker/generator/cli/ExamplesTest.java
index 88067b2..49e536f 100644
---
a/freemarker-generator-cli/src/test/java/org/apache/freemarker/generator/cli/ExamplesTest.java
+++
b/freemarker-generator-cli/src/test/java/org/apache/freemarker/generator/cli/ExamplesTest.java
@@ -105,6 +105,7 @@ public class ExamplesTest extends AbstractMainTest {
@Test
public void shouldRunGrokExamples() throws IOException {
assertValid(execute("-t
src/app/examples/templates/accesslog/combined-access.ftl
src/app/examples/data/accesslog/combined-access.log"));
+ assertValid(execute("-t
src/app/examples/templates/logs/csv/serverlog-to-csv.ftl
src/app/examples/data/logs/server.01.log"));
}
@Test
diff --git
a/freemarker-generator-tools/src/main/java/org/apache/freemarker/generator/tools/grok/GrokTool.java
b/freemarker-generator-tools/src/main/java/org/apache/freemarker/generator/tools/grok/GrokTool.java
index 207c54d..5f3a485 100644
---
a/freemarker-generator-tools/src/main/java/org/apache/freemarker/generator/tools/grok/GrokTool.java
+++
b/freemarker-generator-tools/src/main/java/org/apache/freemarker/generator/tools/grok/GrokTool.java
@@ -18,8 +18,9 @@ package org.apache.freemarker.generator.tools.grok;
import io.krakens.grok.api.Grok;
import io.krakens.grok.api.GrokCompiler;
-import org.apache.freemarker.generator.tools.grok.impl.GrokWrapper;
+import org.apache.freemarker.generator.base.util.Validate;
+import java.util.HashMap;
import java.util.Map;
public class GrokTool {
@@ -27,54 +28,42 @@ public class GrokTool {
private static final String DEFAULT_PATTERN_FILE = "/patterns/patterns";
/**
- * Compile the Grok pattern.
+ * Create a default Grok instance using the the default pattern files
loaded
+ * from the classpath.
*
* @param pattern Grok pattern to compile
- * @return Grok wrapper
+ * @return Grok object
*/
- public GrokWrapper compile(String pattern) {
- return compile(DEFAULT_PATTERN_FILE, pattern);
+ public Grok create(String pattern) {
+ return create(pattern, new HashMap<>());
}
/**
- * Compile the Grok pattern.
+ * Get a default Grok instance using the the default pattern files loaded
+ * from the classpath.
*
* @param pattern Grok pattern to compile
* @param patternDefinitions custom patterns to be registered
- * @return Grok wrapper
+ * @return Grok object
*/
- public GrokWrapper compile(String pattern, Map<String, String>
patternDefinitions) {
- return compile(DEFAULT_PATTERN_FILE, pattern, patternDefinitions);
+ public Grok create(String pattern, Map<String, String> patternDefinitions)
{
+ Validate.notEmpty(pattern, "Grok pattern to compile is empty");
+ final GrokCompiler grokCompiler = grokCompiler();
+ grokCompiler.registerPatternFromClasspath(DEFAULT_PATTERN_FILE);
+ if (patternDefinitions != null) {
+ grokCompiler.register(patternDefinitions);
+ }
+ return grokCompiler.compile(pattern);
}
/**
- * Compile the Grok pattern.
+ * Create a new Grok compiler instance. This is just
+ * a convinience method if the caller requires full control.
*
- * @param path classpath file for default patterns to register
- * @param pattern Grok pattern to compile
- * @return Grok wrapper
- */
- public GrokWrapper compile(String path, String pattern) {
- final GrokCompiler grokCompiler = GrokCompiler.newInstance();
- grokCompiler.registerPatternFromClasspath(path);
- final Grok grok = grokCompiler.compile(pattern);
- return new GrokWrapper(grok);
- }
-
- /**
- * Compile the Grok pattern.
- *
- * @param path classpath file for default patterns to
register
- * @param pattern Grok pattern to compile
- * @param patternDefinitions custom patterns to be registered
- * @return Grok wrapper
+ * @return Grok compiler
*/
- public GrokWrapper compile(String path, String pattern, Map<String,
String> patternDefinitions) {
- final GrokCompiler grokCompiler = GrokCompiler.newInstance();
- grokCompiler.registerPatternFromClasspath(path);
- grokCompiler.register(patternDefinitions);
- final Grok grok = grokCompiler.compile(pattern);
- return new GrokWrapper(grok);
+ public GrokCompiler grokCompiler() {
+ return GrokCompiler.newInstance();
}
@Override
diff --git
a/freemarker-generator-tools/src/main/java/org/apache/freemarker/generator/tools/grok/impl/GrokWrapper.java
b/freemarker-generator-tools/src/main/java/org/apache/freemarker/generator/tools/grok/impl/GrokWrapper.java
deleted file mode 100644
index 0589469..0000000
---
a/freemarker-generator-tools/src/main/java/org/apache/freemarker/generator/tools/grok/impl/GrokWrapper.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.freemarker.generator.tools.grok.impl;
-
-import io.krakens.grok.api.Grok;
-import org.apache.freemarker.generator.base.util.StringUtils;
-
-import java.util.Collections;
-import java.util.Map;
-
-import static java.util.Objects.requireNonNull;
-
-/**
- * Convenience wrapper around Grok instance.
- */
-public class GrokWrapper {
-
- private final Grok grok;
-
- public GrokWrapper(Grok grok) {
- this.grok = requireNonNull(grok);
- }
-
- public Map<String, Object> match(String line) {
- if (StringUtils.isEmpty(line)) {
- return Collections.emptyMap();
- }
-
- return grok.match(line).capture();
- }
-}
diff --git
a/freemarker-generator-tools/src/test/java/org/apache/freemarker/generator/tools/grok/GrokToolTest.java
b/freemarker-generator-tools/src/test/java/org/apache/freemarker/generator/tools/grok/GrokToolTest.java
index 816654b..ef24bf7 100644
---
a/freemarker-generator-tools/src/test/java/org/apache/freemarker/generator/tools/grok/GrokToolTest.java
+++
b/freemarker-generator-tools/src/test/java/org/apache/freemarker/generator/tools/grok/GrokToolTest.java
@@ -16,7 +16,8 @@
*/
package org.apache.freemarker.generator.tools.grok;
-import org.apache.freemarker.generator.tools.grok.impl.GrokWrapper;
+import io.krakens.grok.api.Grok;
+import io.krakens.grok.api.Match;
import org.junit.Test;
import java.util.Map;
@@ -30,9 +31,10 @@ public class GrokToolTest {
@Test
public void shallParseCombinedAccessLog() {
final String line = "112.169.19.192 - - [06/Mar/2013:01:36:30 +0900]
\"GET / HTTP/1.1\" 200 44346 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X
10_8_2) AppleWebKit/537.22 (KHTML, like Gecko) Chrome/25.0.1364.152
Safari/537.22\"";
- final GrokWrapper grok = grokTool().compile("%{COMBINEDAPACHELOG}");
+ final Grok grok = grokTool().create("%{COMBINEDAPACHELOG}");
- final Map<String, Object> map = grok.match(line);
+ final Match match = grok.match(line);
+ final Map<String, Object> map = match.capture();
assertEquals("GET", map.get("verb"));
assertEquals("06/Mar/2013:01:36:30 +0900", map.get("timestamp"));
@@ -52,9 +54,9 @@ public class GrokToolTest {
{ "MY_SERVERLOG", "%{MY_TIMESTAMP}
%{LOGLEVEL}%{SPACE:UNWANTED}%{MY_MODULE} %{MY_THREAD} message response handled
in: %{INT:response_time} ms; %{GREEDYDATA:UNWANTED}" },
}).collect(Collectors.toMap(data -> data[0], data -> data[1]));
- final GrokWrapper grok = grokTool().compile("%{MY_SERVERLOG}",
patternDefinitions);
+ final Grok grok = grokTool().create("%{MY_SERVERLOG}",
patternDefinitions);
- final Map<String, Object> map = grok.match(line);
+ final Map<String, Object> map = grok.match(line).capture();
assertEquals(16, map.size());
assertEquals("2019-05-17", map.get("date"));
diff --git a/pom.xml b/pom.xml
index e0480b5..8a38d18 100644
--- a/pom.xml
+++ b/pom.xml
@@ -149,6 +149,7 @@
</dependencies>
</dependencyManagement>
<build>
+ <defaultGoal>clean install</defaultGoal>
<pluginManagement>
<plugins>
<plugin>
