This is an automated email from the ASF dual-hosted git repository.
sgoeschl pushed a commit to branch FREEMARKER-199
in repository https://gitbox.apache.org/repos/asf/freemarker-generator.git
The following commit(s) were added to refs/heads/FREEMARKER-199 by this push:
new 56b911e FREEMARKER-199 [freemarker-generator] Add "utah-parser-tool"
56b911e is described below
commit 56b911e13ed1d0ba2e09c562a76f30ffdad32901
Author: Siegfried Goeschl <[email protected]>
AuthorDate: Mon Oct 18 08:31:25 2021 +0200
FREEMARKER-199 [freemarker-generator] Add "utah-parser-tool"
---
.../csv/{transform.ftl => transform-streaming.ftl} | 19 ++++++++++--------
.../templates/utahparser/csv/transform.ftl | 11 ++++++-----
.../freemarker/generator/cli/ExamplesTest.java | 4 +++-
.../generator/tools/commonscsv/CommonsCSVTool.java | 5 -----
.../generator/tools/utahparser/UtahParserTool.java | 23 ++++++++++++++++++++++
5 files changed, 43 insertions(+), 19 deletions(-)
diff --git
a/freemarker-generator-cli/src/app/examples/templates/utahparser/csv/transform.ftl
b/freemarker-generator-cli/src/app/examples/templates/utahparser/csv/transform-streaming.ftl
similarity index 62%
copy from
freemarker-generator-cli/src/app/examples/templates/utahparser/csv/transform.ftl
copy to
freemarker-generator-cli/src/app/examples/templates/utahparser/csv/transform-streaming.ftl
index 5063619..ebb2829 100644
---
a/freemarker-generator-cli/src/app/examples/templates/utahparser/csv/transform.ftl
+++
b/freemarker-generator-cli/src/app/examples/templates/utahparser/csv/transform-streaming.ftl
@@ -17,15 +17,18 @@
<#-- Setup Utah-Parser -->
<#assign conf = tools.utahparser.getConfig(dataSources[0])>
<#assign parser = tools.utahparser.getParser(conf, dataSources[1])>
-<#assign records = parser.toList()>
-<#assign firstRecord = records[0]!{}>
-<#-- Setup CSVPrinter -->
-<#assign csvHeaders = tools.csv.extractHeaders(firstRecord)>
-<#assign cvsFormat =
tools.csv.formats.EXCEL.withHeader(csvHeaders).withDelimiter(';')>
-<#assign csvPrinter = tools.csv.printer(cvsFormat)>
+<#assign csvPrinter = "">
<#-- Print records as CSV -->
<#compress>
- <#list records as record>
+ <#list parser.iterator() as record>
+ <#if !csvPrinter?has_content>
+ <#-- Lazy creation of a CSVFormat for writing the records -->
+ <#assign defaultCsvformat =
tools.csv.formats[CSV_TARGET_FORMAT!"DEFAULT"]>
+ <#assign delimiter =
tools.csv.toDelimiter(CSV_TARGET_DELIMITER!defaultCsvformat.getDelimiter())>
+ <#assign csvHeaders = tools.utahparser.getHeaders(record)>
+ <#assign cvsFormat =
defaultCsvformat.withHeader(csvHeaders).withDelimiter(delimiter)>
+ <#assign csvPrinter = tools.csv.printer(cvsFormat)>
+ </#if>
${csvPrinter.printRecord(record, csvHeaders)}
</#list>
-</#compress>
\ No newline at end of file
+</#compress>
diff --git
a/freemarker-generator-cli/src/app/examples/templates/utahparser/csv/transform.ftl
b/freemarker-generator-cli/src/app/examples/templates/utahparser/csv/transform.ftl
index 5063619..f67729c 100644
---
a/freemarker-generator-cli/src/app/examples/templates/utahparser/csv/transform.ftl
+++
b/freemarker-generator-cli/src/app/examples/templates/utahparser/csv/transform.ftl
@@ -14,18 +14,19 @@
specific language governing permissions and limitations
under the License.
-->
-<#-- Setup Utah-Parser -->
+<#-- Setup Utah-Parser and parse all records to determine the headers -->
<#assign conf = tools.utahparser.getConfig(dataSources[0])>
<#assign parser = tools.utahparser.getParser(conf, dataSources[1])>
<#assign records = parser.toList()>
-<#assign firstRecord = records[0]!{}>
+<#assign headers = tools.utahparser.getHeaders(records)>
<#-- Setup CSVPrinter -->
-<#assign csvHeaders = tools.csv.extractHeaders(firstRecord)>
-<#assign cvsFormat =
tools.csv.formats.EXCEL.withHeader(csvHeaders).withDelimiter(';')>
+<#assign defaultCsvformat = tools.csv.formats[CSV_TARGET_FORMAT!"DEFAULT"]>
+<#assign csvDelimiter =
tools.csv.toDelimiter(CSV_TARGET_DELIMITER!defaultCsvformat.getDelimiter())>
+<#assign cvsFormat =
defaultCsvformat.withHeader(headers).withDelimiter(csvDelimiter)>
<#assign csvPrinter = tools.csv.printer(cvsFormat)>
<#-- Print records as CSV -->
<#compress>
<#list records as record>
- ${csvPrinter.printRecord(record, csvHeaders)}
+ ${csvPrinter.printRecord(record, headers)}
</#list>
</#compress>
\ No newline at end of file
diff --git
a/freemarker-generator-cli/src/test/java/org/apache/freemarker/generator/cli/ExamplesTest.java
b/freemarker-generator-cli/src/test/java/org/apache/freemarker/generator/cli/ExamplesTest.java
index 687925f..19eb427 100644
---
a/freemarker-generator-cli/src/test/java/org/apache/freemarker/generator/cli/ExamplesTest.java
+++
b/freemarker-generator-cli/src/test/java/org/apache/freemarker/generator/cli/ExamplesTest.java
@@ -127,7 +127,9 @@ public class ExamplesTest extends AbstractMainTest {
@Test
public void shouldRunUtahParserExamples() throws IOException {
- assertValid(execute("-t
src/app/examples/templates/utahparser/csv/transform.ftl " +
+ assertValid(execute("-PCSV_TARGET_FORMAT=EXCEL " +
+ "-PCSV_TARGET_DELIMITER=SEMICOLON " +
+ "-t src/app/examples/templates/utahparser/csv/transform.ftl " +
"src/app/examples/data/text/utahparser/juniper_bgp_summary_template.xml " +
"src/app/examples/data/text/utahparser/juniper_bgp_summary_example.txt"));
diff --git
a/freemarker-generator-tools/src/main/java/org/apache/freemarker/generator/tools/commonscsv/CommonsCSVTool.java
b/freemarker-generator-tools/src/main/java/org/apache/freemarker/generator/tools/commonscsv/CommonsCSVTool.java
index 519c171..18c8817 100644
---
a/freemarker-generator-tools/src/main/java/org/apache/freemarker/generator/tools/commonscsv/CommonsCSVTool.java
+++
b/freemarker-generator-tools/src/main/java/org/apache/freemarker/generator/tools/commonscsv/CommonsCSVTool.java
@@ -99,11 +99,6 @@ public class CommonsCSVTool {
return new CommonsCSVPrinterFacade(csvFormat);
}
- public String[] extractHeaders(Map<String, Object> map) {
- final Set<String> keySet = map.keySet();
- return keySet.toArray(new String[0]);
- }
-
/**
* Extract the list of unique values (keys) of the column "name".
*
diff --git
a/freemarker-generator-tools/src/main/java/org/apache/freemarker/generator/tools/utahparser/UtahParserTool.java
b/freemarker-generator-tools/src/main/java/org/apache/freemarker/generator/tools/utahparser/UtahParserTool.java
index 87ceb0d..0da3491 100644
---
a/freemarker-generator-tools/src/main/java/org/apache/freemarker/generator/tools/utahparser/UtahParserTool.java
+++
b/freemarker-generator-tools/src/main/java/org/apache/freemarker/generator/tools/utahparser/UtahParserTool.java
@@ -26,6 +26,12 @@ import
org.apache.freemarker.generator.tools.utahparser.impl.ParserWrapper;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeSet;
+import java.util.stream.Collectors;
public class UtahParserTool {
@@ -46,6 +52,23 @@ public class UtahParserTool {
return new ParserWrapper(Parser.parse(config, is));
}
+ public String[] getHeaders(Map<String, Object> map) {
+ final Set<String> keySet = map.keySet();
+ return keySet.toArray(new String[0]);
+ }
+
+ public Set<String> getHeaders(Collection<Map<String, String>> records) {
+ if (records == null || records.isEmpty()) {
+ return Collections.emptySet();
+ }
+
+ return records.stream()
+ .map(record -> record.keySet())
+ .flatMap(Collection::stream)
+ .collect(Collectors.toCollection(() -> new TreeSet<>())
+ );
+ }
+
public String toString() {
return "Parse semi-structured text using regular expressions (see
https://github.com/sonalake/utah-parser)";
}
