This is an automated email from the ASF dual-hosted git repository. ddekany pushed a commit to branch FREEMARKER-154 in repository https://gitbox.apache.org/repos/asf/freemarker-generator.git
commit 56bc016d7cd375cabfd1e9b7cea0be7fb8e245ef Merge: 7a6e620 20d7f13 Author: ddekany <[email protected]> AuthorDate: Mon Oct 25 00:42:45 2021 +0200 Merge remote-tracking branch 'origin/master' into FREEMARKER-154. Also updated website module and DocBook content to be compatible with it. freemarker-generator-base/pom.xml | 2 +- .../generator/base/FreeMarkerConstants.java | 9 ++ .../base/activation/CachingUrlDataSource.java | 1 + .../generator/base/datasource/DataSource.java | 138 ++++++++++++++++++--- .../base/datasource/DataSourceFactory.java | 18 ++- .../base/datasource/DataSourceLoader.java | 2 +- .../base/datasource/DataSourceLoaderFactory.java | 21 ++-- .../generator/base/datasource/DataSources.java | 118 ++++++++++++++---- .../base/datasource/DataSourcesSupplier.java | 48 +++++-- .../generator/base/file/RecursiveFileSupplier.java | 24 ++-- .../freemarker/generator/base/mime/Mimetypes.java | 3 + .../base/mime/MimetypesFileTypeMapFactory.java | 6 +- .../generator/base/output/OutputGenerator.java | 20 ++- .../freemarker/generator/base/table/Table.java | 2 +- .../freemarker/generator/base/util/FileUtils.java | 48 +++++++ .../freemarker/generator/base/util/ListUtils.java | 19 ++- .../base/util/NonClosableWriterWrapper.java | 2 +- .../generator/base/util/OperatingSystem.java | 2 +- .../generator/base/util/PropertiesTransformer.java | 3 +- .../generator/base/util/StringUtils.java | 8 ++ .../generator/datasource/DataSourceTest.java | 18 ++- .../generator/datasource/DataSourcesTest.java | 81 ++++++++---- .../freemarker/generator/util/FileUtilsTest.java | 74 +++++++++++ freemarker-generator-cli/CHANGELOG.md | 16 ++- freemarker-generator-cli/pom.xml | 10 +- .../src/app/config/freemarker-generator.properties | 1 + .../src/app/examples/data/ftl/nginx/nginx.conf.ftl | 2 +- .../src/app/examples/data/nginx/test1-api.nginx | 30 +++++ .../examples/data/nginx/test1-application.nginx | 27 ++++ .../utahparser/juniper_bgp_summary_example.txt | 12 ++ .../utahparser/juniper_bgp_summary_template.xml | 83 +++++++++++++ .../templates/accesslog/combined-access.ftl | 2 +- .../templates/csv/csv/gatling-user-credentials.ftl | 2 +- .../app/examples/templates/csv/fo/transactions.ftl | 2 +- .../app/examples/templates/csv/fo/transform.ftl | 2 +- .../examples/templates/csv/html/transactions.ftl | 2 +- .../src/app/examples/templates/csv/md/filter.ftl | 2 +- .../src/app/examples/templates/csv/shell/curl.ftl | 2 +- .../app/examples/templates/dataframe/example.ftl | 2 +- .../examples/templates/dataframe/html/print.ftl | 2 +- .../src/app/examples/templates/datasources.ftl | 135 ++++++++++++-------- .../src/app/examples/templates/demo.ftl | 10 +- .../app/examples/templates/excel/csv/custom.ftl | 2 +- .../templates/excel/dataframe/transform.ftl | 2 +- .../examples/templates/html/csv/dependencies.ftl | 2 +- .../app/examples/templates/html/txt/licence.ftl | 2 +- .../examples/templates/javafaker/csv/testdata.ftl | 2 +- .../templates/json/csv/swagger-endpoints.ftl | 2 +- .../templates/json/dataframe/github-users.ftl | 2 +- .../examples/templates/json/md/github-users.ftl | 2 +- .../templates/logs/csv/serverlog-to-csv.ftl | 2 +- .../nginx/confluence/nginx-config-parser.ftl | 34 +++++ .../templates/properties/csv/locker-test-users.ftl | 2 +- .../app/examples/templates/tsv/fo/transactions.ftl | 2 +- .../csv/transform.ftl} | 26 ++-- .../templates/utahparser}/json/transform.ftl | 6 +- .../app/examples/templates/xml/txt/recipients.ftl | 2 +- .../app/examples/templates/yaml/txt/transform.ftl | 2 +- .../src/app/scripts/run-examples.bat | 42 ++++++- .../src/app/scripts/run-examples.sh | 42 ++++++- .../src/app/templates/freemarker-generator/cat.ftl | 2 +- .../csv/{md => confluence}/transform.ftl | 7 +- .../freemarker-generator/csv/csv/transform.ftl | 2 +- .../freemarker-generator/csv/html/transform.ftl | 2 +- .../freemarker-generator/csv/md/transform.ftl | 2 +- .../freemarker-generator/excel/csv/transform.ftl | 2 +- .../freemarker-generator/excel/html/transform.ftl | 2 +- .../freemarker-generator/excel/md/transform.ftl | 2 +- .../app/templates/freemarker-generator/info.ftl | 50 ++++++-- .../freemarker-generator/json/yaml/transform.ftl | 2 +- .../freemarker-generator/yaml/json/transform.ftl | 2 +- .../org/apache/freemarker/generator/cli/Main.java | 19 +-- .../cli/config/ConfigurationSupplier.java | 5 + .../generator/cli/config/DataModelSupplier.java | 12 +- .../cli/config/OutputGeneratorsSupplier.java | 111 +++-------------- .../freemarker/generator/cli/config/Settings.java | 43 ++++--- .../freemarker/generator/cli/config/Suppliers.java | 4 +- .../cli/config/output/AbstractOutputGenerator.java | 80 ++++++++++++ .../output/DataSourceSeedingOutputGenerator.java | 112 +++++++++++++++++ .../output/DataSourceSeedingOutputMapper.java | 58 +++++++++ .../TemplateSeedingOutputGenerator.java} | 75 +++-------- .../cli/config/output}/TemplateTransformation.java | 5 +- .../output}/TemplateTransformationsBuilder.java | 4 +- .../generator/cli/picocli/DataModelDefinition.java | 7 ++ .../cli/picocli/DataSourceDefinition.java | 7 ++ .../generator/cli/picocli/GitVersionProvider.java | 9 ++ .../cli/picocli/OutputGeneratorDefinition.java | 35 ++++++ ...Definition.java => OutputMapperDefinition.java} | 15 +-- ...ceDefinition.java => OutputSeedDefinition.java} | 17 ++- .../cli/picocli/TemplateOutputDefinition.java | 7 ++ .../cli/picocli/TemplateSourceDefinition.java | 8 ++ .../picocli/TemplateSourceFilterDefinition.java | 12 +- .../generator/cli/task/FreeMarkerTask.java | 41 ++++-- .../generator/cli/util/TemplateSourceFactory.java | 50 ++++++++ .../generator/cli/wrapper/DataSourcesAdapter.java | 115 +++++++++++++++++ .../wrapper/FreeMarkerGeneratorObjectWrapper.java | 31 +++-- .../src/site/markdown/cli/concepts/data-models.md | 2 +- .../src/site/markdown/cli/concepts/data-sources.md | 54 ++++---- .../src/site/markdown/cli/concepts/named-uris.md | 12 +- .../site/markdown/cli/concepts/transformation.md | 44 +++++-- .../site/markdown/cli/usage/parsing-with-grok.md | 4 +- .../site/markdown/cli/usage/running-examples.md | 98 ++++++--------- .../site/markdown/cli/usage/using-dataframes.md | 6 +- .../src/test/data}/template/application.properties | 0 .../src/test/data}/template/nginx/nginx.conf.ftl | 0 .../freemarker/generator/cli/ExamplesTest.java | 47 ++++++- .../freemarker/generator/cli/ManualTest.java | 4 +- .../freemarker/generator/cli/PicocliTest.java | 2 +- .../cli/config/AbstractOutputGeneratorTest.java | 79 ++++++++++++ .../cli/config/ConfigurationSupplierTest.java | 5 +- .../generator/cli/config/SuppliersTest.java | 4 +- .../config/TemplateSeedingOutputGeneratorTest.java | 121 ++++++++++++++++++ .../DataSourceSeedingOutputGeneratorTest.java | 135 ++++++++++++++++++++ .../output/DataSourceSeedingOutputMapperTest.java | 95 ++++++++++++++ .../TemplateTransformationsBuilderTest.java | 9 +- .../src/test/templates/echo.ftl | 2 +- .../src/test/templates/manual.ftl | 8 +- .../src/test/templates/tools/csv.ftl | 2 +- freemarker-generator-maven-plugin-sample/pom.xml | 2 +- freemarker-generator-maven-plugin/pom.xml | 2 +- freemarker-generator-tools/pom.xml | 8 +- .../generator/tools/commonscsv/CommonsCSVTool.java | 6 +- .../commonscsv/impl/CommonsCSVPrinterFacade.java | 12 ++ .../tools/dataframe/impl/CSVConverter.java | 2 +- .../generator/tools/excel/ExcelTool.java | 2 +- .../freemarker/generator/tools/grok/GrokTool.java | 4 +- .../generator/tools/system/SystemTool.java | 29 ++++- .../generator/tools/utahparser/UtahParserTool.java | 100 +++++++++++++++ .../tools/utahparser/impl/ParserWrapper.java | 86 +++++++++++++ .../utahparser/juniper_bgp_summary_example.txt | 12 ++ .../utahparser/juniper_bgp_summary_template.xml | 69 +++++++++++ .../tools/commonsexec/CommonsExecToolTest.java | 26 ++++ .../generator/tools/system/SystemToolTest.java | 8 +- .../tools/utahparser/UtahParserToolTest.java | 103 +++++++++++++++ freemarker-generator-website/pom.xml | 2 +- .../src/main/docgen/book.xml | 54 ++++---- .../licence.ftl => licences/LICENCE_utahparser.txt | 90 +------------- pom.xml | 10 +- 138 files changed, 2753 insertions(+), 751 deletions(-) diff --cc freemarker-generator-website/pom.xml index a5cc6ff,0000000..f8424c5 mode 100644,000000..100644 --- a/freemarker-generator-website/pom.xml +++ b/freemarker-generator-website/pom.xml @@@ -1,101 -1,0 +1,101 @@@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd";> + + <modelVersion>4.0.0</modelVersion> + + <parent> + <groupId>org.apache.freemarker.generator</groupId> + <artifactId>freemarker-generator</artifactId> - <version>0.1.0-SNAPSHOT</version> ++ <version>0.2.0-SNAPSHOT</version> + </parent> + + <artifactId>freemarker-generator-website</artifactId> + <name>Website</name> + <description>Apache FreeMarker Generator Website</description> + + <properties> + <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> + <maven.compiler.source>1.8</maven.compiler.source> + <maven.compiler.target>1.8</maven.compiler.target> + + <websiteOutputDirectory>${project.build.directory}/website</websiteOutputDirectory> + <freemarkerGeneratorAppHome>${project.basedir}/../freemarker-generator-cli/target/appassembler</freemarkerGeneratorAppHome> + <docgenInsertedExampleTemplatesDirectory>${freemarkerGeneratorAppHome}/examples/templates</docgenInsertedExampleTemplatesDirectory> + <docgenInsertedExampleOutputDirectory>${freemarkerGeneratorAppHome}/target/out</docgenInsertedExampleOutputDirectory> + <docgenInsertedTemplatesDirectory>${freemarkerGeneratorAppHome}/templates</docgenInsertedTemplatesDirectory> + </properties> + + <dependencies> + <dependency> + <groupId>org.apache.freemarker.generator</groupId> + <artifactId>freemarker-generator-cli</artifactId> + <version>${project.version}</version> + </dependency> + </dependencies> + + <build> + <plugins> + <plugin> + <groupId>org.apache.freemarker.docgen</groupId> + <artifactId>freemarker-docgen-maven</artifactId> + <version>0.0.2-SNAPSHOT</version> + <executions> + <execution> + <id>docgen-transform</id> + <phase>package</phase> + <goals> + <goal>transform</goal> + </goals> + </execution> + </executions> + <configuration> + <sourceDirectory>src/main/docgen</sourceDirectory> + <outputDirectory>${websiteOutputDirectory}</outputDirectory> + <insertableFiles> + <templates>${docgenInsertedTemplatesDirectory}/**</templates> + <exampleTemplates>${docgenInsertedExampleTemplatesDirectory}/**</exampleTemplates> + </insertableFiles> + <customVariables> + <version>${project.version}</version> + <freemarkerGeneratorAppHome>${freemarkerGeneratorAppHome}</freemarkerGeneratorAppHome> + </customVariables> + <!-- TODO set it back to false before release --> + <offline>true</offline> + </configuration> + <dependencies> + <dependency> + <groupId>org.apache.freemarker.generator</groupId> + <artifactId>freemarker-generator-cli</artifactId> + <version>${project.version}</version> + </dependency> + </dependencies> + </plugin> + </plugins> + </build> + + <repositories> + <!-- This is need this only when we have freemarker-docgen-maven SNAPSHOT dependency --> + <repository> + <id>apache-snapshot-repository</id> + <url>https://repository.apache.org/content/repositories/snapshots/</url> + <releases><enabled>false</enabled></releases> + <snapshots><enabled>true</enabled></snapshots> + </repository> + </repositories> +</project> diff --cc freemarker-generator-website/src/main/docgen/book.xml index 791051c,0000000..0da2433 mode 100644,000000..100644 --- a/freemarker-generator-website/src/main/docgen/book.xml +++ b/freemarker-generator-website/src/main/docgen/book.xml @@@ -1,1537 -1,0 +1,1537 @@@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +--> +<book conformance="docgen" version="5.0" xml:lang="en" + xmlns="http://docbook.org/ns/docbook"; + xmlns:xlink="http://www.w3.org/1999/xlink"; + xmlns:xi="http://www.w3.org/2001/XInclude"; + xmlns:ns5="http://www.w3.org/1998/Math/MathML"; + xmlns:ns4="http://www.w3.org/1999/xhtml"; + xmlns:ns3="http://www.w3.org/2000/svg"; + xmlns:ns="http://docbook.org/ns/docbook";> + <info> + <title>Apache FreeMarker Generator Manual</title> + + <titleabbrev>FreeMarker Generator Manual</titleabbrev> + + <productname>Apache FreeMarker Generator + [docgen.customVariables.version]</productname> + </info> + + <chapter role="index.html" xml:id="about"> + <title>What's FreeMarker Generator?</title> + + <remark>Up until "Quick fasts", this is mostly new content compared to the + Markdown version.</remark> + + <warning> + <para><emphasis>No backward compatibility is promised!</emphasis> + FreeMarker Generator is in early stage of development, and so new + versions can break backward compatibility. However, the template engine + itself (FreeMarker) keeps backward compatibility, so hopefully the bulk + of your work can be still used in a future versions of FreeMarker + Generator, should you decide to upgrade it.</para> + </warning> + + <para>FreeMarker Generator is a command line tool that generates files + based on FreeMarker template files, and data that's typically provided in + files (such as JSON files, CSV files, etc.) as well. It can be used to + generated source code, configuration files, etc.</para> + + <para>Quick facts:</para> + + <itemizedlist> + <listitem> + <para>Requires Java 8+ on Linux, Mac OS X, or Windows</para> + </listitem> + + <listitem> + <para>Uses stdin, files, and URL-s to load data and templates</para> + </listitem> + + <listitem> + <para>Reads a variety of structured data formats such as access logs, + CSV, Excel, JSON, HTML, YAML, XML</para> + </listitem> + + <listitem> + <para>Provides JSONPath, XPath and DataFrame-s for advanced data + manipulation</para> + </listitem> + + <listitem> + <para>Can transform a whole directory with a single command-line + invocation</para> + </listitem> + + <listitem> + <para>Made for the heavy lifting of data by using lazy-loading and + streaming</para> + </listitem> + </itemizedlist> + + <note> + <para>Currently, FreeMarker Generator can only be invoked via command + line interface (CLI). In the future, we might add other interfaces, like + a Maven plugin. Actually, there's already a FreeMarker Generator Maven + plugin, but that doesn't share code, or concepts with the CLI. In the + future, we hope to unite these into a common engine. But for now the + Maven plugin is considered to be a legacy artifact. However, it works, + and you can use it. But anything you see in this documentation is not + applicable there, except this: [TODO copy and link plugin Maven + site]</para> + </note> + </chapter> + + <chapter xml:id="first-steps"> + <title>First Steps</title> + + <section xml:id="getting-started"> + <title>Getting Started</title> + + <simplesect> + <title>Installation</title> + + <itemizedlist> + <listitem> + <para>Requires Java 1.8+ on Linux, Mac OS X and Windows</para> + </listitem> + + <listitem> + <para><link linkend="download">Download</link> the binary release + of the FreeMarker Generator command line tool. Unpack the archive + in a directory of your choice.</para> + </listitem> + + <listitem> + <para>Add the <literal>bin/freemarker-cli</literal> or + <literal>bin/freemarker-cli.bat</literal> to your + <literal>PATH</literal> variable</para> + </listitem> + </itemizedlist> + </simplesect> + + <simplesect> + <title>Verify Installation</title> + + <para>On my local box (Mac OS 10.15.5) I use the following + setup:</para> + + <remark>This above have used export FREEMARKER_CLI_HOME=..., but I + found that to be confusing, as one my thinks FREEMARKER_CLI_HOME is + recognized by the launcher script something (like JAVA_HOME, ANT_HOME, + etc).</remark> + + <programlisting>export PATH=$PATH:/Applications/Java/freemarker-generator/bin</programlisting> + + <para>Afterwards Apache FreeMarker Generator can be executed from the + command line:</para> + + <programlisting>> which freemarker-generator +/Applications/Java/freemarker-generator/bin/freemarker-generator</programlisting> + + <para>and check the version of Apache FreeMarker CLI:</para> + + <programlisting>[docgen.insertWithOutput] +freemarker-generator -V +[/docgen.insertWithOutput]</programlisting> + </simplesect> + + <simplesect> + <title>Command Line Options</title> + + <para>Apache FreeMarker Generator provides command line help as shown + below:</para> + + <programlisting>[docgen.insertWithOutput] +freemarker-generator -h +[/docgen.insertWithOutput]</programlisting> + </simplesect> + + <simplesect> + <title>The Info Template</title> + + <para>The distribution ships with a couple of FreeMarker templates and + the <literal>templates/freemarker-generator/info.ftl</literal> is + particularly helpful to better understand Apache FreeMarker + CLI.</para> + + <programlisting>[docgen.insertWithOutput] +freemarker-generator -t freemarker-generator/info.ftl +[/docgen.insertWithOutput]</programlisting> + + <para>Above:</para> + + <itemizedlist> + <listitem> + <para>The <quote>FreeMarker Generator Information</quote> section + provides insights into configuration and currently processed + template.</para> + </listitem> + + <listitem> + <para>The <quote>FreeMarker Generator Template Loader + Directories</quote> shows the template directories being searched + to resolve a template path</para> + </listitem> + + <listitem> + <para>The <quote>FreeMarker Generator Tools</quote> section list + the available tools</para> + </listitem> + + <listitem> + <para>The <quote>FreeMarker Generator Data Model</quote> section + shows all available entries in the current FreeMarker + context</para> + </listitem> + </itemizedlist> + </simplesect> + </section> + + <section xml:id="running-examples"> + <title>Running the Examples</title> + + <para>There are many examples (see below) available you can execute - + The examples were tested with Java 1.8 on Mac OS X.</para> + + <para>Run <literal>run-examples.sh</literal> or + <literal>run-examples.bat</literal> in the Apache FreeMarker Generator + installation directory, and have a look at the generated output.</para> + + <remark>Removed console output showing list of examples and ls like + output from here. I think it doesn't help users, and had to be generated + in Maven somehow.</remark> + + <para>Please note that generated PDF files are very likely not found + since they require wkhtmltopdf and Apache FOP installation.</para> + + <section> + <title>Transforming GitHub JSON To Markdown</title> + + <para>A simple example with real JSON data to be transformed into + Markdown.</para> + + <para>You can either use the existing JSON sample</para> + + <programlisting>[docgen.insertWithOutput] +freemarker-generator -t [docgen.wd]/examples/templates/json/md/github-users.ftl [docgen.wd]/examples/data/json/github-users.json +[/docgen.insertWithOutput]</programlisting> + + <para>or pipe a <literal>curl</literal> response</para> + + <programlisting>> curl -s https://api.github.com/users | freemarker-generator -t examples/templates/json/md/github-users.ftl --stdin</programlisting> + + <para>Below you see the Apache FreeMarker Template:</para> + + <programlisting role="template">[docgen.insertFile "@exampleTemplates/json/md/github-users.ftl"]</programlisting> + + <para>This creates the following output:</para> + + <mediaobject> + <imageobject> + <imagedata fileref="images/examples/github.png" width="100%"/> + </imageobject> + </mediaobject> + + <remark>Above are real people, so we shouldn't include this.</remark> + </section> + + <section> + <title>CSV to HTML/Markdown Transformation</title> + + <para>Sometimes you have a CSV file which needs to be translated in + Markdown or HTML - there are on-line solutions + available<remark>[removed external product link from here]</remark>, + but having a local solution gives you more flexibility.</para> + + <programlisting>[docgen.checkCommand] +freemarker-generator -t freemarker-generator/csv/md/transform.ftl [docgen.wd]/examples/data/csv/contract.csv +[/docgen.checkCommand] +[docgen.checkCommand] +freemarker-generator -t freemarker-generator/csv/html/transform.ftl [docgen.wd]/examples/data/csv/contract.csv +[/docgen.checkCommand]</programlisting> + + <para>The FreeMarker template is shown below.</para> + + <programlisting role="template">[docgen.insertFile "@templates/freemarker-generator/csv/md/transform.ftl"]</programlisting> + + <para>The resulting file actually looks pleasant when compared to raw + CSV</para> + + <mediaobject> + <imageobject> + <imagedata fileref="images/examples/contract.png" width="100%"/> + </imageobject> + </mediaobject> + </section> + + <section> + <title>Transform XML To Plain Text</title> + + <para>Of course you can also transform an XML document:</para> + + <programlisting>[docgen.insertWithOutput] +freemarker-generator -t [docgen.wd]/examples/templates/xml/txt/recipients.ftl [docgen.wd]/examples/data/xml/recipients.xml +[/docgen.insertWithOutput]</programlisting> + + <para>This is using the following template:</para> + + <programlisting role="template">[docgen.insertFile "@exampleTemplates/xml/txt/recipients.ftl"]</programlisting> + </section> + + <section> + <title>Transform JSON To CSV</title> + + <para>One day I was asked a to prepare a CSV files containing REST + endpoints described by Swagger - technically this is a JSON to CSV + transformation. Of course I could create that CSV manually but writing + a FTL template doing that was simply more fun and saves time in the + future.</para> + + <remark>Certainly many will find the story telling tone strange. + Especially if we assume that others will contribute to this (who's "I" + then). We should just state what this is an example of, tersely. This + also applies to other parts.</remark> + + <para><remark>Below has no example source code to include. As we don't + provide backward compatibility yet, we better stick to runnable + examples, or else they will become outdated here.</remark></para> + + <programlisting role="template">[docgen.insertFile "@exampleTemplates/json/csv/swagger-endpoints.ftl"]</programlisting> + + <para>Invoking the FTL template, and its output:</para> + + <programlisting>[docgen.insertWithOutput] +freemarker-generator -t [docgen.wd]/examples/templates/json/csv/swagger-endpoints.ftl [docgen.wd]/examples/data/json/swagger-spec.json +[/docgen.insertWithOutput]</programlisting> + </section> + + <section> + <title>Transforming Excel Documents</title> + + <para>Another day my project management asked me to create a CSV + configuration file based on an Excel documents - as usual manual + copying was not an option due to required data cleanup and data + transformation. So I thought about Apache POI which support XLS and + XLSX documents - integration of Apache POI was a breeze but the + resulting code was not particularly useful example. So a more generic + transformation was provided to show the transformation of Excel + documents.</para> + + <programlisting>[docgen.checkCommand] +freemarker-generator -t freemarker-generator/excel/html/transform.ftl [docgen.wd]/examples/data/excel/test.xls +[/docgen.checkCommand] +[docgen.checkCommand] +freemarker-generator -t freemarker-generator/excel/html/transform.ftl [docgen.wd]/examples/data/excel/test.xlsx +[/docgen.checkCommand] +[docgen.checkCommand] +freemarker-generator -t freemarker-generator/excel/html/transform.ftl [docgen.wd]/examples/data/excel/test-multiple-sheets.xlsx +[/docgen.checkCommand] +[docgen.checkCommand] +freemarker-generator -t freemarker-generator/excel/md/transform.ftl [docgen.wd]/examples/data/excel/test-multiple-sheets.xlsx +[/docgen.checkCommand]</programlisting> + + <para>The provided FTL transforms an Excel into a HTML document + supporting multiple Excel sheets:</para> + + <programlisting role="template">[docgen.insertFile "@templates/freemarker-generator/excel/html/transform.ftl"]</programlisting> + + <para>but the result looks reasonable</para> + + <mediaobject> + <imageobject> + <imagedata fileref="images/examples/excel-to-html.png" + width="100%"/> + </imageobject> + </mediaobject> + </section> + + <section> + <title>Transform Property Files To CSV</title> + + <para>In this sample we transform all property files found in a + directory (recursive search using include pattern) to a CSV + file.</para> + + <programlisting>[docgen.insertWithOutput] +freemarker-generator --data-source-include *.properties -t [docgen.wd]/examples/templates/properties/csv/locker-test-users.ftl [docgen.wd]/examples/data/properties +[/docgen.insertWithOutput]</programlisting> + + <para>The template it uses is this:</para> + + <programlisting role="template">[docgen.insertFile "@exampleTemplates/properties/csv/locker-test-users.ftl"]</programlisting> + + <para>The template uses a couple of interesting features:</para> + + <itemizedlist> + <listitem> + <para>We process a list of property files</para> + </listitem> + + <listitem> + <para>The <literal>strip_text</literal> and + <literal>compress</literal> strips any white-spaces and + line-breaks from the output so we can create a proper CSV + file</para> + </listitem> + + <listitem> + <para>We use FTL functions to extract the + <literal>tenant</literal> and <literal>site</literal>, e.g. + <literal>extractTenant</literal></para> + </listitem> + + <listitem> + <para>We add a manual line break using + <literal>${'\n'}</literal></para> + </listitem> + </itemizedlist> + </section> + + <section> + <title>Transform CSV To XML-FO</title> + + <para>For a POC (proof of concept) I created a sample transformation + from CSV to XML-FO in order to create a PDF document using <link + xlink:href="https://xmlgraphics.apache.org/fop";>Apache FOP</link> + using the following template file:</para> + + <programlisting role="template">[docgen.insertFile "@exampleTemplates/csv/fo/transform.ftl"]</programlisting> + + <para>In order to create the PDF you need to execute the following + commands (assuming that you have Apache FOP installed):</para> + + <programlisting>[docgen.insertWithOutput]freemarker-generator -t [docgen.wd]/examples/templates/csv/fo/transform.ftl [docgen.wd]/examples/data/csv/locker-test-users.csv[/docgen.insertWithOutput] > sample.fo + +> fop -fo sample.fo sample.pdf +Dec 29, 2018 10:24:30 PM org.apache.fop.events.LoggingEventListener processEvent +WARNING: Font "Symbol,normal,700" not found. Substituting with "Symbol,normal,400". +Dec 29, 2018 10:24:30 PM org.apache.fop.events.LoggingEventListener processEvent +WARNING: Font "ZapfDingbats,normal,700" not found. Substituting with "ZapfDingbats,normal,400". +Dec 29, 2018 10:24:30 PM org.apache.fop.events.LoggingEventListener processEvent +INFO: Rendered page #1.</programlisting> + + <para>The result does not look very impressive but it is a PDF + :-)</para> + + <mediaobject> + <imageobject> + <imagedata fileref="images/examples/locker-test-users-pdf.png" + width="100%"/> + </imageobject> + </mediaobject> + + <para>Further along the line of the POC we converted a transaction + export from CSV to PDF using Apache FOP:</para> + + <programlisting role="template">[docgen.insertFile "@exampleTemplates/csv/fo/transactions.ftl"]</programlisting> + + <programlisting>[docgen.insertWithOutput]freemarker-generator -t [docgen.wd]/examples/templates/csv/fo/transactions.ftl [docgen.wd]/examples/data/csv/transactions.csv[/docgen.insertWithOutput] > transactions.fo + +> fop -fo transactions.fo transactions.pdf +Jan 16, 2019 11:15:21 PM org.apache.fop.events.LoggingEventListener processEvent +WARNING: Font "Symbol,normal,700" not found. Substituting with "Symbol,normal,400". +Jan 16, 2019 11:15:21 PM org.apache.fop.events.LoggingEventListener processEvent +WARNING: Font "ZapfDingbats,normal,700" not found. Substituting with "ZapfDingbats,normal,400". +Jan 16, 2019 11:15:21 PM org.apache.fop.events.LoggingEventListener processEvent +WARNING: The contents of fo:block line 1 exceed the available area in the inline-progression direction by 11027 millipoints. (See position 1519:51) +Jan 16, 2019 11:15:22 PM org.apache.fop.events.LoggingEventListener processEvent +INFO: Rendered page #1. +Jan 16, 2019 11:15:22 PM org.apache.fop.events.LoggingEventListener processEvent +INFO: Rendered page #2.</programlisting> + + <mediaobject> + <imageobject> + <imagedata fileref="images/examples/transactions.png" width="100%"/> + </imageobject> + </mediaobject> + </section> + + <section> + <title>Transforming HTML To CSV</title> + + <para>Recently I got the rather unusual question how to determine the + list of dependencies of an application - one easy way is the Maven + "dependencies.html" but this is unstructured data. Having said that + the Jsoup library is perfectly able to parse most real-life HTML and + provides a DOM model.</para> + + <programlisting role="template">[docgen.insertFile "@exampleTemplates/html/csv/dependencies.ftl"]</programlisting> + + <para>Your dependencies as CSV can be generated as shown below:</para> + + <programlisting>[docgen.insertWithOutput] +freemarker-generator -t [docgen.wd]/examples/templates/html/csv/dependencies.ftl [docgen.wd]/examples/data/html/dependencies.html +[/docgen.insertWithOutput]</programlisting> + </section> + + <section> + <title>Transform CSV To Shell Script</title> + + <para>For a customer project we wanted to record REST request / + responses using WireMock - really quick and dirty. So we decided to + avoid any sophisticated test tool but generate a ready-to-use shell + script executing cURL commands. It turned out that handling of dollar + signs is a bit tricky.</para> + + <itemizedlist> + <listitem> + <para>Using <literal>noparse</literal> directive to disable + parsing of dollar signs</para> + </listitem> + + <listitem> + <para>Using <literal>${r"${MY_BASE_URL}"</literal> to generate + output with dollar signs</para> + </listitem> + </itemizedlist> + + <para>and the final FTL is found below:</para> + + <programlisting role="template">[docgen.insertFile "@exampleTemplates/csv/shell/curl.ftl"]</programlisting> + + <para>Rendering the FreeMarker template:</para> + + <programlisting>freemarker-generator -t examples/templates/csv/shell/curl.ftl examples/data/csv/user.csv</programlisting> + + <para>Looks a bit complicated but lets dissect the things</para> + + <itemizedlist> + <listitem> + <para><literal>date "+%FT%H:%M:%S" | tr -d '\n'</literal> creates + a timestamp and removes the line feed</para> + </listitem> + + <listitem> + <para><literal>curl --write-out</literal> allows to print runtime + data (see <link + xlink:href="https://ec.haxx.se/usingcurl-writeout.html";>https://ec.haxx.se/usingcurl-writeout.html</link>)</para> + </listitem> + </itemizedlist> + + <para><remark>Shell/curl specifics are irrelevant, as far as + FreeMarker Generator is concerned. We should remove the above + explanation.</remark></para> + + <para><remark>What specialty with generating shell scripts we want to + demonstrate in this example? Is it escaping ${...} maybe? That belongs + to the FreeMarker documentation. But if we still want to show that + here, I would recommend using the [=...] interpolation syntax when + generating something that already uses ${...}.</remark></para> + + <para>Executing the result shell script creates the following output + (which is a nice CSV for further processing):</para> + + <programlisting>time,user,status,duration,size +2019-09-27T21:02:52,AAAAAAA,200,0.522473,206 +2019-09-27T21:02:53,BBBBBBB,200,0.498093,206 +2019-09-27T21:02:54,CCCCCCC,200,0.529013,206 +2019-09-27T21:02:54,DDDDDDD,200,0.528268,206</programlisting> + + <remark>We can't generate the above in Docgen, as curl is might not be + available, or can't fetch remote resource.</remark> + </section> + + <section> + <title>Executing Arbitrary Commands</title> + + <para>Using Apache Commons Exec allows to execute arbitrary commands - + nice but dangerous. It was recently quite useful to to invoke AWS CLI + to generate a Confluence page about the overall setup of our AWS + accounts.</para> + + <para>A few snippets to illustrate the points:</para> + + <programlisting role="template"><#ftl output_format="plainText" strip_whitespace="true"> +<#assign profile = tools.system.getProperty("profile", "default")> +<#assign ec2Instances = ec2Instances()/> + +h3. AWS EC2 Instance +<@printEc2Instances ec2Instances/> + +<#function ec2Instances> + <#local json = awsCliToJson("aws ec2 describe-instances --profile ${profile}")> + <#local instances = json.read("$.Reservations[*].Instances[*]")> + <#return instances?sort_by(['InstanceType'])> +</#function> + +<#function awsCliToJson line> + <#local output = tools.exec.execute(line)> + <#return tools.jsonpath.parse(output)> +</#function> + +<#function getAwsEc2InstanceTag tags name> + <#return tags?filter(x -> x["Key"] == name)?first["Value"]!""> +</#function> + +<#macro printEc2Instances ec2Instances> + <#compress> + || NAME || INSTANCE_TYPE || VCPUS || STATE || PRIVATE_IP_ADDRESS || + <#list ec2Instances as ec2Instance> + <#assign instanceType = ec2Instance["InstanceType"]> + <#assign arn = ec2Instance["IamInstanceProfile"]["Arn"]> + <#assign privateIpAddress = ec2Instance["PrivateIpAddress"]> + <#assign state = ec2Instance["State"]["Name"]> + <#assign launchTime = ec2Instance["LaunchTime"]> + + <#assign coreCount = ec2Instance["CpuOptions"]["CoreCount"]?number> + <#assign threadsPerCore = ec2Instance["CpuOptions"]["ThreadsPerCore"]?number> + <#assign nrOfVirtualCpus = coreCount * threadsPerCore> + + <#assign tags = ec2Instance["Tags"]/> + <#assign awsCloudFormationStackId = getAwsEc2InstanceTag(tags, "aws:cloudformation:stack-id")> + <#assign awsCloudFormationStackName = getAwsEc2InstanceTag(tags, "aws:cloudformation:stack-name")> + <#assign name = getAwsEc2InstanceTag(tags, "Name")> + <#assign country = getAwsEc2InstanceTag(tags, "Country")> + <#assign environment = getAwsEc2InstanceTag(tags, "Environment")> + + | ${name} | ${instanceType} | ${nrOfVirtualCpus} | ${state} | ${privateIpAddress} | + </#list> + </#compress> +</#macro></programlisting> + + <para><remark>Not in runnable examples (see same problem earlier, why + that matters). Although, it's next to impossible to make a portable + example of this. Question is, what do we want to demonstrate here? Lot + of lines to dig through for the reader, and the point is maybe just + <#local output = tools.exec.execute(line)>.</remark></para> + </section> + + <section> + <title>Interactive Templates</title> + + <para>Sometime you need to apply a CSS, JSON or XPath query in an + ad-hoc way without installing <literal>xmllint</literal>, + <literal>jq</literal> or <literal>pup</literal> - in this case you can + pass a FreeMarker template in an interactive fashion.</para> + + <programlisting>[docgen.insertWithOutput] +freemarker-generator -i 'Some template here... ${1 + 1}' +[/docgen.insertWithOutput] + +[docgen.insertWithOutput] - freemarker-generator -i '${tools.jsonpath.parse(dataSources?values[0]).read("$.info.title")}' [docgen.wd]/examples/data/json/swagger-spec.json ++freemarker-generator -i '${tools.jsonpath.parse(dataSources[0]).read("$.info.title")}' [docgen.wd]/examples/data/json/swagger-spec.json +[/docgen.insertWithOutput] + +[docgen.insertWithOutput] - freemarker-generator -i '${tools.xml.parse(dataSources?values[0])["recipients/person[1]/name"]}' [docgen.wd]/examples/data/xml/recipients.xml ++freemarker-generator -i '${tools.xml.parse(dataSources[0])["recipients/person[1]/name"]}' [docgen.wd]/examples/data/xml/recipients.xml +[/docgen.insertWithOutput] + +[docgen.insertWithOutput] - freemarker-generator -i '${tools.jsoup.parse(dataSources?values[0]).select("a")[0]}' [docgen.wd]/examples/data/html/dependencies.html ++freemarker-generator -i '${tools.jsoup.parse(dataSources[0]).select("a")[0]}' [docgen.wd]/examples/data/html/dependencies.html +[/docgen.insertWithOutput]</programlisting> + + <remark>I have deleted some examples above, as they were problematic + to generate. I think one or two would be enough anyway, as we just + demonstrate -i here.</remark> + </section> + + <section> + <title>Filtering & Transforming CSV</title> + + <para>During an integration project we imported large transactions CSV + files (500.000+ records) and in case of import failures the developers + would be happy to get a nice outline of the transactions causing the + problem (the CSV records have 60+ columns) - in essence it is + filtering (based on some primary key) and and transforming into a + human-readable output format (Markdown).</para> + - <para>So lets start the filtering & transformaction using the ++ <para>So lets start the filtering & transformation using the + following command line</para> + + <programlisting>[docgen.insertWithOutput] +freemarker-generator -e UTF-8 -l de_AT -Pcolumn="Order ID" \ + -Pvalues=226939189,957081544 \ + -t [docgen.wd]/examples/templates/csv/md/filter.ftl [docgen.wd]/examples/data/csv/sales-records.csv +[/docgen.insertWithOutput]</programlisting> + + <para>The FreeMarker template referred was this:</para> + + <programlisting role="template">[docgen.insertFile "@exampleTemplates/csv/md/filter.ftl"]</programlisting> + </section> + + <section> + <title>Converting Between JSON And YAML</title> + + <para>Sometimes we simply need to transform a JSON into an equivalent + YAML or the other way around.</para> + + <programlisting>[docgen.insertWithOutput]freemarker-generator -t freemarker-generator/yaml/json/transform.ftl [docgen.wd]/examples/data/yaml/swagger-spec.yaml[/docgen.insertWithOutput] - [docgen.insertWithOutput]freemarker-generator -i '${tools.gson.toJson(tools.yaml.parse(dataSources?values[0]))}' [docgen.wd]/examples/data/yaml/swagger-spec.yaml[/docgen.insertWithOutput] ++[docgen.insertWithOutput]freemarker-generator -i '${tools.gson.toJson(tools.yaml.parse(dataSources[0]))}' [docgen.wd]/examples/data/yaml/swagger-spec.yaml[/docgen.insertWithOutput] +[docgen.insertWithOutput]freemarker-generator -i '${tools.gson.toJson(yaml)}' -m yaml=[docgen.wd]/examples/data/yaml/swagger-spec.yaml[/docgen.insertWithOutput] + +[docgen.insertWithOutput]freemarker-generator -t freemarker-generator/json/yaml/transform.ftl [docgen.wd]/examples/data/json/swagger-spec.json[/docgen.insertWithOutput] - [docgen.insertWithOutput]freemarker-generator -i '${tools.yaml.toYaml(tools.gson.parse(dataSources?values[0]))}' [docgen.wd]/examples/data/json/swagger-spec.json[/docgen.insertWithOutput] ++[docgen.insertWithOutput]freemarker-generator -i '${tools.yaml.toYaml(tools.gson.parse(dataSources[0]))}' [docgen.wd]/examples/data/json/swagger-spec.json[/docgen.insertWithOutput] +[docgen.insertWithOutput]freemarker-generator -i '${tools.yaml.toYaml(json)}' -m json=[docgen.wd]/examples/data/json/swagger-spec.json[/docgen.insertWithOutput]</programlisting> + </section> + + <section> + <title>Using Advanced FreeMarker Features</title> + + <para>There is a <literal>demo.ftl</literal> which shows some advanced + FreeMarker functionality:</para> + + <itemizedlist> + <listitem> + <para>Invoking a Java constructor</para> + </listitem> + + <listitem> + <para>Invoke a static method of non-instantiable class</para> + </listitem> + + <listitem> + <para>Work with Java enumerations</para> + </listitem> + + <listitem> + <para>Access System properties</para> + </listitem> + + <listitem> + <para>Access Environment variables</para> + </listitem> + </itemizedlist> + + <programlisting>[docgen.insertWithOutput] +freemarker-generator -t [docgen.wd]/examples/templates/demo.ftl +[/docgen.insertWithOutput]</programlisting> + </section> + </section> + </chapter> + + <chapter xml:id="concepts"> + <title>Concepts</title> + + <section xml:id="design-goals"> + <title>Design Goals</title> + + <itemizedlist> + <listitem> + <para>Create a proper command-line tool which has Unix look & + feel</para> + </listitem> + + <listitem> + <para>Handle arbitrary large input and output data</para> + </listitem> + + <listitem> + <para>Support multiple source files/directories for a single + transformation</para> + </listitem> + + <listitem> + <para>Support transformation of Java <literal>properties</literal> + (<literal>java.util.Properties</literal>) files<remark> Reworded + this a bit. Especially, to remove reference to the + JDK.</remark></para> + </listitem> + + <listitem> + <para>Support transformation of CSV files using <link + xlink:href="https://commons.apache.org/proper/commons-csv/";>Apache + Commons CSV</link></para> + </listitem> + + <listitem> + <para>Support transformation of JSON using <link + xlink:href="https://github.com/jayway/JsonPath";>Jayway's + JSONPath</link> and <link + xlink:href="https://github.com/google/gson";>GSON</link></para> + </listitem> + + <listitem> + <para>Support transformation of Excel using <link + xlink:href="https://poi.apache.org";>Apache POI</link></para> + </listitem> + + <listitem> + <para>Support transformation of YAML using <link + xlink:href="https://bitbucket.org/asomov/snakeyaml/wiki/Home";>SnakeYAML</link></para> + </listitem> + + <listitem> + <para>Support transformation of HTML using <link + xlink:href="https://jsoup.org";>JSoup</link></para> + </listitem> + + <listitem> + <para>Support transformation of structured log files using <link + xlink:href="https://github.com/thekrakken/java-grok";>Grok</link></para> + </listitem> + + <listitem> + <para>XML & XPath is supported by FreeMarker <link + xlink:href="http://freemarker.org/docs/xgui.html";>out-of-the-box</link></para> + </listitem> + + <listitem> + <para>Support for reading a data source content from STDIN to + integrate with command line tools</para> + </listitem> + + <listitem> + <para>Support execution of arbitrary commands using <link + xlink:href="https://commons.apache.org/proper/commons-exec/";>Apache + Commons Exec</link></para> + </listitem> + + <listitem> + <para>Support creation of test data using <link + xlink:href="https://github.com/DiUS/java-faker/";>JavaFaker</link></para> + </listitem> + + <listitem> + <para>Add some commonly useful information such as Java System + Properties, environment variables</para> + </listitem> + + <listitem> + <para>Support embedding the code in existing applications</para> + </listitem> + </itemizedlist> + </section> + + <section xml:id="datasources"> + <title>Data Sources</title> + + <para>A <literal>DataSource</literal> consists of lazy-loaded data + available in Apache FreeMarker's model. It provides:</para> + + <itemizedlist> + <listitem> + <para>A <literal>name</literal> uniquely identifying a data + source</para> + </listitem> + + <listitem> + <para>An <literal>uri</literal> which as used to create the data + source</para> + </listitem> + + <listitem> + <para>A <literal>contentType</literal> and + <literal>charset</literal></para> + </listitem> + + <listitem> + <para>Access to textual content directly or using a line + iterator</para> + </listitem> + + <listitem> + <para>Access to the underlying data input stream</para> + </listitem> + </itemizedlist> + + <section> + <title>Loading a Data Source</title> + + <para>A <literal>DataSource</literal> can be loaded from the file + system, e.g., as positional command line argument:</para> + - <programlisting>[docgen.insertWithOutput from="FreeMarker Generator DataSources" to=r"^\s*$"] ++ <programlisting>[docgen.insertWithOutput from="FreeMarker Generator Data Sources" to=r"^\s*$"] +freemarker-generator -t freemarker-generator/info.ftl [docgen.wd]/README.md +[/docgen.insertWithOutput]</programlisting> + + <para>from an URL:</para> + - <programlisting>[docgen.insertWithOutput from="FreeMarker Generator DataSources" to=r"^\s*$"] ++ <programlisting>[docgen.insertWithOutput from="FreeMarker Generator Data Sources" to=r"^\s*$"] +freemarker-generator -t freemarker-generator/info.ftl --data-source xkcd=https://xkcd.com/info.0.json +[/docgen.insertWithOutput]</programlisting> + + <para>or from an environment variable, e.g. + <literal>NGINX_CONF</literal> having a JSON payload:</para> + - <programlisting>[docgen.insertWithOutput from="FreeMarker Generator DataSources" to=r"^\s*$" ++ <programlisting>[docgen.insertWithOutput from="FreeMarker Generator Data Sources" to=r"^\s*$" + systemProperties={'freemarker.generator.datasource.envOverride.NGINX_CONF': '{"NGINX_PORT":"8443","NGINX_HOSTNAME":"localhost"}'} +] +freemarker-generator -t freemarker-generator/info.ftl --data-source conf=env:///NGINX_CONF#mimeType=application/json +[/docgen.insertWithOutput]</programlisting> + + <para>Of course you can load multiple data sources directly:</para> + - <programlisting>[docgen.insertWithOutput from="FreeMarker Generator DataSources" to=r"^\s*$"] ++ <programlisting>[docgen.insertWithOutput from="FreeMarker Generator Data Sources" to=r"^\s*$"] +freemarker-generator -t freemarker-generator/info.ftl [docgen.wd]/README.md xkcd=https://xkcd.com/info.0.json +[/docgen.insertWithOutput]</programlisting> + + <para>or load them from a directory:</para> + - <programlisting>[docgen.insertWithOutput from="FreeMarker Generator DataSources" to=r"^\s*$"] ++ <programlisting>[docgen.insertWithOutput from="FreeMarker Generator Data Sources" to=r"^\s*$"] +freemarker-generator -t freemarker-generator/info.ftl --data-source [docgen.wd]/examples/data +[/docgen.insertWithOutput]</programlisting> + + <para>which can be combined with <literal>include</literal> and + <literal>exclude</literal> filters:</para> + - <programlisting>[docgen.insertWithOutput from="FreeMarker Generator DataSources" to=r"^\s*$"] ++ <programlisting>[docgen.insertWithOutput from="FreeMarker Generator Data Sources" to=r"^\s*$"] +freemarker-generator -t freemarker-generator/info.ftl -s [docgen.wd]/examples/data --data-source-include='*.json' +[/docgen.insertWithOutput]</programlisting> + + <para>Access to <literal>stdin</literal> is implemented as + <literal>DataSource</literal> - please note that + <literal>stdin</literal> is read lazily to cater for arbitrary large + input data<remark> (TODO: Docgen can't generate this yet, because of + the cat and pipe)</remark>:</para> + + <programlisting>cat examples/data/csv/contract.csv | bin/freemarker-generator -t freemarker-generator/info.ftl --stdin + - FreeMarker Generator DataSources ++FreeMarker Generator Data Sources +------------------------------------------------------------------------------ +[#1]: name=stdin, group=default, fileName=stdin mimeType=text/plain, charset=UTF-8, length=-1 Bytes +URI : system:///stdin</programlisting> + </section> + + <section> + <title>Selecting a Data Source</title> + + <para>After loading one or more <literal>DataSource</literal> they are + accessible as <literal>dataSource</literal> map in the FreeMarker + model:</para> + + <itemizedlist> + <listitem> - <para><literal>dataSources?values[0]</literal> or - <literal>dataSources?values?first</literal> selects the first data ++ <para><literal>dataSources[0]</literal> or ++ <literal>dataSources?first</literal> selects the first data + source</para> + </listitem> + + <listitem> + <para><literal>dataSources["user.csv"]</literal> selects the data + source with the name <literal>"user.csv"</literal></para> + </listitem> + </itemizedlist> + </section> + + <section> + <title>Iterating Over Data Sources</title> + + <para>The data sources are exposed as map within FreeMarker's data + model<remark> (TODO: There's no example template to insert + here)</remark>:</para> + + <programlisting><#-- Do something with the data sources --> +<#if dataSources?has_content> +Some data sources found +<#else> +No data sources found ... +</#if> + +<#-- Get the number of data sources --> +${dataSources?size} + +<#-- Iterate over a map of data sources --> +<#list dataSources as name, dataSource> +- ${name} => ${dataSource.length} +</#list> + +<#-- Iterate over a list of data sources --> - <#list dataSources?values as dataSource> ++<#list dataSources as dataSource> +- [#${dataSource?counter}]: name=${dataSource.name} +</#list></programlisting> + </section> + + <section> + <title>Filtering of Data Sources</title> + + <para>Combining FreeMarker's <literal>filter</literal> built-in with + the <literal>DataSource.match</literal> methods allows more advanced + selection of data sources (using Apache Commons IO wild-card + matching)<remark> (TODO: There's no example template to insert + here)</remark>:</para> + + <programlisting><#-- List all data sources containing "test" in the name --> - <#list dataSources?values?filter(ds -> ds.match("name", "*test*")) as ds> ++<#list dataSources?filter(ds -> ds.match("name", "*test*")) as ds> +- ${ds.name} +</#list> + +<#-- List all data sources having "json" extension --> - <#list dataSources?values?filter(ds -> ds.match("extension", "json")) as ds> ++<#list dataSources?filter(ds -> ds.match("extension", "json")) as ds> +- ${ds.name} +</#list> + +<#-- List all data sources having "src/test/data/properties" in their file path --> - <#list dataSources?values?filter(ds -> ds.match("filePath", "*/src/test/data/properties")) as ds> ++<#list dataSources?filter(ds -> ds.match("filePath", "*/src/test/data/properties")) as ds> +- ${ds.name} +</#list> + +<#-- List all data sources of a group --> - <#list dataSources?values?filter(ds -> ds.match("group", "default")) as ds> ++<#list dataSources?filter(ds -> ds.match("group", "default")) as ds> +- ${ds.name} +</#list></programlisting> + </section> + + <section> + <title>Using and Inspecting Data Sources</title> + + <para><remark>These were two separate chapters, but they both shown + parts of datasources.ftl, so I unified them.</remark></para> + + <para>In most cases the data source will be passed to a tool, but + there are some useful operations available as shown below:</para> + + <programlisting>[docgen.insertFile "@exampleTemplates/datasources.ftl"]</programlisting> + + <para>will result in:</para> + + <programlisting>[docgen.insertWithOutput] +freemarker-generator -t [docgen.wd]/examples/templates/datasources.ftl --data-source [docgen.wd]/examples/data/csv/contract.csv +[/docgen.insertWithOutput]</programlisting> + + <para>You can also use similar command like above to inspect what data + source you have.</para> + </section> + </section> + + <section xml:id="named-uri"> + <title>Named URI-s</title> + + <para>Named URIs allow identifying <literal>DataSource</literal>-s (not + a JDBC <literal>DataSource</literal>, <link linkend="datasources">but + this</link><remark> - added this, or else it can confuse Java + developers</remark>) and pass additional information.</para> + + <para>A Named URI consists of:</para> + + <itemizedlist> + <listitem> + <para>An optional name</para> + </listitem> + + <listitem> + <para>An URI or simple file name</para> + </listitem> + </itemizedlist> + + <para><remark>There was a figure heer from + https://docs.gomplate.ca/datasources/, which is nicer but I wasn't sure + about how that affects our LICENSE, etc. So I substituted that with this + paragraph. </remark>As a refresher, here's an example URL to show what + components an URI can have in general: + <literal>someScheme://[email protected]:1234/foo/bar/baaz?param1=value1;param2=value2#someFragment</literal>. + Before the <literal>:</literal>, <literal>someScheme</literal> is called + the scheme, then, after the <literal>\\</literal>, + <literal>[email protected]:1234</literal> is called the authority, + then comes <literal>/foo/bar/baaz</literal> with is called the path, + then, after the <literal>?</literal>, + <literal>param1=value1;param2=value2</literal> is the query, and finally + after the <literal>#</literal>, <literal>someFragment</literal> is + called the fragment.</para> + + <para>For our purposes, the scheme and the path components are + especially important, though the other components are used by certain + data sources for particular purposes.</para> + + <orderedlist> + <listitem> + <para>Scheme: All data sources require a scheme (except for file + when using relative paths)</para> + </listitem> + + <listitem> + <para>Authority: Used only by remote data sources, and can be + omitted in some of those cases.</para> + </listitem> + + <listitem> + <para>Path: Can be omitted, but usually used as the basis of the + locator for the datasource.</para> + </listitem> + + <listitem> + <para>Query: Used mainly for HTTP and HTTPS URLs.</para> + </listitem> + + <listitem> + <para>Fragment: Used rarely for providing additional attributes, + e.g. <literal>mimeType</literal> of + <literal>charset</literal></para> + </listitem> + </orderedlist> + + <section> + <title>Using Named URIs For A File</title> + + <para>The following Named URI loads a <literal>user.csv</literal> and + the data source is available as <literal>my_users</literal>:</para> + - <programlisting>[docgen.insertWithOutput from="FreeMarker Generator DataSources" to=r"^\s*$"] ++ <programlisting>[docgen.insertWithOutput from="FreeMarker Generator Data Sources" to=r"^\s*$"] +freemarker-generator -t freemarker-generator/info.ftl my_users=[docgen.wd]/examples/data/csv/user.csv +[/docgen.insertWithOutput]</programlisting> + + <para>A Named URI allows to pass additional information as part of the + fragment, e.g. the charset of the text file:</para> + - <programlisting>[docgen.insertWithOutput from="FreeMarker Generator DataSources" to=r"^\s*$"] ++ <programlisting>[docgen.insertWithOutput from="FreeMarker Generator Data Sources" to=r"^\s*$"] +freemarker-generator -t freemarker-generator/info.ftl my_users=[docgen.wd]/examples/data/csv/user.csv#charset=UTF-16 +[/docgen.insertWithOutput]</programlisting> + + <para>In addition to the simplified file syntax full URIs can be + used:</para> + - <programlisting>[docgen.insertWithOutput from="FreeMarker Generator DataSources" to=r"^\s*$"] ++ <programlisting>[docgen.insertWithOutput from="FreeMarker Generator Data Sources" to=r"^\s*$"] +freemarker-generator -t freemarker-generator/info.ftl https://www.google.com#charset=ISO-8859-1 +[/docgen.insertWithOutput]</programlisting> + + <para>and also combined with a name:</para> + - <programlisting>[docgen.insertWithOutput from="FreeMarker Generator DataSources" to=r"^\s*$"] ++ <programlisting>[docgen.insertWithOutput from="FreeMarker Generator Data Sources" to=r"^\s*$"] +freemarker-generator -t freemarker-generator/info.ftl page=http://www.google.com#charset=ISO-8859-1 +[/docgen.insertWithOutput]</programlisting> + </section> + + <section> + <title>Using Named URIs For Directories</title> + + <para>A Name URI can be also combined with file directories.</para> + + <para>Load all CVS files of a directory using the group "csv":</para> + - <programlisting>[docgen.insertWithOutput from="FreeMarker Generator DataSources" to=r"^\s*$"] ++ <programlisting>[docgen.insertWithOutput from="FreeMarker Generator Data Sources" to=r"^\s*$"] +freemarker-generator -t freemarker-generator/info.ftl :csv=[docgen.wd]/examples/data/csv +[/docgen.insertWithOutput]</programlisting> + + <para>or use a charset for all files of a directory</para> + - <programlisting>[docgen.insertWithOutput from="FreeMarker Generator DataSources" to=r"^\s*$"] - freemarker-generator -t freemarker-generator/info.ftl '[docgen.wd]/examples/data/csv#charset=UTF-16&mimetype=text/plain' ++ <programlisting>[docgen.insertWithOutput from="FreeMarker Generator Data Sources" to=r"^\s*$"] ++freemarker-generator -t freemarker-generator/info.ftl '[docgen.wd]/examples/data/csv#charset=UTF-16&mimeType=text/plain' +[/docgen.insertWithOutput]</programlisting> + + <para>It is also possible to provide data source properties to all + files being loaded</para> + + <programlisting>[docgen.insertWithOutput] +freemarker-generator -t [docgen.wd]/examples/templates/datasources.ftl '[docgen.wd]/examples/data/csv#format=DEFAULT' +[/docgen.insertWithOutput]</programlisting> + + <para/> + </section> + </section> + + <section xml:id="data-models"> + <title>Data Models</title> + + <para>TODO</para> + </section> + + <section xml:id="passing-configuration-data"> + <title>Passing Configuration Data</title> + + <para>TODO</para> + </section> + + <section xml:id="template-loading"> + <title>Template Loading</title> + + <para>TODO</para> + </section> + + <section xml:id="transformation"> + <title>Transformation</title> + + <para>TODO</para> + </section> + + <section xml:id="tools"> + <title>Tools</title> + + <para>TODO</para> + </section> + </chapter> + + <chapter xml:id="usage-details"> + <title>Usage details</title> + + <remark role="editorialNote">In the markdown version this chapter was + "Usage", but I though that clashes with "Getting Started" in + meaning.</remark> + + <section> + <title>Transforming Directories</title> + + <para>TODO</para> + </section> + + <section> + <title>Using DataFrames</title> + + <para>TODO</para> + </section> + + <section> + <title>Transforming CSV</title> + + <para>TODO</para> + </section> + + <section> + <title>Generating test data</title> + + <para>TODO</para> + </section> + + <section> + <title>Parsing with Grok</title> + + <para>Think of <literal>Grok</literal> as modular regular expressions + with a pre-defined functionality to parse access logs or any other data + where you can't comprehend the regular expression any longer, one very + simple example is <literal>QUOTEDSTRING</literal>.</para> + + <programlisting>QUOTEDSTRING (?>(?<!\\)(?>"(?>\\.|[^\\"]+)+"|""|(?>'(?>\\.|[^\\']+)+')|''|(?>`(?>\\.|[^\\`]+)+`)|``))</programlisting> + + <para>And with <literal>Grok</literal> the + <literal>QUOTEDSTRING</literal> is just a building block for an even + more complex regular expression such as + <literal>COMBINEDAPACHELOG</literal>.</para> + + <programlisting>[docgen.insertWithOutput] +freemarker-generator -t [docgen.wd]/examples/templates/accesslog/combined-access.ftl [docgen.wd]/examples/data/accesslog/combined-access.log +[/docgen.insertWithOutput]</programlisting> + + <para>using the following FreeMarker template:</para> + + <programlisting>[docgen.insertFile "@exampleTemplates/accesslog/combined-access.ftl"]</programlisting> + + <para>While this looks small and tidy there are some nifty + features:</para> + + <itemizedlist> + <listitem> + <para><literal>tools.grok.compile("%{COMBINEDAPACHELOG}")</literal> + builds the <literal>Grok</literal> instance to parse access logs in + <literal>Combined Format</literal></para> + </listitem> + + <listitem> + <para>The data source is streamed line by line and not loaded into + memory in one piece</para> + </listitem> + + <listitem> + <para>This also works for using <literal>stdin</literal> so are able + to parse GB of access log or other files</para> + </listitem> + </itemizedlist> + </section> + </chapter> + + <chapter xml:id="advanced-topics"> + <title>Advanced topics</title> + + <section xml:id="freemarker-cli-configuration"> + <title>FreeMarker CLI Configuration</title> + + <para>TODO</para> + </section> + </chapter> + + <chapter xml:id="appendixes"> + <title>Appendixes</title> + + <section xml:id="download"> + <title>Download</title> + + <simplesect> + <title>Latest release: 0.1.0</title> + + <para>Released on 2020-[FIXME]-[FIXME]. Requires Java 1.8 or + higher.</para> + + <para><link linkend="version_0_1_0">See what's new...</link></para> + + <para>Downloads:</para> + + <itemizedlist> + <listitem> + <para><link + xlink:href="https://www.apache.org/dyn/closer.cgi/freemarker/generator/0.1.0/binaries/apache-freemarker-generator-cli-0.1.0-bin.tar.gz";>Binary + release of the command line tool (tar.gz)</link> [<link + xlink:href="https://www.apache.org/dyn/closer.cgi/freemarker/generator/0.1.0/binaries/apache-freemarker-generator-cli-0.1.0-bin.tar.gz.sha512";>SHA512</link>] + [<link + xlink:href="https://www.apache.org/dyn/closer.cgi/freemarker/generator/0.1.0/binaries/apache-freemarker-generator-cli-0.1.0-bin.tar.gz.asc";>ASC</link>]</para> + </listitem> + + <listitem> + <para><link + xlink:href="https://www.apache.org/dyn/closer.cgi/freemarker/generator/0.1.0/binaries/apache-freemarker-generatori-0.1.0-src.tar.gz";>Source + release (tar.gz)</link> [<link + xlink:href="https://www.apache.org/dyn/closer.cgi/freemarker/generator/0.1.0/binaries/apache-freemarker-generatori-0.1.0-src.tar.gz.sha512";>SHA512</link>] + [<link + xlink:href="https://www.apache.org/dyn/closer.cgi/freemarker/generator/0.1.0/binaries/apache-freemarker-generatori-0.1.0-src.tar.gz.asc";>ASC</link>]</para> + </listitem> + </itemizedlist> + </simplesect> + </section> + + <section xml:id="versionsHistory"> + <title>Version history</title> + + <section xml:id="version_0_1_0"> + <title>0.1.0</title> + + <para>This is the first release at Apache. The previous version of the + project was here: <link + xlink:href="https://github.com/sgoeschl/freemarker-cli";>https://github.com/sgoeschl/freemarker-cli</link>. + Compared to that, the most important changes are these:</para> + + <itemizedlist> + <listitem> + <para>[TODO]</para> + </listitem> + </itemizedlist> + </section> + </section> + + <section xml:id="legal"> + <title>Legal</title> + + <section xml:id="license"> + <title>License</title> + + <para>Bellow license applies to the FreeMarker Generator source code. + Binary distributions of FreeMarker Generator may bundles 3rd party + dependencies, in which case see the license included in the binary + distribution for more information about the licenses of bundled + dependencies!</para> + + <programlisting role="unspecified"> + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +============================================================================== +END LICENSE</programlisting> + </section> + + <section xml:id="export_control"> + <title>Export Control</title> + + <para>The FreeMarker Generator source code doesn't include + cryptography. Furthermore its binary (downloadable) forms don't + include any cryptography software. Hence, FreeMarker has no Export + Control Classification Number (ECCN). Where an ECCN should be filled, + the label "not subject to EAR" could be used.</para> + + <para>FreeMarker Generator itself doesn't add any exporting + limitations.</para> + </section> + </section> + </chapter> +</book>
