This is an automated email from the ASF dual-hosted git repository.
sgoeschl pushed a commit to branch feature/FREEMARKER-140
in repository https://gitbox.apache.org/repos/asf/freemarker-generator.git
The following commit(s) were added to refs/heads/feature/FREEMARKER-140 by this
push:
new 57f270e FREEMARKER-140 freemarker-cli: Expose DataSources directly in
the data model
57f270e is described below
commit 57f270ec6e453709dbad5bb376ecf959cfe2e1fa
Author: Siegfried Goeschl <[email protected]>
AuthorDate: Mon Apr 6 21:13:53 2020 +0200
FREEMARKER-140 freemarker-cli: Expose DataSources directly in the data model
---
.../generator/base/datasource/DataSource.java | 35 ++++++++-----
.../base/datasource/DataSourceFactory.java | 5 +-
.../datasource/DataSourceFactoryTest.java | 4 +-
.../data-models.md} | 2 +-
.../src/site/markdown/cli/concepts/named-uris.md | 59 ++++++++++++++++++++++
.../src/site/markdown/index.md | 7 ++-
.../freemarker/generator/cli/ManualTest.java | 4 +-
7 files changed, 97 insertions(+), 19 deletions(-)
diff --git
a/freemarker-generator-base/src/main/java/org/apache/freemarker/generator/base/datasource/DataSource.java
b/freemarker-generator-base/src/main/java/org/apache/freemarker/generator/base/datasource/DataSource.java
index 35cac8e..dd055f2 100644
---
a/freemarker-generator-base/src/main/java/org/apache/freemarker/generator/base/datasource/DataSource.java
+++
b/freemarker-generator-base/src/main/java/org/apache/freemarker/generator/base/datasource/DataSource.java
@@ -31,10 +31,8 @@ import java.io.StringWriter;
import java.net.URI;
import java.nio.charset.Charset;
import java.util.List;
-import java.util.Objects;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
-import java.util.stream.Stream;
import static java.nio.charset.Charset.forName;
import static java.nio.charset.StandardCharsets.UTF_8;
@@ -42,12 +40,16 @@ import static java.util.Objects.requireNonNull;
import static org.apache.commons.io.IOUtils.lineIterator;
import static
org.apache.freemarker.generator.base.FreeMarkerConstants.DATASOURCE_UNKNOWN_LENGTH;
import static
org.apache.freemarker.generator.base.util.StringUtils.emptyToNull;
-import static
org.apache.freemarker.generator.base.util.StringUtils.firstNonEmpty;
+import static org.apache.freemarker.generator.base.util.StringUtils.isNotEmpty;
/**
* Data source which encapsulates data to be used for rendering
* a template. When accessing content it is loaded on demand on not
* kept in memory to allow processing of large volumes of data.
+ * <b/>
+ * There is also special support of <code>UrlDataSource</code> since
+ * the content type & charset might be determined using a network
+ * call.
*/
public class DataSource implements Closeable {
@@ -108,14 +110,16 @@ public class DataSource implements Closeable {
}
public Charset getCharset() {
- return Stream.of(charset,
parseCharsetFromContentType(getContentType()))
- .filter(Objects::nonNull)
- .findFirst()
- .orElse(UTF_8);
+ return charset != null ? charset :
getCharsetFromContentType(contentType(), UTF_8);
}
+ /**
+ * Get the content type without additional parameters, e.g. "charset".
+ *
+ * @return content type
+ */
public String getContentType() {
- return firstNonEmpty(contentType, dataSource.getContentType());
+ return stripExtraParameterFronContentType(contentType());
}
public URI getUri() {
@@ -258,17 +262,24 @@ public class DataSource implements Closeable {
"name='" + name + '\'' +
", group='" + group + '\'' +
", uri=" + uri +
- ", contentType='" + contentType + '\'' +
- ", charset=" + charset +
'}';
}
- private Charset parseCharsetFromContentType(String contentType) {
+ private Charset getCharsetFromContentType(String contentType, Charset def)
{
final Matcher matcher = CHARSET_PATTERN.matcher(contentType);
if (matcher.find()) {
final String name = matcher.group(1).trim().toUpperCase();
return Charset.forName(name);
}
- return null;
+ return def;
+ }
+
+ private String contentType() {
+ return isNotEmpty(contentType) ? contentType :
dataSource.getContentType();
+ }
+
+ private String stripExtraParameterFronContentType(String contentType) {
+ final int end = contentType.indexOf(";");
+ return end > 0 ? contentType.substring(0, end) : contentType;
}
}
diff --git
a/freemarker-generator-base/src/main/java/org/apache/freemarker/generator/base/datasource/DataSourceFactory.java
b/freemarker-generator-base/src/main/java/org/apache/freemarker/generator/base/datasource/DataSourceFactory.java
index 4b5d976..d3b84e0 100644
---
a/freemarker-generator-base/src/main/java/org/apache/freemarker/generator/base/datasource/DataSourceFactory.java
+++
b/freemarker-generator-base/src/main/java/org/apache/freemarker/generator/base/datasource/DataSourceFactory.java
@@ -62,7 +62,7 @@ public class DataSourceFactory {
public static DataSource fromNamedUri(NamedUri namedUri) {
final URI uri = namedUri.getUri();
final String group = namedUri.getGroupOrElse(DEFAULT_GROUP);
- final Charset charset = getCharsetOrElse(namedUri, UTF_8);
+ final Charset charset = getCharsetOrElse(namedUri, null);
final String mimeType = getMimeTypeOrElse(namedUri, null);
if (UriUtils.isHttpURI(uri)) {
@@ -196,7 +196,8 @@ public class DataSourceFactory {
}
private static Charset getCharsetOrElse(NamedUri namedUri, Charset def) {
- return Charset.forName(namedUri.getParameter(NamedUri.CHARSET,
def.name()));
+ final String charsetName = namedUri.getParameter(NamedUri.CHARSET);
+ return StringUtils.isEmpty(charsetName) ? def :
Charset.forName(charsetName);
}
private static URL toURL(URI uri) {
diff --git
a/freemarker-generator-base/src/test/java/org/apache/freemarker/generator/datasource/DataSourceFactoryTest.java
b/freemarker-generator-base/src/test/java/org/apache/freemarker/generator/datasource/DataSourceFactoryTest.java
index 035566e..57029df 100644
---
a/freemarker-generator-base/src/test/java/org/apache/freemarker/generator/datasource/DataSourceFactoryTest.java
+++
b/freemarker-generator-base/src/test/java/org/apache/freemarker/generator/datasource/DataSourceFactoryTest.java
@@ -108,7 +108,7 @@ public class DataSourceFactoryTest {
final DataSource dataSource =
DataSourceFactory.fromUrl("jsonplaceholder.typicode.com", "default", url, null,
null);
assertEquals("jsonplaceholder.typicode.com", dataSource.getName());
- assertEquals("application/json; charset=utf-8",
dataSource.getContentType());
+ assertEquals("application/json", dataSource.getContentType());
assertEquals(UTF_8, dataSource.getCharset());
}
@@ -119,7 +119,7 @@ public class DataSourceFactoryTest {
assertEquals(namedUri.getName(), dataSource.getName());
assertEquals(namedUri.getGroup(), dataSource.getGroup());
- assertEquals(UTF_8, dataSource.getCharset());
+ assertEquals("ISO-8859-1", dataSource.getCharset().toString());
assertEquals(namedUri.getUri().toString(),
dataSource.getUri().toString());
}
}
diff --git
a/freemarker-generator-cli/src/site/markdown/cli/working-with-datamodels.md
b/freemarker-generator-cli/src/site/markdown/cli/concepts/data-models.md
similarity index 98%
rename from
freemarker-generator-cli/src/site/markdown/cli/working-with-datamodels.md
rename to freemarker-generator-cli/src/site/markdown/cli/concepts/data-models.md
index 6b46397..c05323c 100644
--- a/freemarker-generator-cli/src/site/markdown/cli/working-with-datamodels.md
+++ b/freemarker-generator-cli/src/site/markdown/cli/concepts/data-models.md
@@ -1,4 +1,4 @@
-## Working With DataModels
+## DataModels
A `DataModel` is an eagerly loaded `DataSource` available in Apache
FreeMarker's model (context) when rendering a template.
diff --git
a/freemarker-generator-cli/src/site/markdown/cli/concepts/named-uris.md
b/freemarker-generator-cli/src/site/markdown/cli/concepts/named-uris.md
new file mode 100644
index 0000000..2a2469e
--- /dev/null
+++ b/freemarker-generator-cli/src/site/markdown/cli/concepts/named-uris.md
@@ -0,0 +1,59 @@
+# Named URIs
+
+Named URIs allow to identify `DataSources` and pass additional information
+
+A Named URI consists of
+
+* an optional name
+* an URI or simple file name
+
+As a refresher, a URI is made up of the following components (inspired by
https://docs.gomplate.ca/datasources/)
+
+```
+ foo://[email protected]:8042/over/there?name=ferret#nose
+ \_/ \_______________________/\_________/ \_________/ \__/
+ | | | | |
+scheme authority path query fragment
+```
+
+For our purposes, the scheme and the path components are especially important,
though the other components are used by certain datasources for particular
purposes.
+
+| Component | Purpose |
+|-----------|---------|
+| scheme | All datasources require a scheme (except for file when using
relative paths) |
+| authority | Used only by remote datasources, and can be omitted in some
of those cases. Consists of userinfo (user:pass), host, and port. |
+| path | Can be omitted, but usually used as the basis of the locator for
the datasource. |
+| query | Used mainly for HTTP and HTTPS URLs |
+| fragment | Used rarely for providing additional attributes, e.g.
`mimetype` of `charset` |
+
+The following Named URI loads a "user.csv" and the data source is available as
`my_users`
+
+```
+bin/freemarker-cli -t templates/info.ftl my_users=site/sample/csv/user.csv
+[#1], name=my_users, group=default, contentType=text/csv, charset=UTF-8,
length=376 Bytes
+URI : file:site/sample/csv/user.csv
+```
+
+A Named URI allows to pass additional information as part of the fragment,
e.g. the charset of the text file
+
+```
+bin/freemarker-cli -t templates/info.ftl
my_users=site/sample/csv/user.csv#charset=UTF-16
+[#1], name=my_users, group=default, contentType=text/csv, charset=UTF-16,
length=376 Bytes
+URI : file:site/sample/csv/user.csv
+```
+
+In addition to the simplified file syntax full URIs can be used
+
+```
+bin/freemarker-cli -t templates/info.ftl http://google.com?foo=bar
+[#1], name=google.com, group=default, contentType=text/html,
charset=ISO-8859-1, length=-1 Bytes
+URI : http://google.com?foo=bar
+```
+
+and also combined with a name
+
+```
+bin/freemarker-cli -t templates/info.ftl page=http://google.com?foo=bar
+[#1], name=page, group=default, contentType=text/html, charset=ISO-8859-1,
length=-1 Bytes
+URI : http://google.com?foo=bar
+```
diff --git a/freemarker-generator-cli/src/site/markdown/index.md
b/freemarker-generator-cli/src/site/markdown/index.md
index 2fd9f95..c5807bc 100644
--- a/freemarker-generator-cli/src/site/markdown/index.md
+++ b/freemarker-generator-cli/src/site/markdown/index.md
@@ -1 +1,6 @@
-TBD
\ No newline at end of file
+## FreeMarker Generator CLI
+
+### Concepts
+
+* [Named URIs](cli/concepts/named-uris.html)
+* [Data Models](cli/concepts/data-models.html)
\ No newline at end of file
diff --git
a/freemarker-generator-cli/src/test/java/org/apache/freemarker/generator/cli/ManualTest.java
b/freemarker-generator-cli/src/test/java/org/apache/freemarker/generator/cli/ManualTest.java
index 59952de..0dac268 100644
---
a/freemarker-generator-cli/src/test/java/org/apache/freemarker/generator/cli/ManualTest.java
+++
b/freemarker-generator-cli/src/test/java/org/apache/freemarker/generator/cli/ManualTest.java
@@ -45,9 +45,11 @@ public class ManualTest {
// private static final String CMD = "-b ./src/test -t templates/demo.ftl
-m env=env:///";
// private static final String CMD = "-b ./src/test -t templates/demo.ftl
-m api=https://httpbin.org/get";;
// private static final String CMD = "-b ./src/test -t templates/demo.ftl
-m env:///HOME";
- private static final String CMD = "-b ./src/test -t templates/demo.ftl -m
env=./site/sample/properties/user_0001/user.properties";
+ // private static final String CMD = "-b ./src/test -t templates/demo.ftl
-m env=./site/sample/properties/user_0001/user.properties";
// private static final String CMD = "-b ./src/test -t templates/demo.ftl
-m ./site/sample/properties/user_0001/user.properties";
// private static final String CMD = "-b ./src/test --data-model
post=https://jsonplaceholder.typicode.com/posts/2 -t templates/info.ftl";
+ private static final String CMD = "-b ./src/test -t templates/info.ftl
google=https://www.google.com";;
+
public static void main(String[] args) {
Main.execute(toArgs(CMD));
