This is an automated email from the ASF dual-hosted git repository.
sgoeschl pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/freemarker-generator.git
The following commit(s) were added to refs/heads/master by this push:
new 27fd68c FREEMARKER-173 [freemarker-generator] Allow to pass arbitrary
key/value pairs to DataSource when using NamedURIs (#29)
27fd68c is described below
commit 27fd68cb7534f4677aee416e3d542a929e5fa5cd
Author: Siegfried Goeschl <[email protected]>
AuthorDate: Tue Feb 9 00:34:51 2021 +0100
FREEMARKER-173 [freemarker-generator] Allow to pass arbitrary key/value
pairs to DataSource when using NamedURIs (#29)
---
.../generator/base/datasource/DataSource.java | 29 ++++++-
.../base/datasource/DataSourceFactory.java | 68 ++++++++++-----
.../base/datasource/DataSourceLoader.java | 15 +---
.../base/datasource/DataSourcesSupplier.java | 20 +++--
.../datasource/loader/DefaultDataSourceLoader.java | 7 --
.../loader/EnvironmentDataSourceLoader.java | 12 +--
.../datasource/loader/FileDataSourceLoader.java | 20 ++---
.../datasource/loader/HttpDataSourceLoader.java | 16 ++--
.../freemarker/generator/base/uri/NamedUri.java | 87 ++++++++++++-------
.../datasource/DataSourceFactoryTest.java | 5 +-
.../generator/datasource/DataSourceLoaderTest.java | 14 ++--
.../generator/datasource/DataSourceTest.java | 23 ++---
.../datasource/DataSourcesSupplierTest.java | 24 +++++-
.../generator/datasource/DataSourcesTest.java | 4 +-
.../TemplateTransformationsBuilderTest.java | 2 +
.../generator/uri/NamedUriStringParserTest.java | 7 +-
.../generator/util/PropertiesTransformerTest.java | 4 -
freemarker-generator-cli/CHANGELOG.md | 2 +
.../examples/templates/datasources.ftl} | 44 +++++++---
.../src/app/scripts/run-examples.bat | 9 +-
.../src/app/scripts/run-examples.sh | 9 +-
.../freemarker-generator/lib/commons-csv.ftl | 18 ++++
.../cli/config/OutputGeneratorsSupplier.java | 3 +-
.../src/site/markdown/cli/concepts/data-sources.md | 36 +++++++-
.../src/site/markdown/cli/concepts/named-uris.md | 98 +++++++++++++++++++---
.../freemarker/generator/cli/ExamplesTest.java | 5 ++
.../freemarker/generator/cli/ManualTest.java | 2 +-
.../src/test/templates/manual.ftl | 26 ++++--
28 files changed, 433 insertions(+), 176 deletions(-)
diff --git
a/freemarker-generator-base/src/main/java/org/apache/freemarker/generator/base/datasource/DataSource.java
b/freemarker-generator-base/src/main/java/org/apache/freemarker/generator/base/datasource/DataSource.java
index 0bb4b57..5d5d42d 100644
---
a/freemarker-generator-base/src/main/java/org/apache/freemarker/generator/base/datasource/DataSource.java
+++
b/freemarker-generator-base/src/main/java/org/apache/freemarker/generator/base/datasource/DataSource.java
@@ -35,6 +35,7 @@ import java.io.StringWriter;
import java.net.URI;
import java.nio.charset.Charset;
import java.util.Arrays;
+import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
@@ -90,28 +91,44 @@ public class DataSource implements Closeable,
javax.activation.DataSource {
/** The underlying "javax.activation.DataSource" */
private final javax.activation.DataSource dataSource;
- /** Content type of data source either provided by the user or fetched
directly from the data source */
+ /** Content type of data source either provided by the caller or fetched
directly from the data source */
private final String contentType;
/** Charset for directly accessing text-based content */
private final Charset charset;
+ /** Additional properties as name/value pairs */
+ private final Map<String, String> properties;
+
/** Collect all closeables handed out to the caller to be closed when the
data source is closed itself */
private final CloseableReaper closeables;
+ /**
+ * Constructor.
+ *
+ * @param name Human-readable name of the data source
+ * @param group Optional group of data source
+ * @param uri source URI of the data source
+ * @param dataSource JAF data source being wrapped
+ * @param contentType content type of data source either provided by the
caller or fetched directly from the data source
+ * @param charset option charset for directly accessing text-based
content
+ * @param properties optional name/value pairs
+ */
public DataSource(
String name,
String group,
URI uri,
javax.activation.DataSource dataSource,
String contentType,
- Charset charset) {
- this.name = requireNonNull(name);
+ Charset charset,
+ Map<String, String> properties) {
+ this.name = requireNonNull(name).trim();
this.group = StringUtils.emptyToNull(group);
this.uri = requireNonNull(uri);
this.dataSource = requireNonNull(dataSource);
this.contentType = contentType;
this.charset = charset;
+ this.properties = properties != null ? new HashMap<>(properties) : new
HashMap<>();
this.closeables = new CloseableReaper();
}
@@ -206,6 +223,10 @@ public class DataSource implements Closeable,
javax.activation.DataSource {
return uri;
}
+ public Map<String, String> getProperties() {
+ return properties;
+ }
+
/**
* Try to get the length lazily, efficient and without consuming the input
stream.
*
@@ -339,7 +360,7 @@ public class DataSource implements Closeable,
javax.activation.DataSource {
case METADATA_MIME_TYPE:
return getMimeType();
default:
- throw new IllegalArgumentException("Unknown key: " + key);
+ throw new IllegalArgumentException("Unknown metatdata key: " +
key);
}
}
diff --git
a/freemarker-generator-base/src/main/java/org/apache/freemarker/generator/base/datasource/DataSourceFactory.java
b/freemarker-generator-base/src/main/java/org/apache/freemarker/generator/base/datasource/DataSourceFactory.java
index c9467a8..6930e91 100644
---
a/freemarker-generator-base/src/main/java/org/apache/freemarker/generator/base/datasource/DataSourceFactory.java
+++
b/freemarker-generator-base/src/main/java/org/apache/freemarker/generator/base/datasource/DataSourceFactory.java
@@ -36,6 +36,8 @@ import java.net.MalformedURLException;
import java.net.URI;
import java.net.URL;
import java.nio.charset.Charset;
+import java.util.HashMap;
+import java.util.Map;
import java.util.Properties;
import java.util.UUID;
@@ -57,18 +59,39 @@ public abstract class DataSourceFactory {
String group,
URI uri,
javax.activation.DataSource dataSource,
+ Map<String, String> properties) {
+ return new DataSource(name, group, uri, dataSource, null, null,
properties);
+ }
+
+ public static DataSource create(
+ String name,
+ String group,
+ URI uri,
+ javax.activation.DataSource dataSource,
String contentType,
- Charset charset
- ) {
- return new DataSource(name, group, uri, dataSource, contentType,
charset);
+ Charset charset,
+ Map<String, String> properties) {
+ return new DataSource(name, group, uri, dataSource, contentType,
charset, properties);
}
// == URL ===============================================================
- public static DataSource fromUrl(String name, String group, URL url,
String contentType, Charset charset) {
+ public static DataSource fromUrl(String name, String group, URL url) {
+ final URLDataSource dataSource = new CachingUrlDataSource(url);
+ final URI uri = UriUtils.toUri(url);
+ return create(name, group, uri, dataSource, noProperties());
+ }
+
+ public static DataSource fromUrl(
+ String name,
+ String group,
+ URL url,
+ String contentType,
+ Charset charset,
+ Map<String, String> properties) {
final URLDataSource dataSource = new CachingUrlDataSource(url);
final URI uri = UriUtils.toUri(url);
- return create(name, group, uri, dataSource, contentType, charset);
+ return create(name, group, uri, dataSource, contentType, charset,
properties);
}
// == String ============================================================
@@ -76,22 +99,28 @@ public abstract class DataSourceFactory {
public static DataSource fromString(String name, String group, String
content, String contentType) {
final StringDataSource dataSource = new StringDataSource(name,
content, contentType, UTF_8);
final URI uri = UriUtils.toUri(Location.STRING,
UUID.randomUUID().toString());
- return create(name, group, uri, dataSource, contentType, UTF_8);
+ return create(name, group, uri, dataSource, contentType, UTF_8,
noProperties());
}
// == File ==============================================================
public static DataSource fromFile(File file, Charset charset) {
- return fromFile(file.getName(), DEFAULT_GROUP, file, charset);
+ return fromFile(file.getName(), DEFAULT_GROUP, file, charset,
noProperties());
}
- public static DataSource fromFile(String name, String group, File file,
Charset charset) {
+ public static DataSource fromFile(
+ String name,
+ String group,
+ File file,
+ Charset charset,
+ Map<String, String> properties) {
Validate.isTrue(file.exists(), "File not found: " + file);
final FileDataSource dataSource = new FileDataSource(file);
+ // content type is determined from file extension
dataSource.setFileTypeMap(MimetypesFileTypeMapFactory.create());
final String contentType = dataSource.getContentType();
- return create(name, group, file.toURI(), dataSource, contentType,
charset);
+ return create(name, group, file.toURI(), dataSource, contentType,
charset, properties);
}
// == Bytes ============================================================
@@ -99,26 +128,21 @@ public abstract class DataSourceFactory {
public static DataSource fromBytes(String name, String group, byte[]
content, String contentType) {
final ByteArrayDataSource dataSource = new ByteArrayDataSource(name,
content);
final URI uri = UriUtils.toUri(Location.BYTES + ":///");
- return create(name, group, uri, dataSource, contentType, UTF_8);
+ return create(name, group, uri, dataSource, contentType, UTF_8,
noProperties());
}
// == InputStream =======================================================
- public static DataSource fromInputStream(String name, String group,
InputStream is, String contentType, Charset charset) {
- final URI uri = UriUtils.toUri(Location.INPUTSTREAM + ":///");
- return fromInputStream(name, group, uri, is, contentType, charset);
- }
-
public static DataSource fromInputStream(
String name,
String group,
URI uri,
InputStream is,
String contentType,
- Charset charset
- ) {
+ Charset charset,
+ Map<String, String> properties) {
final InputStreamDataSource dataSource = new
InputStreamDataSource(name, is);
- return create(name, group, uri, dataSource, contentType, charset);
+ return create(name, group, uri, dataSource, contentType, charset,
properties);
}
// == Environment =======================================================
@@ -130,7 +154,7 @@ public abstract class DataSourceFactory {
properties.store(writer, null);
final StringDataSource dataSource = new StringDataSource(name,
writer.toString(), contentType, UTF_8);
final URI uri = UriUtils.toUri(Location.ENVIRONMENT, "");
- return create(name, group, uri, dataSource, contentType, UTF_8);
+ return create(name, group, uri, dataSource, contentType, UTF_8,
noProperties());
} catch (IOException e) {
throw new RuntimeException("Unable to load environment variables",
e);
}
@@ -141,7 +165,7 @@ public abstract class DataSourceFactory {
final StringDataSource dataSource = new StringDataSource(name,
System.getenv(key), contentType, UTF_8);
final URI uri = UriUtils.toUri(Location.ENVIRONMENT, key);
- return create(name, group, uri, dataSource, contentType, UTF_8);
+ return create(name, group, uri, dataSource, contentType, UTF_8,
noProperties());
}
public static URL toUrl(String url) {
@@ -151,4 +175,8 @@ public abstract class DataSourceFactory {
throw new IllegalArgumentException(url, e);
}
}
+
+ private static Map<String, String> noProperties() {
+ return new HashMap<>();
+ }
}
diff --git
a/freemarker-generator-base/src/main/java/org/apache/freemarker/generator/base/datasource/DataSourceLoader.java
b/freemarker-generator-base/src/main/java/org/apache/freemarker/generator/base/datasource/DataSourceLoader.java
index 423b648..4a22137 100644
---
a/freemarker-generator-base/src/main/java/org/apache/freemarker/generator/base/datasource/DataSourceLoader.java
+++
b/freemarker-generator-base/src/main/java/org/apache/freemarker/generator/base/datasource/DataSourceLoader.java
@@ -16,14 +16,12 @@
*/
package org.apache.freemarker.generator.base.datasource;
-import java.nio.charset.Charset;
-
public interface DataSourceLoader {
/**
- * Check if the source would be accepted
+ * Check if the data source can be loaded by this instance.
*
- * @param source source
+ * @param source source to be loaded from
* @return true if the instance wold be able to load a data source
*/
boolean accept(String source);
@@ -35,13 +33,4 @@ public interface DataSourceLoader {
* @return DataSource
*/
DataSource load(String source);
-
- /**
- * Load a DataSource using the given charset.
- *
- * @param source source of the data source
- * @param charset charset to use
- * @return DataSource
- */
- DataSource load(String source, Charset charset);
}
diff --git
a/freemarker-generator-base/src/main/java/org/apache/freemarker/generator/base/datasource/DataSourcesSupplier.java
b/freemarker-generator-base/src/main/java/org/apache/freemarker/generator/base/datasource/DataSourcesSupplier.java
index c2604ba..cc919fc 100644
---
a/freemarker-generator-base/src/main/java/org/apache/freemarker/generator/base/datasource/DataSourcesSupplier.java
+++
b/freemarker-generator-base/src/main/java/org/apache/freemarker/generator/base/datasource/DataSourcesSupplier.java
@@ -27,16 +27,17 @@ import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
+import java.util.Map;
import java.util.function.Supplier;
import static java.util.Collections.singletonList;
import static java.util.Objects.requireNonNull;
import static java.util.stream.Collectors.toList;
import static
org.apache.freemarker.generator.base.FreeMarkerConstants.DEFAULT_GROUP;
+import static
org.apache.freemarker.generator.base.datasource.DataSourceFactory.fromFile;
/**
- * Create a list of <code>DataSource</code> based on a list of sources
consisting of
- * URIs, directories and files.
+ * Create a list of <code>DataSource</code> based on a list URIs, directories
and files.
*/
public class DataSourcesSupplier implements Supplier<List<DataSource>> {
@@ -57,14 +58,14 @@ public class DataSourcesSupplier implements
Supplier<List<DataSource>> {
/**
* Constructor.
*
- * @param sources List of source files and/or directories
+ * @param sources List of source files and/or directories supporting
<code>NamedUri</code> syntax
* @param include Optional include pattern for resolving source files or
directory
* @param exclude Optional exclude pattern for resolving source files or
directory
* @param charset The charset for loading text files
*/
public DataSourcesSupplier(Collection<String> sources, String include,
String exclude, Charset charset) {
this.dataSourceLoader = DataSourceLoaderFactory.create();
- this.sources = new ArrayList<>(sources);
+ this.sources = new ArrayList<>(requireNonNull(sources));
this.include = include;
this.exclude = exclude;
this.charset = requireNonNull(charset);
@@ -111,10 +112,11 @@ public class DataSourcesSupplier implements
Supplier<List<DataSource>> {
private static List<DataSource> resolveFileOrDirectory(String source,
String include, String exclude, Charset charset) {
final NamedUri namedUri = NamedUriStringParser.parse(source);
final String path = namedUri.getFile().getPath();
- final String group = namedUri.getGroupOrElse(DEFAULT_GROUP);
- final Charset currCharset = getCharsetOrElse(namedUri, charset);
+ final String group = namedUri.getGroupOrDefault(DEFAULT_GROUP);
+ final Charset currCharset = getCharsetOrDefault(namedUri, charset);
+ final Map<String, String> parameters = namedUri.getParameters();
return fileSupplier(path, include, exclude).get().stream()
- .map(file ->
DataSourceFactory.fromFile(getDataSourceName(namedUri, file), group, file,
currCharset))
+ .map(file -> fromFile(getDataSourceName(namedUri, file),
group, file, currCharset, parameters))
.collect(toList());
}
@@ -122,8 +124,8 @@ public class DataSourcesSupplier implements
Supplier<List<DataSource>> {
return new RecursiveFileSupplier(singletonList(source),
singletonList(include), singletonList(exclude));
}
- private static Charset getCharsetOrElse(NamedUri namedUri, Charset def) {
- return Charset.forName(namedUri.getParameter(NamedUri.CHARSET,
def.name()));
+ private static Charset getCharsetOrDefault(NamedUri namedUri, Charset def)
{
+ return namedUri.getCharsetOrDefault(def);
}
private static boolean isHttpUri(String value) {
diff --git
a/freemarker-generator-base/src/main/java/org/apache/freemarker/generator/base/datasource/loader/DefaultDataSourceLoader.java
b/freemarker-generator-base/src/main/java/org/apache/freemarker/generator/base/datasource/loader/DefaultDataSourceLoader.java
index 467531c..2067aa2 100644
---
a/freemarker-generator-base/src/main/java/org/apache/freemarker/generator/base/datasource/loader/DefaultDataSourceLoader.java
+++
b/freemarker-generator-base/src/main/java/org/apache/freemarker/generator/base/datasource/loader/DefaultDataSourceLoader.java
@@ -19,7 +19,6 @@ package
org.apache.freemarker.generator.base.datasource.loader;
import org.apache.freemarker.generator.base.datasource.DataSource;
import org.apache.freemarker.generator.base.datasource.DataSourceLoader;
-import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.List;
@@ -47,16 +46,10 @@ public class DefaultDataSourceLoader implements
DataSourceLoader {
return get(source).load(source);
}
- @Override
- public DataSource load(String source, Charset charset) {
- return get(source).load(source, charset);
- }
-
private DataSourceLoader get(String source) {
return dataSourceLoaders.stream()
.filter(loader -> loader.accept(source))
.findFirst()
.orElseThrow(() -> new IllegalArgumentException("Don't know
how to load: " + source));
}
-
}
diff --git
a/freemarker-generator-base/src/main/java/org/apache/freemarker/generator/base/datasource/loader/EnvironmentDataSourceLoader.java
b/freemarker-generator-base/src/main/java/org/apache/freemarker/generator/base/datasource/loader/EnvironmentDataSourceLoader.java
index 41eeeef..8d194ff 100644
---
a/freemarker-generator-base/src/main/java/org/apache/freemarker/generator/base/datasource/loader/EnvironmentDataSourceLoader.java
+++
b/freemarker-generator-base/src/main/java/org/apache/freemarker/generator/base/datasource/loader/EnvironmentDataSourceLoader.java
@@ -25,8 +25,6 @@ import org.apache.freemarker.generator.base.uri.NamedUri;
import org.apache.freemarker.generator.base.uri.NamedUriStringParser;
import org.apache.freemarker.generator.base.util.StringUtils;
-import java.nio.charset.Charset;
-
import static
org.apache.freemarker.generator.base.FreeMarkerConstants.DEFAULT_GROUP;
import static
org.apache.freemarker.generator.base.util.StringUtils.firstNonEmpty;
import static org.apache.freemarker.generator.base.util.StringUtils.isNotEmpty;
@@ -47,9 +45,9 @@ public class EnvironmentDataSourceLoader implements
DataSourceLoader {
public DataSource load(String source) {
final NamedUri namedUri = NamedUriStringParser.parse(source);
final String key = stripRootDir(namedUri.getUri().getPath());
- final String contentType =
namedUri.getMimeTypeOrElse(Mimetypes.MIME_TEXT_PLAIN);
+ final String contentType =
namedUri.getMimeTypeOrDefault(Mimetypes.MIME_TEXT_PLAIN);
final String name = firstNonEmpty(namedUri.getName(), key,
Location.ENVIRONMENT);
- final String group = namedUri.getGroupOrElse(DEFAULT_GROUP);
+ final String group = namedUri.getGroupOrDefault(DEFAULT_GROUP);
if (StringUtils.isEmpty(key)) {
return DataSourceFactory.fromEnvironment(name, group, contentType);
} else {
@@ -57,12 +55,6 @@ public class EnvironmentDataSourceLoader implements
DataSourceLoader {
}
}
- @Override
- public DataSource load(String source, Charset charset) {
- // We already habe internal strings so we can ignore the charset
- return load(source);
- }
-
/**
* Environment variables come with a leading "/" to be removed.
*/
diff --git
a/freemarker-generator-base/src/main/java/org/apache/freemarker/generator/base/datasource/loader/FileDataSourceLoader.java
b/freemarker-generator-base/src/main/java/org/apache/freemarker/generator/base/datasource/loader/FileDataSourceLoader.java
index 1ee8b66..2d890f3 100644
---
a/freemarker-generator-base/src/main/java/org/apache/freemarker/generator/base/datasource/loader/FileDataSourceLoader.java
+++
b/freemarker-generator-base/src/main/java/org/apache/freemarker/generator/base/datasource/loader/FileDataSourceLoader.java
@@ -26,6 +26,7 @@ import org.apache.freemarker.generator.base.util.UriUtils;
import java.io.File;
import java.nio.charset.Charset;
+import java.util.Map;
import static java.nio.charset.StandardCharsets.UTF_8;
import static org.apache.freemarker.generator.base.util.StringUtils.isNotEmpty;
@@ -40,20 +41,11 @@ public class FileDataSourceLoader implements
DataSourceLoader {
@Override
public DataSource load(String source) {
final NamedUri namedUri = NamedUriStringParser.parse(source);
- final String group =
namedUri.getGroupOrElse(FreeMarkerConstants.DEFAULT_GROUP);
- final Charset charset = namedUri.getCharsetOrElse(UTF_8);
+ final String group =
namedUri.getGroupOrDefault(FreeMarkerConstants.DEFAULT_GROUP);
+ final Charset charset = namedUri.getCharsetOrDefault(UTF_8);
final File file = namedUri.getFile();
- final String name = namedUri.getNameOrElse(file.getName());
- return DataSourceFactory.fromFile(name, group, file, charset);
+ final String name = namedUri.getNameOrDefault(file.getName());
+ final Map<String, String> parameters = namedUri.getParameters();
+ return DataSourceFactory.fromFile(name, group, file, charset,
parameters);
}
-
- @Override
- public DataSource load(String source, Charset charset) {
- final NamedUri namedUri = NamedUriStringParser.parse(source);
- final String group =
namedUri.getGroupOrElse(FreeMarkerConstants.DEFAULT_GROUP);
- final File file = namedUri.getFile();
- final String name =
namedUri.getNameOrElse(UriUtils.toStringWithoutFragment(file.toURI()));
- return DataSourceFactory.fromFile(name, group, file, charset);
- }
-
}
diff --git
a/freemarker-generator-base/src/main/java/org/apache/freemarker/generator/base/datasource/loader/HttpDataSourceLoader.java
b/freemarker-generator-base/src/main/java/org/apache/freemarker/generator/base/datasource/loader/HttpDataSourceLoader.java
index 10755f9..aa3d8ad 100644
---
a/freemarker-generator-base/src/main/java/org/apache/freemarker/generator/base/datasource/loader/HttpDataSourceLoader.java
+++
b/freemarker-generator-base/src/main/java/org/apache/freemarker/generator/base/datasource/loader/HttpDataSourceLoader.java
@@ -27,6 +27,7 @@ import java.net.MalformedURLException;
import java.net.URI;
import java.net.URL;
import java.nio.charset.Charset;
+import java.util.Map;
import static
org.apache.freemarker.generator.base.FreeMarkerConstants.DEFAULT_GROUP;
import static org.apache.freemarker.generator.base.util.StringUtils.isNotEmpty;
@@ -42,18 +43,13 @@ public class HttpDataSourceLoader implements
DataSourceLoader {
public DataSource load(String source) {
final NamedUri namedUri = NamedUriStringParser.parse(source);
final URI uri = namedUri.getUri();
- final String group = namedUri.getGroupOrElse(DEFAULT_GROUP);
- final Charset charset = namedUri.getCharsetOrElse(null);
+ final String group = namedUri.getGroupOrDefault(DEFAULT_GROUP);
+ final Charset charset = namedUri.getCharset();
final String mimeType = namedUri.getMimeType();
final URL url = toUrl(uri);
- final String name =
namedUri.getNameOrElse(UriUtils.toStringWithoutFragment(uri));
- return DataSourceFactory.fromUrl(name, group, url, mimeType, charset);
- }
-
- @Override
- public DataSource load(String source, Charset charset) {
- // We should pick up the charset from the HTTP server
- return load(source);
+ final String name =
namedUri.getNameOrDefault(UriUtils.toStringWithoutFragment(uri));
+ final Map<String, String> parameters = namedUri.getParameters();
+ return DataSourceFactory.fromUrl(name, group, url, mimeType, charset,
parameters);
}
private static URL toUrl(URI uri) {
diff --git
a/freemarker-generator-base/src/main/java/org/apache/freemarker/generator/base/uri/NamedUri.java
b/freemarker-generator-base/src/main/java/org/apache/freemarker/generator/base/uri/NamedUri.java
index 6003d3c..ad496a8 100644
---
a/freemarker-generator-base/src/main/java/org/apache/freemarker/generator/base/uri/NamedUri.java
+++
b/freemarker-generator-base/src/main/java/org/apache/freemarker/generator/base/uri/NamedUri.java
@@ -17,23 +17,29 @@
package org.apache.freemarker.generator.base.uri;
import org.apache.freemarker.generator.base.util.StringUtils;
+import org.apache.freemarker.generator.base.util.Validate;
import java.io.File;
import java.net.URI;
import java.nio.charset.Charset;
+import java.util.HashMap;
import java.util.Map;
-import static java.util.Objects.requireNonNull;
-import static
org.apache.freemarker.generator.base.util.StringUtils.emptyToNull;
-import static org.apache.freemarker.generator.base.util.StringUtils.isEmpty;
-
/**
* Captures the information of a user-supplied "named URI".
+ * <p>
+ * <ul>
+ * <li><code>name</code> is optional</li>
+ * <li><code>group</code> is optional</li>
+ * </ul>
*/
public class NamedUri {
- public static final String CHARSET = "charset";
- public static final String MIMETYPE = "mimeType";
+ // Pre-defined parameter names
+ private static final String NAME_KEY = "name";
+ private static final String GROUP_KEY = "group";
+ private static final String CHARSET_KEY = "charset";
+ private static final String MIMETYPE_KEY = "mimeType";
/** User-supplied name */
private final String name;
@@ -47,34 +53,59 @@ public class NamedUri {
/** Name/value pairs parsed from URI fragment */
private final Map<String, String> parameters;
+ /**
+ * Constructor.
+ * <p>
+ * The <code>name</code> and <code>group</code> a read from
<code>parameters</code>.
+ *
+ * @param uri URI
+ * @param parameters map of parameters
+ */
public NamedUri(URI uri, Map<String, String> parameters) {
- this.name = null;
- this.group = null;
- this.uri = requireNonNull(uri);
- this.parameters = requireNonNull(parameters);
- }
+ Validate.notNull(uri, "uri is null");
+ Validate.notNull(parameters, "parameters are null");
+
+ this.uri = uri;
+ this.name = StringUtils.emptyToNull(parameters.get(NAME_KEY));
+ this.group = StringUtils.emptyToNull(parameters.get(GROUP_KEY));
+ this.parameters = new HashMap<>(parameters);
+ }
+
+ /**
+ * Constructor.
+ * <p>
+ * For empty <code>name</code> and <code>group</code> a fallback to
<code>parameters</code> is provided.
+ *
+ * @param name optional name of the named URI
+ * @param group optional group of the named URI
+ * @param uri URI
+ * @param parameters map of parameters
+ */
public NamedUri(String name, String group, URI uri, Map<String, String>
parameters) {
- this.name = emptyToNull(name);
- this.group = emptyToNull(group);
- this.uri = requireNonNull(uri);
- this.parameters = requireNonNull(parameters);
+ Validate.notNull(uri, "uri is null");
+ Validate.notNull(parameters, "parameters are null");
+
+ this.uri = uri;
+ this.name = StringUtils.firstNonEmpty(name, parameters.get(NAME_KEY));
+ this.group = StringUtils.firstNonEmpty(group,
parameters.get(GROUP_KEY));
+ this.parameters = new HashMap<>(parameters);
}
public String getName() {
return name;
}
- public String getNameOrElse(String def) {
- return isEmpty(name) ? def : name;
+ public String getNameOrDefault(String def) {
+ return StringUtils.isEmpty(name) ? def : name;
}
public String getGroup() {
return group;
}
- public String getGroupOrElse(String def) {
- return isEmpty(group) ? def : group;
+ public String getGroupOrDefault(String def) {
+ return StringUtils.isEmpty(group) ? def : group;
}
public URI getUri() {
@@ -89,16 +120,16 @@ public class NamedUri {
return parameters.get(key);
}
- public String getParameter(String key, String defaultValue) {
+ public String getParameterOrDefault(String key, String defaultValue) {
return parameters.getOrDefault(key, defaultValue);
}
public boolean hasName() {
- return !isEmpty(this.name);
+ return !StringUtils.isEmpty(this.name);
}
public boolean hasGroup() {
- return !isEmpty(this.group);
+ return !StringUtils.isEmpty(this.group);
}
public File getFile() {
@@ -106,20 +137,20 @@ public class NamedUri {
}
public String getMimeType() {
- return getParameter(NamedUri.MIMETYPE);
+ return getParameter(NamedUri.MIMETYPE_KEY);
}
- public String getMimeTypeOrElse(String def) {
- return getParameter(NamedUri.MIMETYPE, def);
+ public String getMimeTypeOrDefault(String def) {
+ return getParameterOrDefault(NamedUri.MIMETYPE_KEY, def);
}
public Charset getCharset() {
- final String charsetName = getParameter(NamedUri.CHARSET);
+ final String charsetName = getParameter(NamedUri.CHARSET_KEY);
return Charset.forName(charsetName);
}
- public Charset getCharsetOrElse(Charset def) {
- final String charsetName = getParameter(NamedUri.CHARSET);
+ public Charset getCharsetOrDefault(Charset def) {
+ final String charsetName = getParameter(NamedUri.CHARSET_KEY);
return StringUtils.isEmpty(charsetName) ? def :
Charset.forName(charsetName);
}
diff --git
a/freemarker-generator-base/src/test/java/org/apache/freemarker/generator/datasource/DataSourceFactoryTest.java
b/freemarker-generator-base/src/test/java/org/apache/freemarker/generator/datasource/DataSourceFactoryTest.java
index 37d1f92..20733c7 100644
---
a/freemarker-generator-base/src/test/java/org/apache/freemarker/generator/datasource/DataSourceFactoryTest.java
+++
b/freemarker-generator-base/src/test/java/org/apache/freemarker/generator/datasource/DataSourceFactoryTest.java
@@ -30,6 +30,7 @@ import java.io.InputStream;
import java.net.URI;
import java.net.URL;
import java.nio.charset.Charset;
+import java.util.HashMap;
import static java.nio.charset.StandardCharsets.UTF_8;
import static junit.framework.TestCase.assertFalse;
@@ -85,7 +86,7 @@ public class DataSourceFactoryTest {
public void shouldCreateDataSourceFromInputStream() {
final URI uri = UriUtils.toUri(Location.INPUTSTREAM + ":///");
final InputStream is = new
ByteArrayInputStream(ANY_TEXT.getBytes(UTF_8));
- final DataSource dataSource =
DataSourceFactory.fromInputStream("test.txt", "default", uri, is, "text/plain",
UTF_8);
+ final DataSource dataSource =
DataSourceFactory.fromInputStream("test.txt", "default", uri, is, "text/plain",
UTF_8, new HashMap<>());
assertEquals("test.txt", dataSource.getName());
assertEquals(UTF_8, dataSource.getCharset());
@@ -98,7 +99,7 @@ public class DataSourceFactoryTest {
@Ignore
public void shouldCreateDataSourceFromURL() throws IOException {
final URL url = new
URL("https://jsonplaceholder.typicode.com/posts/2";);
- final DataSource dataSource =
DataSourceFactory.fromUrl("jsonplaceholder.typicode.com", "default", url, null,
null);
+ final DataSource dataSource =
DataSourceFactory.fromUrl("jsonplaceholder.typicode.com", "default", url);
assertEquals("jsonplaceholder.typicode.com", dataSource.getName());
assertEquals("jsonplaceholder.typicode.com", dataSource.getFileName());
diff --git
a/freemarker-generator-base/src/test/java/org/apache/freemarker/generator/datasource/DataSourceLoaderTest.java
b/freemarker-generator-base/src/test/java/org/apache/freemarker/generator/datasource/DataSourceLoaderTest.java
index 1567699..2bc284e 100644
---
a/freemarker-generator-base/src/test/java/org/apache/freemarker/generator/datasource/DataSourceLoaderTest.java
+++
b/freemarker-generator-base/src/test/java/org/apache/freemarker/generator/datasource/DataSourceLoaderTest.java
@@ -26,7 +26,6 @@ import org.junit.Test;
import java.io.File;
import static java.lang.String.format;
-import static java.nio.charset.StandardCharsets.UTF_16;
import static java.nio.charset.StandardCharsets.UTF_8;
import static junit.framework.TestCase.assertFalse;
import static
org.apache.freemarker.generator.base.FreeMarkerConstants.DEFAULT_GROUP;
@@ -174,11 +173,14 @@ public class DataSourceLoaderTest {
@Test
public void shouldLoadDataSourceWithCharset() {
- final DataSource utf8DataSource =
dataSourceLoader().load("./src/test/data/txt/utf8.txt", UTF_8);
- final DataSource utf16DataSource =
dataSourceLoader().load("./src/test/data/txt/utf16.txt", UTF_16);
-
- // skip the first line before comparing
- assertEquals(utf8DataSource.getLines().subList(1, 5),
utf16DataSource.getLines().subList(1, 5));
+ final String utf8Uri = "./src/test/data/txt/utf8.txt#charset=UTF-8";
+ final String utf16Uri = "./src/test/data/txt/utf16.txt#charset=UTF-16";
+ try (DataSource utf8DataSource = dataSourceLoader().load(utf8Uri)) {
+ try (DataSource utf16DataSource =
dataSourceLoader().load(utf16Uri)) {
+ // skip the first line before comparing
+ assertEquals(utf8DataSource.getLines().subList(1, 5),
utf16DataSource.getLines().subList(1, 5));
+ }
+ }
}
private DataSourceLoader dataSourceLoader() {
diff --git
a/freemarker-generator-base/src/test/java/org/apache/freemarker/generator/datasource/DataSourceTest.java
b/freemarker-generator-base/src/test/java/org/apache/freemarker/generator/datasource/DataSourceTest.java
index 8961359..e1a8f80 100644
---
a/freemarker-generator-base/src/test/java/org/apache/freemarker/generator/datasource/DataSourceTest.java
+++
b/freemarker-generator-base/src/test/java/org/apache/freemarker/generator/datasource/DataSourceTest.java
@@ -28,6 +28,7 @@ import java.io.File;
import java.io.IOException;
import java.nio.charset.Charset;
import java.util.Iterator;
+import java.util.Map;
import static java.nio.charset.StandardCharsets.UTF_8;
import static
org.apache.freemarker.generator.base.FreeMarkerConstants.DEFAULT_GROUP;
@@ -84,7 +85,7 @@ public class DataSourceTest {
@Test
@Ignore("Requires internet access")
public void shouldSupportUrlDataSource() {
- try (DataSource dataSource =
DataSourceFactory.fromUrl("www.google.com", DEFAULT_GROUP,
toUrl("https://www.google.com/?foo=bar";), null, null)) {
+ try (DataSource dataSource =
DataSourceFactory.fromUrl("www.google.com", DEFAULT_GROUP,
toUrl("https://www.google.com/?foo=bar";), null, null, null)) {
assertEquals("www.google.com", dataSource.getName());
assertEquals(DEFAULT_GROUP, dataSource.getGroup());
assertEquals("", dataSource.getBaseName());
@@ -125,15 +126,17 @@ public class DataSourceTest {
@Test
public void shouldGetMetadata() {
try (DataSource dataSource = stringDataSource()) {
- assertEquals(8, dataSource.getMetadata().size());
- assertEquals("", dataSource.getMetadata().get("basename"));
- assertEquals("", dataSource.getMetadata().get("extension"));
- assertEquals("", dataSource.getMetadata().get("filename"));
- assertEquals("/", dataSource.getMetadata().get("filepath"));
- assertEquals("default", dataSource.getMetadata().get("group"));
- assertEquals("stdin", dataSource.getMetadata().get("name"));
-
assertTrue(dataSource.getMetadata().get("uri").startsWith("string://"));
- assertEquals("text/plain",
dataSource.getMetadata().get("mimetype"));
+ final Map<String, String> metadata = dataSource.getMetadata();
+
+ assertEquals(8, metadata.size());
+ assertEquals("", metadata.get("basename"));
+ assertEquals("", metadata.get("extension"));
+ assertEquals("", metadata.get("filename"));
+ assertEquals("/", metadata.get("filepath"));
+ assertEquals("default", metadata.get("group"));
+ assertEquals("stdin", metadata.get("name"));
+ assertTrue(metadata.get("uri").startsWith("string://"));
+ assertEquals("text/plain", metadata.get("mimetype"));
}
}
diff --git
a/freemarker-generator-base/src/test/java/org/apache/freemarker/generator/datasource/DataSourcesSupplierTest.java
b/freemarker-generator-base/src/test/java/org/apache/freemarker/generator/datasource/DataSourcesSupplierTest.java
index 779485c..1fb5fe7 100644
---
a/freemarker-generator-base/src/test/java/org/apache/freemarker/generator/datasource/DataSourcesSupplierTest.java
+++
b/freemarker-generator-base/src/test/java/org/apache/freemarker/generator/datasource/DataSourcesSupplierTest.java
@@ -35,6 +35,8 @@ public class DataSourcesSupplierTest {
private static final String NO_EXCLUDE = null;
private static final String DATA_DIRECTORY = "./src/test/data";
+ private static final String DATA_DIRECTORY_WITH_FRAGMENT =
"./src/test/data#scope=test";
+ private static final String DATA_DIRECTORY_WITH_GROUP =
":data=src/test/data";
private static final String PWD = FilenameUtils.separatorsToUnix(new
File("").getAbsolutePath());
@Test
@@ -106,6 +108,26 @@ public class DataSourcesSupplierTest {
}
@Test
+ public void shouldUseFragmentForDataSourceWhenResolvingDirectory() {
+ final List<DataSource> dataSources =
supplier(DATA_DIRECTORY_WITH_FRAGMENT, "*.*", NO_EXCLUDE).get();
+
+ for (DataSource dataSource : dataSources) {
+ assertEquals(1, dataSource.getProperties().size());
+ assertEquals("test", dataSource.getProperties().get("scope"));
+ }
+ }
+
+ @Test
+ public void shouldUseGroupNameForDataSourceWhenResolvingDirectory() {
+ final List<DataSource> dataSources =
supplier(DATA_DIRECTORY_WITH_GROUP, "*.*", NO_EXCLUDE).get();
+
+ for (DataSource dataSource : dataSources) {
+ assertEquals(0, dataSource.getProperties().size());
+ assertEquals("data", dataSource.getGroup());
+ }
+ }
+
+ @Test
public void shouldResolveEnvironmentVariable() {
assertEquals(1, supplier("env:///PATH", "*", NO_EXCLUDE).get().size());
assertEquals(1, supplier("path=env:///PATH", "*",
NO_EXCLUDE).get().size());
@@ -118,7 +140,7 @@ public class DataSourcesSupplierTest {
}
@Test(expected = RuntimeException.class)
- public void shouldThrowExceptionForNonexistingSourceDirectory() {
+ public void shouldThrowRuntimeExceptionForNonexistingSourceDirectory() {
assertEquals(0, supplier("/does-not-exist", "*", null).get().size());
}
diff --git
a/freemarker-generator-base/src/test/java/org/apache/freemarker/generator/datasource/DataSourcesTest.java
b/freemarker-generator-base/src/test/java/org/apache/freemarker/generator/datasource/DataSourcesTest.java
index 3591e38..030ea62 100644
---
a/freemarker-generator-base/src/test/java/org/apache/freemarker/generator/datasource/DataSourcesTest.java
+++
b/freemarker-generator-base/src/test/java/org/apache/freemarker/generator/datasource/DataSourcesTest.java
@@ -24,6 +24,7 @@ import org.junit.Test;
import java.io.File;
import java.net.MalformedURLException;
import java.net.URL;
+import java.util.HashMap;
import static java.nio.charset.StandardCharsets.UTF_8;
import static java.util.Arrays.asList;
@@ -32,7 +33,6 @@ import static
org.apache.freemarker.generator.base.FreeMarkerConstants.DEFAULT_G
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertTrue;
public class DataSourcesTest {
@@ -133,7 +133,7 @@ public class DataSourcesTest {
}
private static DataSource urlDataSource() {
- return DataSourceFactory.fromUrl("server.invalid?foo=bar", "default",
toUrl(ANY_URL), "plain/text", UTF_8);
+ return DataSourceFactory.fromUrl("server.invalid?foo=bar", "default",
toUrl(ANY_URL), "plain/text", UTF_8, new HashMap<>());
}
private static URL toUrl(String value) {
diff --git
a/freemarker-generator-base/src/test/java/org/apache/freemarker/generator/template/TemplateTransformationsBuilderTest.java
b/freemarker-generator-base/src/test/java/org/apache/freemarker/generator/template/TemplateTransformationsBuilderTest.java
index 822b66a..7170cd7 100644
---
a/freemarker-generator-base/src/test/java/org/apache/freemarker/generator/template/TemplateTransformationsBuilderTest.java
+++
b/freemarker-generator-base/src/test/java/org/apache/freemarker/generator/template/TemplateTransformationsBuilderTest.java
@@ -23,6 +23,7 @@ import
org.apache.freemarker.generator.base.template.TemplateSource.Origin;
import org.apache.freemarker.generator.base.template.TemplateTransformation;
import
org.apache.freemarker.generator.base.template.TemplateTransformationsBuilder;
import org.apache.freemarker.generator.base.util.NonClosableWriterWrapper;
+import org.junit.Ignore;
import org.junit.Test;
import java.io.BufferedWriter;
@@ -181,6 +182,7 @@ public class TemplateTransformationsBuilderTest {
// === Template URL ===============================================
@Test
+ @Ignore("Requires internet access")
public void shouldCreateFromTemplateUrl() {
final List<TemplateTransformation> transformations = builder()
.setTemplateSource(ANY_TEMPLATE_URL)
diff --git
a/freemarker-generator-base/src/test/java/org/apache/freemarker/generator/uri/NamedUriStringParserTest.java
b/freemarker-generator-base/src/test/java/org/apache/freemarker/generator/uri/NamedUriStringParserTest.java
index ebcb84e..01e68bb 100644
---
a/freemarker-generator-base/src/test/java/org/apache/freemarker/generator/uri/NamedUriStringParserTest.java
+++
b/freemarker-generator-base/src/test/java/org/apache/freemarker/generator/uri/NamedUriStringParserTest.java
@@ -240,13 +240,14 @@ public class NamedUriStringParserTest {
@Test
public void shouldParseNamedGroupUrlWithQueryAndFragment() {
- final NamedUri namedURI =
parse("google:web=http://google.com?foo=bar#charset=UTF-16";);
+ final NamedUri namedURI =
parse("google:web=http://google.com?foo=bar#charset=UTF-16&name=value";);
assertEquals("google", namedURI.getName());
assertEquals("web", namedURI.getGroup());
- assertEquals("http://google.com?foo=bar#charset=UTF-16";,
namedURI.getUri().toString());
- assertEquals(1, namedURI.getParameters().size());
+ assertEquals("http://google.com?foo=bar#charset=UTF-16&name=value";,
namedURI.getUri().toString());
+ assertEquals(2, namedURI.getParameters().size());
assertEquals("UTF-16", namedURI.getParameters().get("charset"));
+ assertEquals("value", namedURI.getParameters().get("name"));
}
@Test(expected = RuntimeException.class)
diff --git
a/freemarker-generator-base/src/test/java/org/apache/freemarker/generator/util/PropertiesTransformerTest.java
b/freemarker-generator-base/src/test/java/org/apache/freemarker/generator/util/PropertiesTransformerTest.java
index de0c483..5a116e0 100644
---
a/freemarker-generator-base/src/test/java/org/apache/freemarker/generator/util/PropertiesTransformerTest.java
+++
b/freemarker-generator-base/src/test/java/org/apache/freemarker/generator/util/PropertiesTransformerTest.java
@@ -18,8 +18,6 @@ package org.apache.freemarker.generator.util;
import org.junit.Test;
-import java.util.HashMap;
-import java.util.Map;
import java.util.Properties;
import static
org.apache.freemarker.generator.base.FreeMarkerConstants.Configuration.TOOLS_PREFIX;
@@ -29,8 +27,6 @@ import static org.junit.Assert.assertEquals;
public class PropertiesTransformerTest {
- private final Map<String, Object> settings = new HashMap<>();
-
@Test
public void shouldFilterKeyPrefix() {
final Properties properties = filterKeyPrefix(properties(),
TOOLS_PREFIX);
diff --git a/freemarker-generator-cli/CHANGELOG.md
b/freemarker-generator-cli/CHANGELOG.md
index c6a5cce..7dbdd16 100644
--- a/freemarker-generator-cli/CHANGELOG.md
+++ b/freemarker-generator-cli/CHANGELOG.md
@@ -20,6 +20,7 @@ All notable changes to this project will be documented in
this file. We try to a
* [FREEMARKER-129] Provide a `toString()` method for all tools
### Changed
+* [FREEMARKER-173] Allow to pass arbitrary key/value pairs to DataSource when
using NamedURIs
* [FREEMARKER-172] Use lower-case keys for DataSource metadata map
* [FREEMARKER-148] Make usage of "DataSources" more "Freemarker" like
* [FREEMARKER-155] Migrate the FTL code to terser dotter form
@@ -70,3 +71,4 @@ All notable changes to this project will be documented in
this file. We try to a
[FREEMARKER-164]: https://issues.apache.org/jira/browse/FREEMARKER-164
[FREEMARKER-168]: https://issues.apache.org/jira/browse/FREEMARKER-168
[FREEMARKER-172]: https://issues.apache.org/jira/browse/FREEMARKER-172
+[FREEMARKER-173]: https://issues.apache.org/jira/browse/FREEMARKER-175
diff --git a/freemarker-generator-cli/src/test/templates/manual.ftl
b/freemarker-generator-cli/src/app/examples/templates/datasources.ftl
similarity index 71%
copy from freemarker-generator-cli/src/test/templates/manual.ftl
copy to freemarker-generator-cli/src/app/examples/templates/datasources.ftl
index 3ad42c6..364a7ba 100644
--- a/freemarker-generator-cli/src/test/templates/manual.ftl
+++ b/freemarker-generator-cli/src/app/examples/templates/datasources.ftl
@@ -1,4 +1,4 @@
-<#ftl output_format="plainText" >
+<#ftl output_format="plainText">
<#--
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
@@ -16,43 +16,63 @@
under the License.
-->
Support FreeMarker Directives
----------------------------------------------------------------------------
-Has Content: ${dataSources?has_content?c}
-Nr. of Documents: ${dataSources?size}
+==============================================================================
+has_content: ${dataSources?has_content?c}
+size: ${dataSources?size}
Use FTL Array-style Access
----------------------------------------------------------------------------
-${dataSources?values[0].toString()}
-${dataSources?values?first.toString()}
+==============================================================================
+${dataSources?values[0].name}
+${dataSources?values?first.name}
Get Document Names As Keys
----------------------------------------------------------------------------
+==============================================================================
<#list dataSources?keys as name>
${name}<#lt>
</#list>
Iterate Over Names & DataSources
----------------------------------------------------------------------------
+==============================================================================
<#list dataSources as name, dataSource>
${name} => ${dataSource.uri}<#lt>
</#list>
+<#if dataSources?has_content>
+ <#list dataSources?values as dataSource>
+ <@writeDataSource dataSource/>
+ </#list>
+<#else>
+ No data sources found ...
+</#if>
+
+<#macro writeDataSource dataSource>
+
+${dataSource.name}
+==============================================================================
+
Invoke Arbitrary Methods On DataSource
---------------------------------------------------------------------------
-<#if dataSources?has_content>
<#assign dataSource=dataSources?values?first>
Name : ${dataSource.name}
+Group : ${dataSource.group}
Nr of lines : ${dataSource.lines?size}
Content Type : ${dataSource.contentType}
Charset : ${dataSource.charset}
Extension : ${dataSource.extension}
Nr of chars : ${dataSource.text?length}
Nr of bytes : ${dataSource.bytes?size}
-File name : ${dataSource.metadata["filename"]}
+File name : ${dataSource.fileName}
+URI schema : ${dataSource.uri.scheme}
Iterating Over Metadata Of A Datasource
---------------------------------------------------------------------------
<#list dataSource.metadata as name, value>
${name?right_pad(15)} : ${value}
</#list>
-</#if>
+
+Iterating Over Properties Of A Datasource
+---------------------------------------------------------------------------
+<#list dataSource.properties as name, value>
+${name?right_pad(15)} : ${value}
+</#list>
+</#macro>
diff --git a/freemarker-generator-cli/src/app/scripts/run-examples.bat
b/freemarker-generator-cli/src/app/scripts/run-examples.bat
index 6ef6f9a..5fc0e1c 100644
--- a/freemarker-generator-cli/src/app/scripts/run-examples.bat
+++ b/freemarker-generator-cli/src/app/scripts/run-examples.bat
@@ -37,6 +37,13 @@ REM
=========================================================================
echo "examples\templates\demo.ftl"
%FREEMARKER_CMD% -t examples\templates\demo.ftl README.md --output-encoding
CP1252 > target\out\demo.txt
+#############################################################################
+# DataSources
+#############################################################################
+
+echo "examples\templates\datasources.ftl"
+$FREEMARKER_CMD -t examples\templates\datasources.ftl -s :data=examples/data >
target\out\datasources.txt
+
REM =========================================================================
REM Interactive Mode
REM =========================================================================
@@ -177,4 +184,4 @@ echo
"templates\freemarker-generator\yaml\json\transform.ftl"
%FREEMARKER_CMD% -t freemarker-generator\yaml\json\transform.ftl
examples\data\yaml\swagger-spec.yaml > target\out\swagger-spec.json
echo "Created the following sample files in .\target\out"
-dir .\target\out
\ No newline at end of file
+dir .\target\out
diff --git a/freemarker-generator-cli/src/app/scripts/run-examples.sh
b/freemarker-generator-cli/src/app/scripts/run-examples.sh
index 034774f..d0f42b7 100755
--- a/freemarker-generator-cli/src/app/scripts/run-examples.sh
+++ b/freemarker-generator-cli/src/app/scripts/run-examples.sh
@@ -43,6 +43,13 @@ echo "examples/templates/demo.ftl"
$FREEMARKER_CMD -t examples/templates/demo.ftl README.md > target/out/demo.txt
|| { echo >&2 "Test failed. Aborting."; exit 1; }
#############################################################################
+# DataSources
+#############################################################################
+
+echo "examples/templates/datasources.ftl"
+$FREEMARKER_CMD -t examples/templates/datasources.ftl -s :data=examples/data >
target/out/datasources.txt || { echo >&2 "Test failed. Aborting."; exit 1; }
+
+#############################################################################
# Interactive Mode
#############################################################################
@@ -198,4 +205,4 @@ echo
"templates/freemarker-generator/yaml/json/transform.ftl"
$FREEMARKER_CMD -t freemarker-generator/yaml/json/transform.ftl
examples/data/yaml/swagger-spec.yaml > target/out/swagger-spec.json || { echo
>&2 "Test failed. Aborting."; exit 1; }
echo "Created the following sample files in ./target/out"
-ls -l ./target/out
\ No newline at end of file
+ls -l ./target/out
diff --git
a/freemarker-generator-cli/src/app/templates/freemarker-generator/lib/commons-csv.ftl
b/freemarker-generator-cli/src/app/templates/freemarker-generator/lib/commons-csv.ftl
index e8b0538..881a2f2 100644
---
a/freemarker-generator-cli/src/app/templates/freemarker-generator/lib/commons-csv.ftl
+++
b/freemarker-generator-cli/src/app/templates/freemarker-generator/lib/commons-csv.ftl
@@ -17,6 +17,24 @@
-->
<#---
+ Detemine the CSV format for reading a CSV files from properties of the
data source.
+
+ * format - see
https://commons.apache.org/proper/commons-csv/apidocs/org/apache/commons/csv/CSVFormat.html
+ * delimiter - symbolic name of delimiter, e.g. "COLON" or "SEMICOLON"
+ * header - whether the first rows are headers
+-->
+<#function dataSourceFormat dataSource>
+ <#assign format =
tools.csv.formats[dataSource.properties.format!"DEFAULT"]>
+ <#assign delimiter =
tools.csv.toDelimiter(dataSource.properties.delimiter!format.getDelimiter())>
+ <#assign withHeader = dataSource.properties.header!"true">
+ <#assign format = format.withDelimiter(delimiter)>
+ <#if withHeader?boolean>
+ <#assign format = format.withFirstRecordAsHeader()>
+ </#if>
+ <#return format>
+</#function>
+
+<#---
Detemine the CSV format for reading a CSV files using user-supplied
parameters from the data model.
diff --git
a/freemarker-generator-cli/src/main/java/org/apache/freemarker/generator/cli/config/OutputGeneratorsSupplier.java
b/freemarker-generator-cli/src/main/java/org/apache/freemarker/generator/cli/config/OutputGeneratorsSupplier.java
index 085ad60..059a9ab 100644
---
a/freemarker-generator-cli/src/main/java/org/apache/freemarker/generator/cli/config/OutputGeneratorsSupplier.java
+++
b/freemarker-generator-cli/src/main/java/org/apache/freemarker/generator/cli/config/OutputGeneratorsSupplier.java
@@ -31,6 +31,7 @@ import
org.apache.freemarker.generator.cli.picocli.TemplateSourceDefinition;
import java.net.URI;
import java.util.ArrayList;
import java.util.Collection;
+import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.function.Supplier;
@@ -136,6 +137,6 @@ public class OutputGeneratorsSupplier implements
Supplier<List<OutputGenerator>>
private static DataSource stdinDataSource() {
final URI uri = UriUtils.toUri(Location.SYSTEM, STDIN);
- return DataSourceFactory.fromInputStream(STDIN, DEFAULT_GROUP, uri,
System.in, MIME_TEXT_PLAIN, UTF_8);
+ return DataSourceFactory.fromInputStream(STDIN, DEFAULT_GROUP, uri,
System.in, MIME_TEXT_PLAIN, UTF_8, new HashMap<>());
}
}
diff --git
a/freemarker-generator-cli/src/site/markdown/cli/concepts/data-sources.md
b/freemarker-generator-cli/src/site/markdown/cli/concepts/data-sources.md
index b4205e3..7ad0daf 100644
--- a/freemarker-generator-cli/src/site/markdown/cli/concepts/data-sources.md
+++ b/freemarker-generator-cli/src/site/markdown/cli/concepts/data-sources.md
@@ -171,7 +171,7 @@ Charset : ${dataSource.charset}
Extension : ${dataSource.extension}
Nr of chars : ${dataSource.text?length}
Nr of bytes : ${dataSource.bytes?size}
-File name : ${dataSource.metadata["filename"]}
+File name : ${dataSource.fileName}
Iterating Over Metadata Of A Datasource
---------------------------------------------------------------------------
@@ -207,4 +207,38 @@ uri :
file:/Users/sgoeschl/work/github/apache/freemarker-generator/f
group : default
```
+### Inspecting A DataSource
+
+```
+> freemarker-generator -t examples/templates/datasources.ftl
user:csv=examples/data/csv/transactions.csv#delimiter=TAB
+
+Invoke Arbitrary Methods On DataSource
+---------------------------------------------------------------------------
+Name : user
+Group : csv
+Nr of lines : 101
+Content Type : text/csv
+Charset : UTF-8
+Extension : csv
+Nr of chars : 12,643
+Nr of bytes : 12,643
+File name : transactions.csv
+URI schema : file
+
+Iterating Over Metadata Of A Datasource
+---------------------------------------------------------------------------
+extension : csv
+basename : transactions
+filename : transactions.csv
+filepath :
/Users/sgoeschl/work/github/apache/freemarker-generator/freemarker-generator-cli/target/appassembler/examples/data/csv
+name : user
+mimetype : text/csv
+uri :
file:/Users/sgoeschl/work/github/apache/freemarker-generator/freemarker-generator-cli/target/appassembler/examples/data/csv/transactions.csv
+group : csv
+
+Iterating Over Properties Of A Datasource
+---------------------------------------------------------------------------
+delimiter : TAB
+```
+
diff --git
a/freemarker-generator-cli/src/site/markdown/cli/concepts/named-uris.md
b/freemarker-generator-cli/src/site/markdown/cli/concepts/named-uris.md
index 55bf420..f6a80a3 100644
--- a/freemarker-generator-cli/src/site/markdown/cli/concepts/named-uris.md
+++ b/freemarker-generator-cli/src/site/markdown/cli/concepts/named-uris.md
@@ -9,7 +9,7 @@ A Named URI consists of
As a refresher, a URI is made up of the following components (inspired by
https://docs.gomplate.ca/datasources/)
-```
+```text
foo://[email protected]:8042/over/there?name=ferret#nose
\_/ \_______________________/\_________/ \_________/ \__/
| | | | |
@@ -20,15 +20,17 @@ For our purposes, the scheme and the path components are
especially important, t
| Component | Purpose
|
|-----------|-----------------------------------------------------------------------------------------------------------|
-| scheme | All data sources require a scheme (except for file when using
relative paths) |
-| authority | Used only by remote data sources, and can be omitted in some
of those cases. |
+| scheme | All data sources require a scheme (except for file when using
relative paths) |
+| authority | Used only by remote data sources, and can be omitted in some
of those cases. |
| path | Can be omitted, but usually used as the basis of the locator for
the datasource. |
| query | Used mainly for HTTP and HTTPS URLs
|
| fragment | Used rarely for providing additional attributes, e.g.
`mimeType` of `charset` |
-The following Named URI loads a "user.csv" and the data source is available as
`my_users`
+### Using Named URIs For A File
-```
+The following Named URI loads a "user.csv" and the data source is available as
`my_users`
+
+```text
freemarker-generator -t freemarker-generator/info.ftl
my_users=examples/data/csv/user.csv
FreeMarker Generator DataSources
@@ -39,7 +41,7 @@ URI :
file:/Users/sgoeschl/work/github/apache/freemarker-generator/freemarker-ge
A Named URI allows to pass additional information as part of the fragment,
e.g. the charset of the text file
-```
+```text
freemarker-generator -t freemarker-generator/info.ftl
my_users=examples/data/csv/user.csv#charset=UTF-16
FreeMarker Generator DataSources
@@ -50,8 +52,8 @@ URI :
file:/Users/sgoeschl/work/github/apache/freemarker-generator/freemarker-ge
In addition to the simplified file syntax full URIs can be used
-```
-freemarker-generator -t freemarker-generator/info.ftl http://google.com?foo=bar
+```text
+freemarker-generator -t freemarker-generator/info.ftl
'http://google.com?foo=bar'
FreeMarker Generator DataSources
------------------------------------------------------------------------------
@@ -61,11 +63,87 @@ URI : http://google.com?foo=bar
and also combined with a name
-```
-freemarker-generator -t freemarker-generator/info.ftl
page=http://google.com\?foo\=bar
+```text
+freemarker-generator -t freemarker-generator/info.ftl
'page=http://google.com?foo=bar'
FreeMarker Generator DataSources
------------------------------------------------------------------------------
[#1]: name=page, group=default, fileName=page mimeType=text/html,
charset=ISO-8859-1, length=-1 Bytes
URI : http://google.com?foo=bar
```
+
+### Using Named URIs For Directories
+
+A Name URI can be also combined with file directories.
+
+Load all CVS files of a directory using the group "csv"
+
+```text
+freemarker-generator -t freemarker-generator/info.ftl :csv=examples/data/csv
+
+FreeMarker Generator DataSources
+------------------------------------------------------------------------------
+[#1]:
name=file:/Users/sgoeschl/work/github/apache/freemarker-generator/freemarker-generator-cli/target/appassembler/examples/data/csv/contract.csv,
group=csv, fileName=contract.csv, mimeType=text/csv, charset=UTF-8,
length=6,328 Bytes
+URI :
file:/Users/sgoeschl/work/github/apache/freemarker-generator/freemarker-generator-cli/target/appassembler/examples/data/csv/contract.csv
+...
+[#7]:
name=file:/Users/sgoeschl/work/github/apache/freemarker-generator/freemarker-generator-cli/target/appassembler/examples/data/csv/user.csv,
group=csv, fileName=user.csv, mimeType=text/csv, charset=UTF-8, length=376
Bytes
+URI :
file:/Users/sgoeschl/work/github/apache/freemarker-generator/freemarker-generator-cli/target/appassembler/examples/data/csv/user.csv
+```
+
+or use a charset for all files of a directory
+
+```text
+freemarker-generator -t freemarker-generator/info.ftl
'examples/data/csv#charset=UTF-16&mimetype=text/plain'
+
+FreeMarker Generator DataSources
+------------------------------------------------------------------------------
+[#1]:
name=file:/Users/sgoeschl/work/github/apache/freemarker-generator/freemarker-generator-cli/target/appassembler/examples/data/csv/contract.csv,
group=default, fileName=contract.csv, mimeType=text/csv, charset=UTF-16,
length=6,328 Bytes
+URI :
file:/Users/sgoeschl/work/github/apache/freemarker-generator/freemarker-generator-cli/target/appassembler/examples/data/csv/contract.csv
+...
+[#7]:
name=file:/Users/sgoeschl/work/github/apache/freemarker-generator/freemarker-generator-cli/target/appassembler/examples/data/csv/user.csv,
group=default, fileName=user.csv, mimeType=text/csv, charset=UTF-16,
length=376 Bytes
+URI :
file:/Users/sgoeschl/work/github/apache/freemarker-generator/freemarker-generator-cli/target/appassembler/examples/data/csv/user.csv
+```
+
+It is also possible to provide data source properties to all files being loaded
+
+```text
+freemarker-generator -t examples/templates/datasources.ftl
'examples/data/csv#format=DEFAULT'
+
+file:/Users/sgoeschl/work/github/apache/freemarker-generator/freemarker-generator-cli/target/appassembler/examples/data/csv/user.csv
+==============================================================================
+
+Invoke Arbitrary Methods On DataSource
+---------------------------------------------------------------------------
+Name :
file:/Users/sgoeschl/work/github/apache/freemarker-generator/freemarker-generator-cli/target/appassembler/examples/data/csv/user.csv
+Group : default
+Nr of lines : 5
+Content Type : text/csv
+Charset : UTF-8
+Extension : csv
+Nr of chars : 376
+Nr of bytes : 376
+File name : user.csv
+URI schema : file
+
+Iterating Over Metadata Of A Datasource
+---------------------------------------------------------------------------
+extension : csv
+basename : user
+filename : user.csv
+filepath :
/Users/sgoeschl/work/github/apache/freemarker-generator/freemarker-generator-cli/target/appassembler/examples/data/csv
+name :
file:/Users/sgoeschl/work/github/apache/freemarker-generator/freemarker-generator-cli/target/appassembler/examples/data/csv/user.csv
+mimetype : text/csv
+uri :
file:/Users/sgoeschl/work/github/apache/freemarker-generator/freemarker-generator-cli/target/appassembler/examples/data/csv/user.csv
+group : default
+
+Iterating Over Properties Of A Datasource
+---------------------------------------------------------------------------
+format : DEFAULT
+
+```
+
+
+
+
+
+
diff --git
a/freemarker-generator-cli/src/test/java/org/apache/freemarker/generator/cli/ExamplesTest.java
b/freemarker-generator-cli/src/test/java/org/apache/freemarker/generator/cli/ExamplesTest.java
index 5c5ebd5..88067b2 100644
---
a/freemarker-generator-cli/src/test/java/org/apache/freemarker/generator/cli/ExamplesTest.java
+++
b/freemarker-generator-cli/src/test/java/org/apache/freemarker/generator/cli/ExamplesTest.java
@@ -48,6 +48,11 @@ public class ExamplesTest extends AbstractMainTest {
}
@Test
+ public void shouldRunDataSourceExamples() throws IOException {
+ assertValid(execute("-t src/app/examples/templates/datasources.ftl -s
:csv=src/app/examples/data/csv"));
+ }
+
+ @Test
public void shouldRunCsvExamples() throws IOException {
assertValid(execute("-t freemarker-generator/csv/html/transform.ftl
src/app/examples/data/csv/contract.csv"));
assertValid(execute("-t freemarker-generator/csv/md/transform.ftl
src/app/examples/data/csv/contract.csv"));
diff --git
a/freemarker-generator-cli/src/test/java/org/apache/freemarker/generator/cli/ManualTest.java
b/freemarker-generator-cli/src/test/java/org/apache/freemarker/generator/cli/ManualTest.java
index 54be597..1e6600a 100644
---
a/freemarker-generator-cli/src/test/java/org/apache/freemarker/generator/cli/ManualTest.java
+++
b/freemarker-generator-cli/src/test/java/org/apache/freemarker/generator/cli/ManualTest.java
@@ -24,7 +24,7 @@ import java.io.IOException;
public class ManualTest extends AbstractMainTest {
// private static final String CMD = "-V";
- private static final String CMD = "-t src/test/templates/manual.ftl -s
src/app/examples/data/csv";
+ private static final String CMD = "-t
src/app/examples/templates/datasources.ftl -s
:csv-data=src/app/examples/data/csv#separator=COLON
https://xkcd.com/info.0.json envvars=env:///";
@Override
public String execute(String commandLine) throws IOException {
diff --git a/freemarker-generator-cli/src/test/templates/manual.ftl
b/freemarker-generator-cli/src/test/templates/manual.ftl
index 3ad42c6..c1112be 100644
--- a/freemarker-generator-cli/src/test/templates/manual.ftl
+++ b/freemarker-generator-cli/src/test/templates/manual.ftl
@@ -1,4 +1,4 @@
-<#ftl output_format="plainText" >
+<#ftl output_format="plainText" strip_whitespace=true>
<#--
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
@@ -16,30 +16,34 @@
under the License.
-->
Support FreeMarker Directives
----------------------------------------------------------------------------
+==============================================================================
Has Content: ${dataSources?has_content?c}
Nr. of Documents: ${dataSources?size}
Use FTL Array-style Access
----------------------------------------------------------------------------
+==============================================================================
${dataSources?values[0].toString()}
${dataSources?values?first.toString()}
Get Document Names As Keys
----------------------------------------------------------------------------
+==============================================================================
<#list dataSources?keys as name>
${name}<#lt>
</#list>
Iterate Over Names & DataSources
----------------------------------------------------------------------------
+==============================================================================
<#list dataSources as name, dataSource>
${name} => ${dataSource.uri}<#lt>
</#list>
+<#if dataSources?has_content>
+ <#list dataSources?values as dataSource>
+[#${dataSource?counter}] - ${dataSource.name}
+==============================================================================
+
Invoke Arbitrary Methods On DataSource
---------------------------------------------------------------------------
-<#if dataSources?has_content>
<#assign dataSource=dataSources?values?first>
Name : ${dataSource.name}
Nr of lines : ${dataSource.lines?size}
@@ -55,4 +59,14 @@ Iterating Over Metadata Of A Datasource
<#list dataSource.metadata as name, value>
${name?right_pad(15)} : ${value}
</#list>
+
+Iterating Over Properties Of A Datasource
+---------------------------------------------------------------------------
+<#list dataSource.properties as name, value>
+${name?right_pad(15)} : ${value}
+</#list>
+
+ </#list>
+<#else>
+No data sources found ...
</#if>
