This is an automated email from the ASF dual-hosted git repository.
sgoeschl pushed a commit to branch FREEMARKER-144
in repository https://gitbox.apache.org/repos/asf/freemarker-generator.git
The following commit(s) were added to refs/heads/FREEMARKER-144 by this push:
new 2fa0f73 FREEMARKER-144 Proof Of Concept for providing DataFrames
2fa0f73 is described below
commit 2fa0f73b757182c3dbb4126ac50f73206f6f136b
Author: Siegfried Goeschl <[email protected]>
AuthorDate: Mon Jun 1 23:00:49 2020 +0200
FREEMARKER-144 Proof Of Concept for providing DataFrames
---
freemarker-generator-cli/README.md | 2 +-
freemarker-generator-cli/run-samples.sh | 7 +++++++
.../site/sample/csv/data_join_a.csv | 5 -----
.../site/sample/csv/data_join_b.csv | 5 -----
.../site/sample/csv/dataframe.csv | 10 +++++++++
.../src/main/scripts/run-samples.sh | 9 +++++++-
.../freemarker/generator/cli/ExamplesTest.java | 5 +++++
.../freemarker/generator/cli/ManualTest.java | 3 +--
.../templates/dataframe/example.ftl | 3 +--
.../generator/tools/commonscsv/CommonsCSVTool.java | 24 ++++++++++++++++++++--
.../tools/dataframe/DataFrameToolTest.java | 13 +++++++-----
11 files changed, 63 insertions(+), 23 deletions(-)
diff --git a/freemarker-generator-cli/README.md
b/freemarker-generator-cli/README.md
index 587dadb..6c6ee23 100644
--- a/freemarker-generator-cli/README.md
+++ b/freemarker-generator-cli/README.md
@@ -1119,7 +1119,7 @@ Sometimes we simply need to transform a JSON into an
equivalent YAML or the othe
> ./bin/freemarker-cli -i '${GsonTool.toJson(yaml)}' -m
> yaml=site/sample/yaml/swagger-spec.yaml
> ./bin/freemarker-cli -t templates/json/yaml/transform.ftl
> site/sample/json/swagger-spec.json
-> ./bin/freemarker-cli -i
'${YamlTool.toYaml(GsonTool.parse(DataSources.get(0)))}'
site/sample/json/swagger-spec.json
+> ./bin/freemarker-cli -i
'${YamlTool.toYaml(GsonTool.toMap(DataSources.get(0)))}'
site/sample/json/swagger-spec.json
> ./bin/freemarker-cli -i '${YamlTool.toYaml(json)}' -m
> json=site/sample/json/swagger-spec.json
```
diff --git a/freemarker-generator-cli/run-samples.sh
b/freemarker-generator-cli/run-samples.sh
index 99a64d4..cdc3d6a 100755
--- a/freemarker-generator-cli/run-samples.sh
+++ b/freemarker-generator-cli/run-samples.sh
@@ -101,6 +101,13 @@ if hash wkhtmltopdf 2>/dev/null; then
fi
#############################################################################
+# DataFrame
+#############################################################################
+
+echo "templates/dataframe/example.ftl"
+$FREEMARKER_CMD -DCSV_TOOL_DELIMITER=SEMICOLON -DCSV_TOOL_HEADERS=true -t
templates/dataframe/example.ftl site/sample/csv/dataframe.csv >
target/out/dataframe.txt || { echo >&2 "Test failed. Aborting."; exit 1; }
+
+#############################################################################
# Grok
#############################################################################
diff --git a/freemarker-generator-cli/site/sample/csv/data_join_a.csv
b/freemarker-generator-cli/site/sample/csv/data_join_a.csv
deleted file mode 100644
index cc05775..0000000
--- a/freemarker-generator-cli/site/sample/csv/data_join_a.csv
+++ /dev/null
@@ -1,5 +0,0 @@
-GENE_ID;FPKM;CHR
-A;5;1
-B;4;2
-C;6;3
-D;6;1
\ No newline at end of file
diff --git a/freemarker-generator-cli/site/sample/csv/data_join_b.csv
b/freemarker-generator-cli/site/sample/csv/data_join_b.csv
deleted file mode 100644
index c84d1a0..0000000
--- a/freemarker-generator-cli/site/sample/csv/data_join_b.csv
+++ /dev/null
@@ -1,5 +0,0 @@
-TRANSCRIPT_ID;GENE_ID;FPKM;TRANSCRIPT_NUMBER
-TA;A;7;1
-TB;A;3;2
-TC;B;6;1
-TD;E;4;1
\ No newline at end of file
diff --git a/freemarker-generator-cli/site/sample/csv/dataframe.csv
b/freemarker-generator-cli/site/sample/csv/dataframe.csv
new file mode 100644
index 0000000..b841ef9
--- /dev/null
+++ b/freemarker-generator-cli/site/sample/csv/dataframe.csv
@@ -0,0 +1,10 @@
+name;age;country
+Schmitt;24;Germany
+Parker;45;USA
+Meier;20;Germany
+Schmitt;30;France
+Peter;44;Germany
+Meier;24;Germany
+Green;33;UK
+Schmitt;30;Germany
+Meier;30;Germany
\ No newline at end of file
diff --git a/freemarker-generator-cli/src/main/scripts/run-samples.sh
b/freemarker-generator-cli/src/main/scripts/run-samples.sh
index c005f09..ee8571e 100755
--- a/freemarker-generator-cli/src/main/scripts/run-samples.sh
+++ b/freemarker-generator-cli/src/main/scripts/run-samples.sh
@@ -50,7 +50,7 @@ $FREEMARKER_CMD -i
'${JsonPathTool.parse(DataSources.first).read("$.info.title")
$FREEMARKER_CMD -i
'${XmlTool.parse(DataSources.first)["recipients/person[1]/name"]}'
site/sample/xml/recipients.xml > target/out/interactive-xml.txt || { echo >&2
"Test failed. Aborting."; exit 1; }
$FREEMARKER_CMD -i '${JsoupTool.parse(DataSources.first).select("a")[0]}'
site/sample/html/dependencies.html > target/out/interactive-html.txt || { echo
>&2 "Test failed. Aborting."; exit 1; }
$FREEMARKER_CMD -i '${GsonTool.toJson(YamlTool.parse(DataSources.get(0)))}'
site/sample/yaml/swagger-spec.yaml > target/out/interactive-swagger.json || {
echo >&2 "Test failed. Aborting."; exit 1; }
-$FREEMARKER_CMD -i '${YamlTool.toYaml(GsonTool.parse(DataSources.get(0)))}'
site/sample/json/swagger-spec.json > target/out/interactive-swagger.yaml || {
echo >&2 "Test failed. Aborting."; exit 1; }
+$FREEMARKER_CMD -i '${YamlTool.toYaml(GsonTool.toMap(DataSources.get(0)))}'
site/sample/json/swagger-spec.json > target/out/interactive-swagger.yaml || {
echo >&2 "Test failed. Aborting."; exit 1; }
#############################################################################
# CSV
@@ -101,6 +101,13 @@ if hash wkhtmltopdf 2>/dev/null; then
fi
#############################################################################
+# DataFrame
+#############################################################################
+
+echo "templates/dataframe/example.ftl"
+$FREEMARKER_CMD -DCSV_TOOL_DELIMITER=SEMICOLON -DCSV_TOOL_HEADERS=true -t
templates/dataframe/example.ftl site/sample/csv/dataframe.csv >
target/out/dataframe.txt || { echo >&2 "Test failed. Aborting."; exit 1; }
+
+#############################################################################
# Grok
#############################################################################
diff --git
a/freemarker-generator-cli/src/test/java/org/apache/freemarker/generator/cli/ExamplesTest.java
b/freemarker-generator-cli/src/test/java/org/apache/freemarker/generator/cli/ExamplesTest.java
index dc48323..a595aee 100644
---
a/freemarker-generator-cli/src/test/java/org/apache/freemarker/generator/cli/ExamplesTest.java
+++
b/freemarker-generator-cli/src/test/java/org/apache/freemarker/generator/cli/ExamplesTest.java
@@ -103,6 +103,11 @@ public class ExamplesTest extends AbstractMainTest {
}
@Test
+ public void shouldRunDataFrameExamples() throws IOException {
+ assertValid(execute("-DCSV_TOOL_DELIMITER=SEMICOLON
-DCSV_TOOL_HEADERS=true -t templates/dataframe/example.ftl
site/sample/csv/dataframe.csv"));
+ }
+
+ @Test
public void shouldRunInteractiveTemplateExamples() throws IOException {
assertValid(execute("-i
${JsonPathTool.parse(DataSources.first).read(\"$.info.title\")}
site/sample/json/swagger-spec.json"));
assertValid(execute("-i
${XmlTool.parse(DataSources.first)[\"recipients/person[1]/name\"]}
site/sample/xml/recipients.xml"));
diff --git
a/freemarker-generator-cli/src/test/java/org/apache/freemarker/generator/cli/ManualTest.java
b/freemarker-generator-cli/src/test/java/org/apache/freemarker/generator/cli/ManualTest.java
index acdfd85..fd2c11c 100644
---
a/freemarker-generator-cli/src/test/java/org/apache/freemarker/generator/cli/ManualTest.java
+++
b/freemarker-generator-cli/src/test/java/org/apache/freemarker/generator/cli/ManualTest.java
@@ -48,8 +48,7 @@ public class ManualTest {
// private static final String CMD = "-b ./src/test -t templates/demo.ftl
-m env=./site/sample/properties/user_0001/user.properties";
// private static final String CMD = "-b ./src/test -t templates/demo.ftl
-m ./site/sample/properties/user_0001/user.properties";
// private static final String CMD = "-b ./src/test --data-model
post=https://jsonplaceholder.typicode.com/posts/2 -t templates/info.ftl";
- // private static final String CMD = "-b ./src/test -t
templates/dataframe/html/print.ftl ./site/sample/csv/data_join_a.csv
./site/sample/csv/data_join_b.csv";
- private static final String CMD = "-b ./src/test -t
templates/dataframe/example.ftl
https://raw.githubusercontent.com/nRo/DataFrame/master/src/test/resources/users.csv";;
+ private static final String CMD = "-DCSV_TOOL_DELIMITER=SEMICOLON
-DCSV_TOOL_HEADERS=true -b ./src/test -t templates/dataframe/example.ftl
https://raw.githubusercontent.com/nRo/DataFrame/master/src/test/resources/users.csv";;
public static void main(String[] args) {
diff --git a/freemarker-generator-cli/templates/dataframe/example.ftl
b/freemarker-generator-cli/templates/dataframe/example.ftl
index 7983b53..8acda9c 100644
--- a/freemarker-generator-cli/templates/dataframe/example.ftl
+++ b/freemarker-generator-cli/templates/dataframe/example.ftl
@@ -15,8 +15,7 @@
specific language governing permissions and limitations
under the License.
-->
-<#assign cvsFormat =
CSVTool.formats["DEFAULT"].withHeader().withDelimiter(';')>
-<#assign csvParser = CSVTool.parse(DataSources.get(0), cvsFormat)>
+<#assign csvParser = CSVTool.parse(DataSources.get(0))>
<#assign users = DataFrameTool.toDataFrame(csvParser)>
Original Data
diff --git
a/freemarker-generator-tools/src/main/java/org/apache/freemarker/generator/tools/commonscsv/CommonsCSVTool.java
b/freemarker-generator-tools/src/main/java/org/apache/freemarker/generator/tools/commonscsv/CommonsCSVTool.java
index 31c5d7c..296f903 100644
---
a/freemarker-generator-tools/src/main/java/org/apache/freemarker/generator/tools/commonscsv/CommonsCSVTool.java
+++
b/freemarker-generator-tools/src/main/java/org/apache/freemarker/generator/tools/commonscsv/CommonsCSVTool.java
@@ -37,6 +37,7 @@ import java.util.Map;
import java.util.function.Function;
import java.util.stream.Collectors;
+import static java.lang.Boolean.parseBoolean;
import static java.nio.charset.StandardCharsets.UTF_8;
import static java.util.Objects.requireNonNull;
import static java.util.stream.Collectors.toList;
@@ -44,8 +45,10 @@ import static org.apache.commons.io.IOUtils.toInputStream;
public class CommonsCSVTool {
+ private final CSVFormat defaulCSVFormat = csvFormat();
+
public CSVParser parse(DataSource dataSource) {
- return parse(dataSource, CSVFormat.DEFAULT);
+ return parse(dataSource, defaulCSVFormat);
}
public CSVParser parse(DataSource dataSource, CSVFormat format) {
@@ -65,7 +68,7 @@ public class CommonsCSVTool {
}
public CSVParser parse(String csv) {
- return parse(csv, CSVFormat.DEFAULT);
+ return parse(csv, defaulCSVFormat);
}
public CSVParser parse(String csv, CSVFormat format) {
@@ -259,6 +262,23 @@ public class CommonsCSVTool {
return result;
}
+ private CSVFormat csvFormat() {
+
+ CSVFormat csvFormat =
CSVFormat.valueOf(System.getProperty("CSV_TOOL_FORMAT", "Default"));
+
+ final String delimiter = System.getProperty("CSV_TOOL_DELIMITER");
+ if (StringUtils.isNotEmpty(delimiter)) {
+ csvFormat = csvFormat.withDelimiter(toDelimiter(delimiter));
+ }
+
+ final boolean withHeader =
parseBoolean(System.getProperty("CSV_TOOL_HEADERS",
Boolean.toString(!csvFormat.getSkipHeaderRecord())));
+ if (withHeader) {
+ csvFormat = csvFormat.withHeader();
+ }
+
+ return csvFormat;
+ }
+
private static final class ValueResolver implements Function<CSVRecord,
String> {
private final Integer index;
diff --git
a/freemarker-generator-tools/src/test/java/org/apache/freemarker/generator/tools/dataframe/DataFrameToolTest.java
b/freemarker-generator-tools/src/test/java/org/apache/freemarker/generator/tools/dataframe/DataFrameToolTest.java
index 735e927..101a3fd 100644
---
a/freemarker-generator-tools/src/test/java/org/apache/freemarker/generator/tools/dataframe/DataFrameToolTest.java
+++
b/freemarker-generator-tools/src/test/java/org/apache/freemarker/generator/tools/dataframe/DataFrameToolTest.java
@@ -35,8 +35,11 @@ import static org.apache.commons.csv.CSVFormat.DEFAULT;
public class DataFrameToolTest {
- private static final File DATA_JOIN_A = new
File("./src/test/data/csv/data_join_a.csv");
- private static final File DATA_JOIN_B = new
File("./src/test/data/csv/data_join_b.csv");
+ private static final String CSV_WITH_HEADER = "GENE_ID;FPKM;CHR\n" +
+ "A;5;1\n" +
+ "B;4;2\n" +
+ "C;6;3\n" +
+ "D;6;1";
private static final String JSON_ARRAY = "[\n" +
" {\n" +
@@ -63,7 +66,7 @@ public class DataFrameToolTest {
@Test
public void shouldParseCsvFileWithHeader() {
- final CSVParser csvParser = csvParser(DATA_JOIN_A,
DEFAULT.withHeader().withDelimiter(';'));
+ final CSVParser csvParser = csvParser(CSV_WITH_HEADER,
DEFAULT.withHeader().withDelimiter(';'));
final DataFrame dataFrame = dataFrameTool().toDataFrame(csvParser);
assertEquals(3, dataFrame.getColumns().size());
@@ -100,7 +103,7 @@ public class DataFrameToolTest {
return DataSourceFactory.fromFile(file, UTF_8);
}
- private CSVParser csvParser(File file, CSVFormat csvFormat) {
- return commonsCSVTool().parse(dataSource(file), csvFormat);
+ private CSVParser csvParser(String csv, CSVFormat csvFormat) {
+ return commonsCSVTool().parse(csv, csvFormat);
}
}
