This is an automated email from the ASF dual-hosted git repository. sgoeschl pushed a commit to branch FREEMARKER-144 in repository https://gitbox.apache.org/repos/asf/freemarker-generator.git
commit eafb4eb02a2df438466ca50b22bfa9817234b02f Author: Siegfried Goeschl <[email protected]> AuthorDate: Sun Jun 14 22:11:19 2020 +0200 FREEMARKER-144 Proof Of Concept for providing DataFrames --- .../freemarker/generator/base/table/Table.java | 259 ++++++++++++++------- .../freemarker/generator/base/util/ArrayUtils.java | 72 ------ .../freemarker/generator/base/util/ListUtils.java | 76 ++++++ .../freemarker/generator/base/util/MapBuilder.java | 26 ++- .../freemarker/generator/table/TableTest.java | 118 ++++++++-- .../templates/dataframe/example.ftl | 6 +- .../generator/tools/dataframe/DataFrameTool.java | 30 ++- .../CSVConverter.java} | 4 +- .../tools/dataframe/converter/ConverterUtils.java | 106 +++++++++ .../tools/dataframe/converter/ListConverter.java | 21 ++ .../tools/dataframe/converter/MapConverter.java | 39 ++++ .../tools/dataframe/impl/MapConverter.java | 85 ------- .../generator/tools/excel/ExcelTool.java | 8 +- .../tools/dataframe/DataFrameToolTest.java | 27 ++- .../generator/tools/excel/ExcelToolTest.java | 8 +- 15 files changed, 598 insertions(+), 287 deletions(-) diff --git a/freemarker-generator-base/src/main/java/org/apache/freemarker/generator/base/table/Table.java b/freemarker-generator-base/src/main/java/org/apache/freemarker/generator/base/table/Table.java index 7530844..aaba268 100644 --- a/freemarker-generator-base/src/main/java/org/apache/freemarker/generator/base/table/Table.java +++ b/freemarker-generator-base/src/main/java/org/apache/freemarker/generator/base/table/Table.java @@ -1,167 +1,258 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.freemarker.generator.base.table; -import org.apache.freemarker.generator.base.util.ArrayUtils; -import org.apache.freemarker.generator.base.util.Validate; +import org.apache.freemarker.generator.base.util.ListUtils; -import java.util.Arrays; +import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.stream.Collectors; -import java.util.stream.IntStream; import static java.util.Objects.requireNonNull; /** - * Simple table model filled from a Map. + * Simple table model filled from maps or lists representing tabular data. */ public class Table { - private final String[] columnNames; - private final Class<?>[] columnTypes; - private final Object[][] values; + + /** List of column names */ + private final List<String> columnNames; + + /** Column types derived from tabular data */ + private final List<Class<?>> columnTypes; + + /** Table data as rows */ + private final List<List<Object>> values; + + /** Map column names to numeric column values */ private final Map<String, Integer> columnMap; - private Table(String[] columnNames, Class<?>[] columnTypes, Object[][] columnValuesList) { - this.columnNames = requireNonNull(columnNames); - this.columnTypes = requireNonNull(columnTypes); - this.values = ArrayUtils.transpose(requireNonNull(columnValuesList)); + private Table() { + this.columnNames = new ArrayList<>(); + this.columnTypes = new ArrayList<>(); + this.values = new ArrayList<>(); + this.columnMap = new HashMap<>(); + } + + private Table(Collection<String> columnNames, Collection<Class<?>> columnTypes, List<List<Object>> columnValuesList) { + this.columnNames = new ArrayList<>(requireNonNull(columnNames)); + this.columnTypes = new ArrayList<>(requireNonNull(columnTypes)); + this.values = ListUtils.transpose(requireNonNull(columnValuesList)); this.columnMap = new HashMap<>(); - for (int i = 0; i < this.columnNames.length; i++) { - this.columnMap.put(this.columnNames[i], i); + for (int i = 0; i < this.columnNames.size(); i++) { + this.columnMap.put(this.columnNames.get(i), i); } } - public String[] getColumnNames() { + public List<String> getColumnNames() { return columnNames; } - public Class<?>[] getColumnTypes() { + public List<Class<?>> getColumnTypes() { return columnTypes; } + public List<List<Object>> getValues() { + return values; + } + public int getNrOfColumns() { - return columnNames.length; + return columnNames.isEmpty() ? (values.isEmpty() ? 0 : values.get(0).size()) : columnNames.size(); + } + + public int size() { + return values.size(); + } + + public boolean isEmpty() { + return values.isEmpty(); + } + + public List<Object> getRow(int row) { + return values.get(row); } - public int getNrOfRows() { - return values.length; + public Object get(int row, int column) { + return values.get(row).get(column); } - public Object[] getRowValues(int row) { - return values[row]; + public Object get(int row, String column) { + return values.get(row).get(columnMap.get(column)); } - public Row getRow(int row) { - return new Row(columnMap, getRowValues(row)); + public boolean hasColumnHeaderRow() { + return !columnNames.isEmpty(); } + /** + * Create a table from a list of maps. Non-tabular data is supported, + * i.e. not all maps contains all possible keys. + * + * @param maps list of maps + * @return table + */ public static Table fromMaps(List<Map<String, Object>> maps) { - Validate.notNull(maps, "list is null"); + if (maps == null || maps.isEmpty()) { + return new Table(); + } final List<String> columnNames = columnNames(maps); - final Object[][] columnValuesList = columnValuesList(maps, columnNames); + final List<List<Object>> columnValuesList = columnValuesList(maps, columnNames); final List<Class<?>> columnTypes = columnTypes(columnValuesList); return new Table( - columnNames.toArray(new String[0]), - columnTypes.toArray(new Class[0]), + columnNames, + columnTypes, columnValuesList); } - public static Table fromLists(List<List<Object>> list) { - Validate.notNull(list, "list is null"); + /** + * Create a table from a list of lists representing tabular data. + * + * @param lists row values as lists + * @return table + */ + public static Table fromLists(List<List<Object>> lists) { + requireNonNull(lists, "lists is null"); - final List<String> columnNames = Arrays.asList(ArrayUtils.copy(list.toArray(new Object[0]))); - final Object[][] columnValuesList = columnValuesList(list.subList(1, list.size())); + final List<List<Object>> columnValuesList = ListUtils.transpose(lists); final List<Class<?>> columnTypes = columnTypes(columnValuesList); return new Table( - columnNames.toArray(new String[0]), - columnTypes.toArray(new Class[0]), + new ArrayList<>(), + columnTypes, columnValuesList); } - public static final class Row { - private final Map<String, Integer> columnMap; - private final Object[] values; - - Row(Map<String, Integer> columnMap, Object[] values) { - this.columnMap = columnMap; - this.values = values; + /** + * Create a table from a list of lists representing tabular data + * where the first row may consists of column headers. + * + * @param lists row values as lists + * @param withFirstRowAsColumnNames column names as first row? + * @return table + */ + public static Table fromLists(List<List<Object>> lists, boolean withFirstRowAsColumnNames) { + requireNonNull(lists, "lists is null"); + + if (withFirstRowAsColumnNames) { + final List<String> columnNames = columnNames(lists.get(0)); + final List<List<Object>> table = lists.subList(1, lists.size()); + return fromLists(columnNames, table); + } else { + return fromLists(lists); } + } - public Object[] getValues() { - return values; - } + /** + * Create a table from column names and row values. + * + * @param columnNames list of column names + * @param lists row values as lists + * @return table + */ + public static Table fromLists(Collection<String> columnNames, List<List<Object>> lists) { + requireNonNull(columnNames, "columnNames is null"); + requireNonNull(lists, "lists is null"); - public Object getValue(int column) { - return values[column]; - } + final List<List<Object>> columnValuesList = ListUtils.transpose(lists); + final List<Class<?>> columnTypes = columnTypes(columnValuesList); - public Object getValue(String column) { - return getValue(columnMap.get(column)); - } + return new Table( + new ArrayList<>(columnNames), + columnTypes, + columnValuesList); } - private static List<String> columnNames(List<Map<String, Object>> list) { - return list.stream() + /** + * Determine column names based on all available keys of the maps. + * + * @param maps list of maps + * @return column names + */ + private static List<String> columnNames(Collection<Map<String, Object>> maps) { + return maps.stream() .map(Map::keySet) .flatMap(Collection::stream) .distinct() .collect(Collectors.toList()); } - private static Object[][] columnValuesList(List<Map<String, Object>> list, List<String> columnNames) { - return columnNames.stream() - .map(columnName -> columnValues(list, columnName)) - .collect(Collectors.toList()) - .toArray(new Object[0][0]); + /** + * Determine column names based on a single list. + * + * @param list list of column names + * @return column names + */ + private static List<String> columnNames(List<Object> list) { + return list.stream() + .map(Object::toString) + .collect(Collectors.toList()); } - private static Object[][] columnValuesList(List<List<Object>> lists) { - if (lists.isEmpty()) { - return new Object[0][0]; - } - - return IntStream.range(0, lists.get(0).size()) - .mapToObj(i -> columnValues(lists, i)) - .collect(Collectors.toList()) - .toArray(new Object[0][0]); + /** + * Determine all column values. + * + * @param maps list of maps + * @param columnNames column names + * @return list of column values + */ + private static List<List<Object>> columnValuesList(List<Map<String, Object>> maps, List<String> columnNames) { + return columnNames.stream() + .map(columnName -> columnValues(maps, columnName)) + .collect(Collectors.toList()); } - private static Object[] columnValues(List<Map<String, Object>> maps, String columnName) { + /** + * Determine the values of a single column. + * + * @param maps list of maps + * @param columnName column name + * @return values of the given column + */ + private static List<Object> columnValues(List<Map<String, Object>> maps, String columnName) { return maps.stream() .map(map -> map.getOrDefault(columnName, null)) - .toArray(); - } - - private static Object[] columnValues(List<List<Object>> lists, int column) { - return lists.stream() - .map(list -> list.get(column)) - .toArray(); + .collect(Collectors.toList()); } - private static List<Class<?>> columnTypes(Object[][] columnValuesList) { - return Arrays.stream(columnValuesList) + /** + * Determine the column types based on the first non-null values. + * + * @param columnValuesList list of column values + * @return classes of the first non-null value + */ + private static List<Class<?>> columnTypes(List<List<Object>> columnValuesList) { + return columnValuesList.stream() .map(Table::columnType) .collect(Collectors.toList()); } /** - * Determine the column type. + * Determine the column type based on the first non-null value. * * @param columnValues column values * @return class of the first non-null value */ - private static Class<?> columnType(Object[] columnValues) { - for (final Object columnValue : columnValues) { - if (columnValue != null) { - return columnValue.getClass(); - } - } - - throw new IllegalArgumentException("No column value found!!!"); + private static Class<?> columnType(List<Object> columnValues) { + return ListUtils.coalesce(columnValues).getClass(); } } diff --git a/freemarker-generator-base/src/main/java/org/apache/freemarker/generator/base/util/ArrayUtils.java b/freemarker-generator-base/src/main/java/org/apache/freemarker/generator/base/util/ArrayUtils.java deleted file mode 100644 index 82be8fa..0000000 --- a/freemarker-generator-base/src/main/java/org/apache/freemarker/generator/base/util/ArrayUtils.java +++ /dev/null @@ -1,72 +0,0 @@ -package org.apache.freemarker.generator.base.util; - -import java.lang.reflect.Array; -import java.util.Arrays; - -import static java.util.Objects.requireNonNull; - -public class ArrayUtils { - - /** - * Transposes the given array, swapping rows with columns. The given array might contain arrays as elements that are - * not all of the same length. The returned array will have {@code null} values at those places. - * - * @param <T> the type of the array - * @param array the array - * @return the transposed array - * @throws NullPointerException if the given array is {@code null} - */ - @SuppressWarnings("unchecked") - public static <T> T[][] transpose(final T[][] array) { - requireNonNull(array); - // get y count - final int yCount = Arrays.stream(array).mapToInt(a -> a.length).max().orElse(0); - final int xCount = array.length; - final Class<?> componentType = array.getClass().getComponentType().getComponentType(); - final T[][] result = (T[][]) Array.newInstance(componentType, yCount, xCount); - for (int x = 0; x < xCount; x++) { - for (int y = 0; y < yCount; y++) { - if (array[x] == null || y >= array[x].length) { - break; - } - result[y][x] = array[x][y]; - } - } - return result; - } - - /** - * Copy an array to another array while casting to <code>R</code>. - * - * @param array array to copy - * @param <T> the source type of the array - * @param <R> the target type of the array - * @return copied array - */ - @SuppressWarnings("unchecked") - public static <T, R> R[] copy(final T[] array) { - final Class<?> componentType = array.getClass().getComponentType(); - final R[] result = (R[]) Array.newInstance(componentType, array.length); - for (int i = 0; i < array.length; i++) { - result[i] = (R) array[i]; - } - return result; - } - - /** - * Returns the first non-null value of the array. - * - * @param array array - * @param <T> the type of the array - * @return copied array - */ - @SuppressWarnings("unchecked") - public static <T> T coalesce(T... array) { - for (T i : array) { - if (i != null) { - return i; - } - } - return null; - } -} diff --git a/freemarker-generator-base/src/main/java/org/apache/freemarker/generator/base/util/ListUtils.java b/freemarker-generator-base/src/main/java/org/apache/freemarker/generator/base/util/ListUtils.java new file mode 100644 index 0000000..406a280 --- /dev/null +++ b/freemarker-generator-base/src/main/java/org/apache/freemarker/generator/base/util/ListUtils.java @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.freemarker.generator.base.util; + +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; + +public class ListUtils { + + /** + * Transposes the given tabular data, swapping rows with columns. + * + * @param <T> the type of the table + * @param table the table + * @return the transposed table + * @throws NullPointerException if the given table is {@code null} + */ + public static <T> List<List<T>> transpose(final List<List<T>> table) { + if (table.isEmpty()) { + return new ArrayList<>(); + } + + final List<List<T>> result = new ArrayList<>(); + for (int i = 0; i < table.get(0).size(); i++) { + final List<T> col = new ArrayList<>(); + for (List<T> row : table) { + col.add(row.get(i)); + } + result.add(col); + } + return result; + } + + /** + * Returns the first non-null value of the list. + * + * @param list array + * @param <T> the type of the array + * @return copied array + */ + public static <T> T coalesce(List<T> list) { + return list.stream().filter(Objects::nonNull).findFirst().orElseGet(() -> null); + } + + /** + * Copy an array to another array while casting to <code>R</code>. + * + * @param array array to copy + * @param <T> the source type of the array + * @param <R> the target type of the array + * @return copied array + */ + @SuppressWarnings("unchecked") + public static <T, R> List<R> copy(final List<T> array) { + final List<R> result = new ArrayList<>(); + for (int i = 0; i < array.size(); i++) { + result.set(i, (R) array.get(i)); + } + return result; + } +} diff --git a/freemarker-generator-base/src/main/java/org/apache/freemarker/generator/base/util/MapBuilder.java b/freemarker-generator-base/src/main/java/org/apache/freemarker/generator/base/util/MapBuilder.java index ab60a5e..60daa80 100644 --- a/freemarker-generator-base/src/main/java/org/apache/freemarker/generator/base/util/MapBuilder.java +++ b/freemarker-generator-base/src/main/java/org/apache/freemarker/generator/base/util/MapBuilder.java @@ -1,12 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.freemarker.generator.base.util; import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.Map; public class MapBuilder { - public static HashMap<String, Object> toHashMap(Object... data) { + public static Map<String, Object> toLinkedMap(Object... data) { - final HashMap<String, Object> result = new HashMap<>(); + final HashMap<String, Object> map = new LinkedHashMap<>(); if (data.length % 2 != 0) { throw new IllegalArgumentException("Odd number of arguments"); @@ -25,11 +43,11 @@ public class MapBuilder { currKey = value.toString(); continue; case 1: - result.put(currKey, value); + map.put(currKey, value); break; } } - return result; + return map; } } diff --git a/freemarker-generator-base/src/test/java/org/apache/freemarker/generator/table/TableTest.java b/freemarker-generator-base/src/test/java/org/apache/freemarker/generator/table/TableTest.java index 82b560a..695e140 100644 --- a/freemarker-generator-base/src/test/java/org/apache/freemarker/generator/table/TableTest.java +++ b/freemarker-generator-base/src/test/java/org/apache/freemarker/generator/table/TableTest.java @@ -1,3 +1,19 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.freemarker.generator.table; import org.apache.freemarker.generator.base.table.Table; @@ -10,50 +26,120 @@ import java.util.List; import java.util.Map; import static junit.framework.TestCase.assertEquals; +import static junit.framework.TestCase.assertNull; public class TableTest { - public final List<Map<String, Object>> books = Arrays.asList( - MapBuilder.toHashMap( + public final List<Map<String, Object>> booksMaps = Arrays.asList( + MapBuilder.toLinkedMap( "Book ID", "1", "Book Name", "Computer Architecture", "Category", "Computers", "In Stock", true, "Price", 125.60), - MapBuilder.toHashMap( + MapBuilder.toLinkedMap( "Book ID", "2", "Book Name", "Asp.Net 4 Blue Book", "Category", "Programming", "In Stock", null, "Price", 56), - MapBuilder.toHashMap( + MapBuilder.toLinkedMap( "Book ID", "3", "Book Name", "Popular Science", "Category", "Science", "Price", 210.40) ); + public final List<String> booksHeader = Arrays.asList( + "Book ID", + "Book Name", + "Category", + "In Stock", + "Price"); + + + public final List<List<Object>> booksList = Arrays.asList( + Arrays.asList("1", "Computer Architecture", "Computers", true, 125.60), + Arrays.asList("2", "Asp.Net 4 Blue Book", "Programming", null, 56), + Arrays.asList("3", "Popular Science", "Science", null, 210.40) + ); + + public final List<List<Object>> booksListWithHeaders = Arrays.asList( + Arrays.asList("Book ID", "Book Name", "Category", "In Stock", "Price"), + Arrays.asList("1", "Computer Architecture", "Computers", true, 125.60), + Arrays.asList("2", "Asp.Net 4 Blue Book", "Programming", null, 56), + Arrays.asList("3", "Popular Science", "Science", null, 210.40) + ); + @Test - public void shouldConvertFromMapList() { - final Table table = Table.fromMaps(books); + public void shouldConvertFromMaps() { + final Table table = Table.fromMaps(booksMaps); - assertEquals(5, table.getNrOfColumns()); - assertEquals(3, table.getNrOfRows()); - assertEquals("1", table.getRow(0).getValue("Book ID")); - assertEquals("2", table.getRow(1).getValue("Book ID")); - assertEquals("3", table.getRow(2).getValue("Book ID")); + validateBooks(table); + assertEquals(booksHeader, table.getColumnNames()); } @Test - public void shouldConvertFromEmptyMapList() { + public void shouldConvertFromEmptyMaps() { final Table table = Table.fromMaps(new ArrayList<>()); assertEquals(0, table.getNrOfColumns()); - assertEquals(0, table.getNrOfRows()); + assertEquals(0, table.size()); } - @Test(expected = IllegalArgumentException.class) - public void shouldConvertFromNullpList() { - Table.fromMaps(null); + @Test + public void shouldConvertFromNullMap() { + final Table table = Table.fromMaps(null); + + assertEquals(0, table.getNrOfColumns()); + assertEquals(0, table.size()); + } + + @Test + public void shouldConvertFromListsWithExplicitHeaders() { + final Table table = Table.fromLists(booksHeader, booksList); + + validateBooks(table); + assertEquals(booksHeader, table.getColumnNames()); + } + + @Test + public void shouldConvertFromListsWithImplicitHeaders() { + final Table table = Table.fromLists(booksListWithHeaders, true); + + validateBooks(table); + assertEquals(booksHeader, table.getColumnNames()); + } + + @Test + public void shouldConvertFromListsWithEmptyHeaders() { + final Table table = Table.fromLists(booksList); + + validateBooks(table); + } + + public void validateBooks(Table table) { + assertEquals(5, table.getNrOfColumns()); + assertEquals(3, table.size()); + // Book Id + assertEquals("1", table.get(0, 0)); + assertEquals("2", table.get(1, 0)); + assertEquals("3", table.get(2, 0)); + // Book Name + assertEquals("Computer Architecture", table.get(0, 1)); + assertEquals("Asp.Net 4 Blue Book", table.get(1, 1)); + assertEquals("Popular Science", table.get(2, 1)); + // Category + assertEquals("Computers", table.get(0, 2)); + assertEquals("Programming", table.get(1, 2)); + assertEquals("Science", table.get(2, 2)); + // In Stock + assertEquals(true, table.get(0, 3)); + assertNull(table.get(1, 3)); + assertNull(table.get(2, 3)); + // Price + assertEquals(125.60, table.get(0, 4)); + assertEquals(56, table.get(1, 4)); + assertEquals(210.40, table.get(2, 4)); } } diff --git a/freemarker-generator-cli/templates/dataframe/example.ftl b/freemarker-generator-cli/templates/dataframe/example.ftl index 8acda9c..0ac3193 100644 --- a/freemarker-generator-cli/templates/dataframe/example.ftl +++ b/freemarker-generator-cli/templates/dataframe/example.ftl @@ -16,7 +16,7 @@ under the License. --> <#assign csvParser = CSVTool.parse(DataSources.get(0))> -<#assign users = DataFrameTool.toDataFrame(csvParser)> +<#assign users = DataFrameTool.fromCSVParser(csvParser)> Original Data ============================================================================= @@ -26,8 +26,8 @@ Select By Name & Country ============================================================================= <#assign country = "Germany"> ${DataFrameTool.print(users - .select("(name == 'Schmitt' || name == 'Meier') && country == '${country}'") - .sort("name", DataFrameTool.sortOrder["ASCENDING"]))} +.select("(name == 'Schmitt' || name == 'Meier') && country == '${country}'") +.sort("name", DataFrameTool.sortOrder["ASCENDING"]))} Head of Users ============================================================================= diff --git a/freemarker-generator-tools/src/main/java/org/apache/freemarker/generator/tools/dataframe/DataFrameTool.java b/freemarker-generator-tools/src/main/java/org/apache/freemarker/generator/tools/dataframe/DataFrameTool.java index 67375aa..695cd04 100644 --- a/freemarker-generator-tools/src/main/java/org/apache/freemarker/generator/tools/dataframe/DataFrameTool.java +++ b/freemarker-generator-tools/src/main/java/org/apache/freemarker/generator/tools/dataframe/DataFrameTool.java @@ -23,8 +23,9 @@ import de.unknownreality.dataframe.transform.ColumnDataFrameTransform; import de.unknownreality.dataframe.transform.CountTransformer; import org.apache.commons.csv.CSVParser; import org.apache.freemarker.generator.base.util.Validate; -import org.apache.freemarker.generator.tools.dataframe.impl.CommonsCSVConverter; -import org.apache.freemarker.generator.tools.dataframe.impl.MapConverter; +import org.apache.freemarker.generator.tools.dataframe.converter.CSVConverter; +import org.apache.freemarker.generator.tools.dataframe.converter.ListConverter; +import org.apache.freemarker.generator.tools.dataframe.converter.MapConverter; import java.io.Writer; import java.util.HashMap; @@ -59,20 +60,29 @@ public class DataFrameTool { * @param csvParser CSV Parser * @return data frame */ - public DataFrame toDataFrame(CSVParser csvParser) { - return CommonsCSVConverter.toDataFrame(csvParser); + public DataFrame fromCSVParser(CSVParser csvParser) { + return CSVConverter.toDataFrame(csvParser); } /** - * Create a data frame from a list of maps. It is assumed - * that the map represent tabular data without missing - * values. + * Create a data frame from a list of maps. * - * @param list map to build the data frame + * @param maps maps to build the data frame * @return data frame */ - public DataFrame toDataFrame(List<Map<String, Object>> list) { - return MapConverter.toDataFrame(list); + public DataFrame fromMaps(List<Map<String, Object>> maps) { + return MapConverter.toDataFrame(maps); + } + + /** + * Create a data frame from a list of rows. + * + * @param lists lists to build the data frame from + * @param withFirstRowAsColumnNames column names as first row? + * @return data frame + */ + public DataFrame fromLists(List<List<Object>> lists, boolean withFirstRowAsColumnNames) { + return ListConverter.toDataFrame(lists, withFirstRowAsColumnNames); } /** diff --git a/freemarker-generator-tools/src/main/java/org/apache/freemarker/generator/tools/dataframe/impl/CommonsCSVConverter.java b/freemarker-generator-tools/src/main/java/org/apache/freemarker/generator/tools/dataframe/converter/CSVConverter.java similarity index 95% rename from freemarker-generator-tools/src/main/java/org/apache/freemarker/generator/tools/dataframe/impl/CommonsCSVConverter.java rename to freemarker-generator-tools/src/main/java/org/apache/freemarker/generator/tools/dataframe/converter/CSVConverter.java index 7506de0..06863fb 100644 --- a/freemarker-generator-tools/src/main/java/org/apache/freemarker/generator/tools/dataframe/impl/CommonsCSVConverter.java +++ b/freemarker-generator-tools/src/main/java/org/apache/freemarker/generator/tools/dataframe/converter/CSVConverter.java @@ -1,4 +1,4 @@ -package org.apache.freemarker.generator.tools.dataframe.impl; +package org.apache.freemarker.generator.tools.dataframe.converter; import de.unknownreality.dataframe.DataFrame; import de.unknownreality.dataframe.DataFrameBuilder; @@ -8,7 +8,7 @@ import org.apache.commons.csv.CSVRecord; import java.io.IOException; import java.util.List; -public class CommonsCSVConverter { +public class CSVConverter { /** * Create a data frame from Apache Commons CSV Parser. diff --git a/freemarker-generator-tools/src/main/java/org/apache/freemarker/generator/tools/dataframe/converter/ConverterUtils.java b/freemarker-generator-tools/src/main/java/org/apache/freemarker/generator/tools/dataframe/converter/ConverterUtils.java new file mode 100644 index 0000000..9a1e4fb --- /dev/null +++ b/freemarker-generator-tools/src/main/java/org/apache/freemarker/generator/tools/dataframe/converter/ConverterUtils.java @@ -0,0 +1,106 @@ +package org.apache.freemarker.generator.tools.dataframe.converter; + +import de.unknownreality.dataframe.DataFrame; +import de.unknownreality.dataframe.DataFrameBuilder; +import org.apache.freemarker.generator.base.table.Table; + +import java.util.List; + +public class ConverterUtils { + + static DataFrame toDataFrame(Table table) { + final DataFrame dataFrame = create(table); + return appendValues(dataFrame, table); + } + + private static DataFrameBuilder addColumn(DataFrameBuilder builder, String name, Class<?> clazz) { + switch (clazz.getName()) { + case "java.lang.Boolean": + return builder.addBooleanColumn(name); + case "java.lang.Byte": + return builder.addByteColumn(name); + case "java.lang.Double": + return builder.addDoubleColumn(name); + case "java.lang.Float": + return builder.addFloatColumn(name); + case "java.lang.Integer": + return builder.addIntegerColumn(name); + case "java.lang.Long": + return builder.addLongColumn(name); + case "java.lang.Short": + return builder.addShortColumn(name); + case "java.lang.String": + return builder.addStringColumn(name); + case "java.time.LocalDate": + return builder.addStringColumn(name); + case "java.time.LocalTime": + return builder.addStringColumn(name); + case "java.util.Date": + return builder.addStringColumn(name); + default: + throw new RuntimeException("Unable to add colum for the following type: " + clazz.getName()); + } + } + + private static Comparable<?>[] toComparables(List<Object> values) { + final int size = values.size(); + final Comparable<?>[] comparables = new Comparable<?>[size]; + for (int i = 0; i < size; i++) { + comparables[i] = (Comparable<?>) values.get(i); + } + return comparables; + } + + private static Comparable<?>[] toComparables(Object[] values) { + final Comparable<?>[] comparables = new Comparable<?>[values.length]; + for (int i = 0; i < values.length; i++) { + comparables[i] = (Comparable<?>) values[i]; + } + return comparables; + } + + /** + * Create a <code>DataFrame</code> from a table. + * + * @param table table + * @return data frame + */ + private static DataFrame create(Table table) { + final DataFrameBuilder builder = DataFrameBuilder.create(); + + if (table.hasColumnHeaderRow()) { + for (int i = 0; i < table.getColumnNames().size(); i++) { + addColumn(builder, table.getColumnNames().get(i), table.getColumnTypes().get(i)); + } + } else { + if (!table.isEmpty()) { + final List<Object> firstRecord = table.getRow(0); + for (int i = 0; i < firstRecord.size(); i++) { + builder.addStringColumn(getAlphaColumnName(i + 1)); + } + } + } + + return builder.build(); + } + + private static DataFrame appendValues(DataFrame dataFrame, Table table) { + // TODO the conversion to Comparable[] is ugly + for (int i = 0; i < table.size(); i++) { + dataFrame.append(ConverterUtils.toComparables(table.getRow(i))); + } + return dataFrame; + } + + private static String getAlphaColumnName(int num) { + String result = ""; + while (num > 0) { + num--; // 1 => a, not 0 => a + final int remainder = num % 26; + final char digit = (char) (remainder + 65); + result = digit + result; + num = (num - remainder) / 26; + } + return result; + } +} diff --git a/freemarker-generator-tools/src/main/java/org/apache/freemarker/generator/tools/dataframe/converter/ListConverter.java b/freemarker-generator-tools/src/main/java/org/apache/freemarker/generator/tools/dataframe/converter/ListConverter.java new file mode 100644 index 0000000..95bc6bf --- /dev/null +++ b/freemarker-generator-tools/src/main/java/org/apache/freemarker/generator/tools/dataframe/converter/ListConverter.java @@ -0,0 +1,21 @@ +package org.apache.freemarker.generator.tools.dataframe.converter; + +import de.unknownreality.dataframe.DataFrame; +import org.apache.freemarker.generator.base.table.Table; + +import java.util.List; + +public class ListConverter { + + /** + * Create a data frame from a list of lists. It is assumed + * that the lists represent tabular data. + * + * @param lists lists to build the data frame + * @return <code>DataFrame</code> + */ + public static DataFrame toDataFrame(List<List<Object>> lists, boolean withFirstRowAsColumnNames) { + final Table table = Table.fromLists(lists, withFirstRowAsColumnNames); + return ConverterUtils.toDataFrame(table); + } +} diff --git a/freemarker-generator-tools/src/main/java/org/apache/freemarker/generator/tools/dataframe/converter/MapConverter.java b/freemarker-generator-tools/src/main/java/org/apache/freemarker/generator/tools/dataframe/converter/MapConverter.java new file mode 100644 index 0000000..e96d938 --- /dev/null +++ b/freemarker-generator-tools/src/main/java/org/apache/freemarker/generator/tools/dataframe/converter/MapConverter.java @@ -0,0 +1,39 @@ +package org.apache.freemarker.generator.tools.dataframe.converter; + +import de.unknownreality.dataframe.DataFrame; +import de.unknownreality.dataframe.DataFrameBuilder; +import org.apache.freemarker.generator.base.table.Table; + +import java.util.Collections; +import java.util.List; +import java.util.Map; + +public class MapConverter { + + /** + * Create a data frame from a list of maps. It is assumed + * that the map represent tabular data. + * + * @param map map to build the data frame + * @return <code>DataFrame</code> + */ + public static DataFrame toDataFrame(Map<String, Object> map) { + return toDataFrame(Collections.singletonList(map)); + } + + /** + * Create a data frame from a list of maps. It is assumed + * that the map represent tabular data. + * + * @param maps list of map to build the data frame + * @return <code>DataFrame</code> + */ + public static DataFrame toDataFrame(List<Map<String, Object>> maps) { + if (maps == null || maps.isEmpty()) { + return DataFrameBuilder.createDefault(); + } + + final Table table = Table.fromMaps(maps); + return ConverterUtils.toDataFrame(table); + } +} diff --git a/freemarker-generator-tools/src/main/java/org/apache/freemarker/generator/tools/dataframe/impl/MapConverter.java b/freemarker-generator-tools/src/main/java/org/apache/freemarker/generator/tools/dataframe/impl/MapConverter.java deleted file mode 100644 index 8ced6f7..0000000 --- a/freemarker-generator-tools/src/main/java/org/apache/freemarker/generator/tools/dataframe/impl/MapConverter.java +++ /dev/null @@ -1,85 +0,0 @@ -package org.apache.freemarker.generator.tools.dataframe.impl; - -import de.unknownreality.dataframe.DataFrame; -import de.unknownreality.dataframe.DataFrameBuilder; -import org.apache.freemarker.generator.base.table.Table; - -import java.util.Collections; -import java.util.List; -import java.util.Map; - -public class MapConverter { - - /** - * Create a data frame from a list of maps. It is assumed - * that the map represent tabular data. - * - * @param map map to build the data frame - * @return <code>DataFrame</code> - */ - public static DataFrame toDataFrame(Map<String, Object> map) { - return toDataFrame(Collections.singletonList(map)); - } - - /** - * Create a data frame from a list of maps. It is assumed - * that the map represent tabular data. - * - * @param maps list of map to build the data frame - * @return <code>DataFrame</code> - */ - public static DataFrame toDataFrame(List<Map<String, Object>> maps) { - if (maps == null || maps.isEmpty()) { - return DataFrameBuilder.createDefault(); - } - - final Table table = Table.fromMaps(maps); - - // build dataframe with headers - final DataFrameBuilder builder = DataFrameBuilder.create(); - for (int i = 0; i < table.getColumnNames().length; i++) { - addColumn(builder, table.getColumnNames()[i], table.getColumnTypes()[i]); - } - final DataFrame dataFrame = builder.build(); - - // populate rows - for (int i = 0; i < table.getNrOfRows(); i++) { - final Object[] values = table.getRowValues(i); - dataFrame.append(toComparables(values)); - } - - return dataFrame; - } - - private static DataFrameBuilder addColumn(DataFrameBuilder builder, String name, Class<?> clazz) { - switch (clazz.getName()) { - case "java.lang.Boolean": - return builder.addBooleanColumn(name); - case "java.lang.Byte": - return builder.addByteColumn(name); - case "java.lang.Double": - return builder.addDoubleColumn(name); - case "java.lang.Float": - return builder.addFloatColumn(name); - case "java.lang.Integer": - return builder.addIntegerColumn(name); - case "java.lang.Long": - return builder.addLongColumn(name); - case "java.lang.Short": - return builder.addShortColumn(name); - case "java.lang.String": - return builder.addStringColumn(name); - default: - throw new RuntimeException("Unable to add colum for the following type: " + clazz.getName()); - } - } - - private static Comparable<?>[] toComparables(Object[] values) { - final Comparable<?>[] comparables = new Comparable<?>[values.length]; - for (int i = 0; i < values.length; i++) { - comparables[i] = (Comparable<?>) values[i]; - } - return comparables; - } - -} diff --git a/freemarker-generator-tools/src/main/java/org/apache/freemarker/generator/tools/excel/ExcelTool.java b/freemarker-generator-tools/src/main/java/org/apache/freemarker/generator/tools/excel/ExcelTool.java index f03df80..b5ead1d 100644 --- a/freemarker-generator-tools/src/main/java/org/apache/freemarker/generator/tools/excel/ExcelTool.java +++ b/freemarker-generator-tools/src/main/java/org/apache/freemarker/generator/tools/excel/ExcelTool.java @@ -78,10 +78,10 @@ public class ExcelTool { * @param sheet Excel sheet * @return Table containing formatted cell values as strings */ - public List<List<String>> toTable(Sheet sheet) { + public List<List<Object>> toTable(Sheet sheet) { final DataFormatter dataFormatter = dataFormatter(); final Iterator<Row> iterator = sheet.iterator(); - final List<List<String>> result = new ArrayList<>(); + final List<List<Object>> result = new ArrayList<>(); while (iterator.hasNext()) { final Row row = iterator.next(); @@ -117,8 +117,8 @@ public class ExcelTool { return "Process Excels files (XLS, XLSX) using Apache POI (see https://poi.apache.org)"; } - private static List<String> toColumns(Row row, DataFormatter dataFormatter) { - final List<String> columnValues = new ArrayList<>(); + private static List<Object> toColumns(Row row, DataFormatter dataFormatter) { + final List<Object> columnValues = new ArrayList<>(); for (int columnIndex = 0; columnIndex < row.getLastCellNum(); columnIndex++) { final Cell cell = row.getCell(columnIndex, CREATE_NULL_AS_BLANK); final String formatedCellValue = dataFormatter.formatCellValue(cell).trim(); diff --git a/freemarker-generator-tools/src/test/java/org/apache/freemarker/generator/tools/dataframe/DataFrameToolTest.java b/freemarker-generator-tools/src/test/java/org/apache/freemarker/generator/tools/dataframe/DataFrameToolTest.java index dd6bd70..8c93dd3 100644 --- a/freemarker-generator-tools/src/test/java/org/apache/freemarker/generator/tools/dataframe/DataFrameToolTest.java +++ b/freemarker-generator-tools/src/test/java/org/apache/freemarker/generator/tools/dataframe/DataFrameToolTest.java @@ -19,8 +19,11 @@ package org.apache.freemarker.generator.tools.dataframe; import de.unknownreality.dataframe.DataFrame; import org.apache.commons.csv.CSVFormat; import org.apache.commons.csv.CSVParser; +import org.apache.freemarker.generator.base.datasource.DataSourceFactory; import org.apache.freemarker.generator.tools.commonscsv.CommonsCSVTool; +import org.apache.freemarker.generator.tools.excel.ExcelTool; import org.apache.freemarker.generator.tools.gson.GsonTool; +import org.apache.poi.ss.usermodel.Workbook; import org.junit.Test; import java.util.List; @@ -67,7 +70,7 @@ public class DataFrameToolTest { @Test public void shouldParseCsvFileWithoutHeader() { final CSVParser csvParser = csvParser(CSV_WITHOUT_HEADER, DEFAULT.withDelimiter(';')); - final DataFrame dataFrame = dataFrameTool().toDataFrame(csvParser); + final DataFrame dataFrame = dataFrameTool().fromCSVParser(csvParser); assertEquals(3, dataFrame.getColumns().size()); assertEquals(4, dataFrame.getRows().size()); @@ -79,7 +82,7 @@ public class DataFrameToolTest { @Test public void shouldParseCsvFileWithHeader() { final CSVParser csvParser = csvParser(CSV_WITH_HEADER, DEFAULT.withHeader().withDelimiter(';')); - final DataFrame dataFrame = dataFrameTool().toDataFrame(csvParser); + final DataFrame dataFrame = dataFrameTool().fromCSVParser(csvParser); assertEquals(3, dataFrame.getColumns().size()); assertEquals(4, dataFrame.getRows().size()); @@ -95,7 +98,7 @@ public class DataFrameToolTest { public void shouldParseJsonTable() { final String columnName = "Book ID"; final List<Map<String, Object>> json = (List<Map<String, Object>>) gsonTool().parse(JSON_ARRAY); - final DataFrame dataFrame = dataFrameTool().toDataFrame(json); + final DataFrame dataFrame = dataFrameTool().fromMaps(json); assertEquals(5, dataFrame.getColumns().size()); assertEquals(3, dataFrame.getRows().size()); @@ -104,6 +107,20 @@ public class DataFrameToolTest { assertEquals("3", dataFrame.getColumn(columnName).get(2)); } + // === Excel ============================================================ + + @Test + public void shouldParseExcelSheet() { + final ExcelTool excelTool = excelTool(); + final Workbook workbook = excelTool.parse(DataSourceFactory.create("./src/test/data/excel/test.xls")); + final List<List<Object>> sheet = excelTool.toTable(workbook.getSheetAt(0)); + + final DataFrame dataFrame = dataFrameTool().fromLists(sheet, true); + + return; + + } + private DataFrameTool dataFrameTool() { return new DataFrameTool(); } @@ -116,6 +133,10 @@ public class DataFrameToolTest { return new GsonTool(); } + private ExcelTool excelTool() { + return new ExcelTool(); + } + private CSVParser csvParser(String csv, CSVFormat csvFormat) { return commonsCSVTool().parse(csv, csvFormat); } diff --git a/freemarker-generator-tools/src/test/java/org/apache/freemarker/generator/tools/excel/ExcelToolTest.java b/freemarker-generator-tools/src/test/java/org/apache/freemarker/generator/tools/excel/ExcelToolTest.java index 9c8ef8b..089c2d0 100644 --- a/freemarker-generator-tools/src/test/java/org/apache/freemarker/generator/tools/excel/ExcelToolTest.java +++ b/freemarker-generator-tools/src/test/java/org/apache/freemarker/generator/tools/excel/ExcelToolTest.java @@ -39,7 +39,7 @@ public class ExcelToolTest { final Workbook workbook = workbook(TEST_XLS); final List<Sheet> sheets = excelTool().getSheets(workbook); - final List<List<String>> records = excelTool().toTable(sheets.get(0)); + final List<List<Object>> records = excelTool().toTable(sheets.get(0)); assertEquals(1, sheets.size()); assertEquals(3, records.size()); @@ -50,7 +50,7 @@ public class ExcelToolTest { final Workbook workbook = workbook(TEST_XLSX); final List<Sheet> sheets = excelTool().getSheets(workbook); - final List<List<String>> records = excelTool().toTable(sheets.get(0)); + final List<List<Object>> records = excelTool().toTable(sheets.get(0)); assertEquals(1, sheets.size()); assertEquals(3, records.size()); @@ -71,9 +71,9 @@ public class ExcelToolTest { public void shouldConvertSheetToTable() { final Workbook workbook = workbook(TEST_XLSX); final List<Sheet> sheets = excelTool().getSheets(workbook); - final List<List<String>> records = excelTool().toTable(sheets.get(0)); + final List<List<Object>> records = excelTool().toTable(sheets.get(0)); - final List<String> record = records.get(1); + final List<Object> record = records.get(1); assertEquals("Row 1", record.get(0)); assertEquals("01/31/17", record.get(1));
