This is an automated email from the ASF dual-hosted git repository. sgoeschl pushed a commit to branch FREEMARKER-144 in repository https://gitbox.apache.org/repos/asf/freemarker-generator.git
commit 8071c1403f77dab159454d6b30f61cc65e69109d Author: Siegfried Goeschl <[email protected]> AuthorDate: Wed Jun 10 15:25:08 2020 +0200 FREEMARKER-144 Proof Of Concept for providing DataFrames --- .../freemarker/generator/base/table/Table.java | 78 ++++++++-------- .../freemarker/generator/base/util/ArrayUtils.java | 72 +++++++++++++++ .../generator/tools/dataframe/DataFrameTool.java | 102 +-------------------- .../tools/dataframe/impl/CommonsCSVConverter.java | 63 +++++++++++++ .../tools/dataframe/impl/MapConverter.java | 85 +++++++++++++++++ 5 files changed, 265 insertions(+), 135 deletions(-) diff --git a/freemarker-generator-base/src/main/java/org/apache/freemarker/generator/base/table/Table.java b/freemarker-generator-base/src/main/java/org/apache/freemarker/generator/base/table/Table.java index 3667e88..7530844 100644 --- a/freemarker-generator-base/src/main/java/org/apache/freemarker/generator/base/table/Table.java +++ b/freemarker-generator-base/src/main/java/org/apache/freemarker/generator/base/table/Table.java @@ -1,14 +1,15 @@ package org.apache.freemarker.generator.base.table; +import org.apache.freemarker.generator.base.util.ArrayUtils; import org.apache.freemarker.generator.base.util.Validate; -import java.lang.reflect.Array; import java.util.Arrays; import java.util.Collection; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.stream.Collectors; +import java.util.stream.IntStream; import static java.util.Objects.requireNonNull; @@ -24,7 +25,7 @@ public class Table { private Table(String[] columnNames, Class<?>[] columnTypes, Object[][] columnValuesList) { this.columnNames = requireNonNull(columnNames); this.columnTypes = requireNonNull(columnTypes); - this.values = transpose(requireNonNull(columnValuesList)); + this.values = ArrayUtils.transpose(requireNonNull(columnValuesList)); this.columnMap = new HashMap<>(); for (int i = 0; i < this.columnNames.length; i++) { @@ -56,17 +57,30 @@ public class Table { return new Row(columnMap, getRowValues(row)); } - public static Table fromMaps(List<Map<String, Object>> list) { + public static Table fromMaps(List<Map<String, Object>> maps) { + Validate.notNull(maps, "list is null"); + + final List<String> columnNames = columnNames(maps); + final Object[][] columnValuesList = columnValuesList(maps, columnNames); + final List<Class<?>> columnTypes = columnTypes(columnValuesList); + + return new Table( + columnNames.toArray(new String[0]), + columnTypes.toArray(new Class[0]), + columnValuesList); + } + + public static Table fromLists(List<List<Object>> list) { Validate.notNull(list, "list is null"); - final List<String> columnNames = columnNames(list); - final Object[][] tableValues = columnValues(list, columnNames); - final List<Class<?>> columnTypes = columnTypes(tableValues); + final List<String> columnNames = Arrays.asList(ArrayUtils.copy(list.toArray(new Object[0]))); + final Object[][] columnValuesList = columnValuesList(list.subList(1, list.size())); + final List<Class<?>> columnTypes = columnTypes(columnValuesList); return new Table( columnNames.toArray(new String[0]), columnTypes.toArray(new Class[0]), - tableValues); + columnValuesList); } public static final class Row { @@ -99,19 +113,36 @@ public class Table { .collect(Collectors.toList()); } - private static Object[][] columnValues(List<Map<String, Object>> list, List<String> columnNames) { + private static Object[][] columnValuesList(List<Map<String, Object>> list, List<String> columnNames) { return columnNames.stream() .map(columnName -> columnValues(list, columnName)) .collect(Collectors.toList()) .toArray(new Object[0][0]); } - private static Object[] columnValues(List<Map<String, Object>> list, String columnName) { - return list.stream() + private static Object[][] columnValuesList(List<List<Object>> lists) { + if (lists.isEmpty()) { + return new Object[0][0]; + } + + return IntStream.range(0, lists.get(0).size()) + .mapToObj(i -> columnValues(lists, i)) + .collect(Collectors.toList()) + .toArray(new Object[0][0]); + } + + private static Object[] columnValues(List<Map<String, Object>> maps, String columnName) { + return maps.stream() .map(map -> map.getOrDefault(columnName, null)) .toArray(); } + private static Object[] columnValues(List<List<Object>> lists, int column) { + return lists.stream() + .map(list -> list.get(column)) + .toArray(); + } + private static List<Class<?>> columnTypes(Object[][] columnValuesList) { return Arrays.stream(columnValuesList) .map(Table::columnType) @@ -133,31 +164,4 @@ public class Table { throw new IllegalArgumentException("No column value found!!!"); } - - /** - * Transposes the given array, swapping rows with columns. The given array might contain arrays as elements that are - * not all of the same length. The returned array will have {@code null} values at those places. - * - * @param <T> the type of the array - * @param array the array - * @return the transposed array - * @throws NullPointerException if the given array is {@code null} - */ - public static <T> T[][] transpose(final T[][] array) { - requireNonNull(array); - // get y count - final int yCount = Arrays.stream(array).mapToInt(a -> a.length).max().orElse(0); - final int xCount = array.length; - final Class<?> componentType = array.getClass().getComponentType().getComponentType(); - @SuppressWarnings("unchecked") final T[][] newArray = (T[][]) Array.newInstance(componentType, yCount, xCount); - for (int x = 0; x < xCount; x++) { - for (int y = 0; y < yCount; y++) { - if (array[x] == null || y >= array[x].length) { - break; - } - newArray[y][x] = array[x][y]; - } - } - return newArray; - } } diff --git a/freemarker-generator-base/src/main/java/org/apache/freemarker/generator/base/util/ArrayUtils.java b/freemarker-generator-base/src/main/java/org/apache/freemarker/generator/base/util/ArrayUtils.java new file mode 100644 index 0000000..82be8fa --- /dev/null +++ b/freemarker-generator-base/src/main/java/org/apache/freemarker/generator/base/util/ArrayUtils.java @@ -0,0 +1,72 @@ +package org.apache.freemarker.generator.base.util; + +import java.lang.reflect.Array; +import java.util.Arrays; + +import static java.util.Objects.requireNonNull; + +public class ArrayUtils { + + /** + * Transposes the given array, swapping rows with columns. The given array might contain arrays as elements that are + * not all of the same length. The returned array will have {@code null} values at those places. + * + * @param <T> the type of the array + * @param array the array + * @return the transposed array + * @throws NullPointerException if the given array is {@code null} + */ + @SuppressWarnings("unchecked") + public static <T> T[][] transpose(final T[][] array) { + requireNonNull(array); + // get y count + final int yCount = Arrays.stream(array).mapToInt(a -> a.length).max().orElse(0); + final int xCount = array.length; + final Class<?> componentType = array.getClass().getComponentType().getComponentType(); + final T[][] result = (T[][]) Array.newInstance(componentType, yCount, xCount); + for (int x = 0; x < xCount; x++) { + for (int y = 0; y < yCount; y++) { + if (array[x] == null || y >= array[x].length) { + break; + } + result[y][x] = array[x][y]; + } + } + return result; + } + + /** + * Copy an array to another array while casting to <code>R</code>. + * + * @param array array to copy + * @param <T> the source type of the array + * @param <R> the target type of the array + * @return copied array + */ + @SuppressWarnings("unchecked") + public static <T, R> R[] copy(final T[] array) { + final Class<?> componentType = array.getClass().getComponentType(); + final R[] result = (R[]) Array.newInstance(componentType, array.length); + for (int i = 0; i < array.length; i++) { + result[i] = (R) array[i]; + } + return result; + } + + /** + * Returns the first non-null value of the array. + * + * @param array array + * @param <T> the type of the array + * @return copied array + */ + @SuppressWarnings("unchecked") + public static <T> T coalesce(T... array) { + for (T i : array) { + if (i != null) { + return i; + } + } + return null; + } +} diff --git a/freemarker-generator-tools/src/main/java/org/apache/freemarker/generator/tools/dataframe/DataFrameTool.java b/freemarker-generator-tools/src/main/java/org/apache/freemarker/generator/tools/dataframe/DataFrameTool.java index 35fcdfa..67375aa 100644 --- a/freemarker-generator-tools/src/main/java/org/apache/freemarker/generator/tools/dataframe/DataFrameTool.java +++ b/freemarker-generator-tools/src/main/java/org/apache/freemarker/generator/tools/dataframe/DataFrameTool.java @@ -17,17 +17,15 @@ package org.apache.freemarker.generator.tools.dataframe; import de.unknownreality.dataframe.DataFrame; -import de.unknownreality.dataframe.DataFrameBuilder; import de.unknownreality.dataframe.DataFrameWriter; import de.unknownreality.dataframe.sort.SortColumn.Direction; import de.unknownreality.dataframe.transform.ColumnDataFrameTransform; import de.unknownreality.dataframe.transform.CountTransformer; import org.apache.commons.csv.CSVParser; -import org.apache.commons.csv.CSVRecord; -import org.apache.freemarker.generator.base.table.Table; import org.apache.freemarker.generator.base.util.Validate; +import org.apache.freemarker.generator.tools.dataframe.impl.CommonsCSVConverter; +import org.apache.freemarker.generator.tools.dataframe.impl.MapConverter; -import java.io.IOException; import java.io.Writer; import java.util.HashMap; import java.util.List; @@ -62,36 +60,7 @@ public class DataFrameTool { * @return data frame */ public DataFrame toDataFrame(CSVParser csvParser) { - try { - final List<String> headerNames = csvParser.getHeaderNames(); - final DataFrameBuilder builder = DataFrameBuilder.create(); - final List<CSVRecord> records = csvParser.getRecords(); - final CSVRecord firstRecord = records.get(0); - - // build dataframe with headers - if (headerNames != null && !headerNames.isEmpty()) { - headerNames.forEach(builder::addStringColumn); - } else { - for (int i = 0; i < firstRecord.size(); i++) { - builder.addStringColumn(getAlpha(i + 1)); - } - } - - final DataFrame dataFrame = builder.build(); - - // populate rows - final String[] currValues = new String[firstRecord.size()]; - for (CSVRecord csvRecord : records) { - for (int i = 0; i < currValues.length; i++) { - currValues[i] = csvRecord.get(i); - } - dataFrame.append(currValues); - } - - return dataFrame; - } catch (IOException e) { - throw new RuntimeException("Unable to create DataFrame", e); - } + return CommonsCSVConverter.toDataFrame(csvParser); } /** @@ -103,26 +72,7 @@ public class DataFrameTool { * @return data frame */ public DataFrame toDataFrame(List<Map<String, Object>> list) { - if (list == null || list.isEmpty()) { - return DataFrameBuilder.createDefault(); - } - - final Table table = Table.fromMaps(list); - - // build dataframe with headers - final DataFrameBuilder builder = DataFrameBuilder.create(); - for (int i = 0; i < table.getColumnNames().length; i++) { - addColumn(builder, table.getColumnNames()[i], table.getColumnTypes()[i]); - } - final DataFrame dataFrame = builder.build(); - - // populate rows - for (int i = 0; i < table.getNrOfRows(); i++) { - final Object[] values = table.getRowValues(i); - dataFrame.append(toComparables(values)); - } - - return dataFrame; + return MapConverter.toDataFrame(list); } /** @@ -163,51 +113,7 @@ public class DataFrameTool { return "Bridge to nRo/DataFrame (see https://github.com/nRo/DataFrame)"; } - private static DataFrameBuilder addColumn(DataFrameBuilder builder, String name, Class<?> clazz) { - switch (clazz.getName()) { - case "java.lang.Boolean": - return builder.addBooleanColumn(name); - case "java.lang.Byte": - return builder.addByteColumn(name); - case "java.lang.Double": - return builder.addDoubleColumn(name); - case "java.lang.Float": - return builder.addFloatColumn(name); - case "java.lang.Integer": - return builder.addIntegerColumn(name); - case "java.lang.Long": - return builder.addLongColumn(name); - case "java.lang.Short": - return builder.addShortColumn(name); - case "java.lang.String": - return builder.addStringColumn(name); - default: - throw new RuntimeException("Unable to add colum for the following type: " + clazz.getName()); - } - } - private static CountTransformer countTransformer(boolean ignoreNA) { return new CountTransformer(ignoreNA); } - - private static Comparable<?>[] toComparables(Object[] values) { - final Comparable<?>[] comparables = new Comparable<?>[values.length]; - for (int i = 0; i < values.length; i++) { - comparables[i] = (Comparable<?>) values[i]; - } - return comparables; - } - - private static String getAlpha(int num) { - String result = ""; - while (num > 0) { - num--; // 1 => a, not 0 => a - int remainder = num % 26; - char digit = (char) (remainder + 65); - result = digit + result; - num = (num - remainder) / 26; - } - return result; - } - } diff --git a/freemarker-generator-tools/src/main/java/org/apache/freemarker/generator/tools/dataframe/impl/CommonsCSVConverter.java b/freemarker-generator-tools/src/main/java/org/apache/freemarker/generator/tools/dataframe/impl/CommonsCSVConverter.java new file mode 100644 index 0000000..7506de0 --- /dev/null +++ b/freemarker-generator-tools/src/main/java/org/apache/freemarker/generator/tools/dataframe/impl/CommonsCSVConverter.java @@ -0,0 +1,63 @@ +package org.apache.freemarker.generator.tools.dataframe.impl; + +import de.unknownreality.dataframe.DataFrame; +import de.unknownreality.dataframe.DataFrameBuilder; +import org.apache.commons.csv.CSVParser; +import org.apache.commons.csv.CSVRecord; + +import java.io.IOException; +import java.util.List; + +public class CommonsCSVConverter { + + /** + * Create a data frame from Apache Commons CSV Parser. + * + * @param csvParser CSV Parser + * @return data frame + */ + public static DataFrame toDataFrame(CSVParser csvParser) { + try { + final List<String> headerNames = csvParser.getHeaderNames(); + final DataFrameBuilder builder = DataFrameBuilder.create(); + final List<CSVRecord> records = csvParser.getRecords(); + final CSVRecord firstRecord = records.get(0); + + // build dataframe with headers + if (headerNames != null && !headerNames.isEmpty()) { + headerNames.forEach(builder::addStringColumn); + } else { + for (int i = 0; i < firstRecord.size(); i++) { + builder.addStringColumn(getAlphaColumnName(i + 1)); + } + } + + final DataFrame dataFrame = builder.build(); + + // populate rows + final String[] currValues = new String[firstRecord.size()]; + for (CSVRecord csvRecord : records) { + for (int i = 0; i < currValues.length; i++) { + currValues[i] = csvRecord.get(i); + } + dataFrame.append(currValues); + } + + return dataFrame; + } catch (IOException e) { + throw new RuntimeException("Unable to create DataFrame", e); + } + } + + private static String getAlphaColumnName(int num) { + String result = ""; + while (num > 0) { + num--; // 1 => a, not 0 => a + final int remainder = num % 26; + final char digit = (char) (remainder + 65); + result = digit + result; + num = (num - remainder) / 26; + } + return result; + } +} diff --git a/freemarker-generator-tools/src/main/java/org/apache/freemarker/generator/tools/dataframe/impl/MapConverter.java b/freemarker-generator-tools/src/main/java/org/apache/freemarker/generator/tools/dataframe/impl/MapConverter.java new file mode 100644 index 0000000..8ced6f7 --- /dev/null +++ b/freemarker-generator-tools/src/main/java/org/apache/freemarker/generator/tools/dataframe/impl/MapConverter.java @@ -0,0 +1,85 @@ +package org.apache.freemarker.generator.tools.dataframe.impl; + +import de.unknownreality.dataframe.DataFrame; +import de.unknownreality.dataframe.DataFrameBuilder; +import org.apache.freemarker.generator.base.table.Table; + +import java.util.Collections; +import java.util.List; +import java.util.Map; + +public class MapConverter { + + /** + * Create a data frame from a list of maps. It is assumed + * that the map represent tabular data. + * + * @param map map to build the data frame + * @return <code>DataFrame</code> + */ + public static DataFrame toDataFrame(Map<String, Object> map) { + return toDataFrame(Collections.singletonList(map)); + } + + /** + * Create a data frame from a list of maps. It is assumed + * that the map represent tabular data. + * + * @param maps list of map to build the data frame + * @return <code>DataFrame</code> + */ + public static DataFrame toDataFrame(List<Map<String, Object>> maps) { + if (maps == null || maps.isEmpty()) { + return DataFrameBuilder.createDefault(); + } + + final Table table = Table.fromMaps(maps); + + // build dataframe with headers + final DataFrameBuilder builder = DataFrameBuilder.create(); + for (int i = 0; i < table.getColumnNames().length; i++) { + addColumn(builder, table.getColumnNames()[i], table.getColumnTypes()[i]); + } + final DataFrame dataFrame = builder.build(); + + // populate rows + for (int i = 0; i < table.getNrOfRows(); i++) { + final Object[] values = table.getRowValues(i); + dataFrame.append(toComparables(values)); + } + + return dataFrame; + } + + private static DataFrameBuilder addColumn(DataFrameBuilder builder, String name, Class<?> clazz) { + switch (clazz.getName()) { + case "java.lang.Boolean": + return builder.addBooleanColumn(name); + case "java.lang.Byte": + return builder.addByteColumn(name); + case "java.lang.Double": + return builder.addDoubleColumn(name); + case "java.lang.Float": + return builder.addFloatColumn(name); + case "java.lang.Integer": + return builder.addIntegerColumn(name); + case "java.lang.Long": + return builder.addLongColumn(name); + case "java.lang.Short": + return builder.addShortColumn(name); + case "java.lang.String": + return builder.addStringColumn(name); + default: + throw new RuntimeException("Unable to add colum for the following type: " + clazz.getName()); + } + } + + private static Comparable<?>[] toComparables(Object[] values) { + final Comparable<?>[] comparables = new Comparable<?>[values.length]; + for (int i = 0; i < values.length; i++) { + comparables[i] = (Comparable<?>) values[i]; + } + return comparables; + } + +}
