This is an automated email from the ASF dual-hosted git repository. bchapuis pushed a commit to branch calcite in repository https://gitbox.apache.org/repos/asf/incubator-baremaps.git
commit e7ef5c9e3e3fc9635954b9d837e2f1cf4c31dfaa Author: Bertil Chapuis <[email protected]> AuthorDate: Sat Jun 24 12:22:50 2023 +0200 Isolate type conversions in dedicated classes --- .../baremaps/database/calcite/SqlDataTable.java | 58 +-------------------- .../{SqlDataTable.java => SqlTypeConversion.java} | 39 +------------- .../storage/flatgeobuf/FlatGeoBufDataTable.java | 9 ++-- ...versions.java => FlatGeoBufTypeConversion.java} | 6 +-- .../storage/postgres/PostgresDataSchema.java | 52 +++---------------- .../storage/postgres/PostgresTypeConversion.java | 59 ++++++++++++++++++++++ ...aSchemaTest.java => PostgresTileStoreTest.java} | 2 +- 7 files changed, 76 insertions(+), 149 deletions(-) diff --git a/baremaps-core/src/main/java/org/apache/baremaps/database/calcite/SqlDataTable.java b/baremaps-core/src/main/java/org/apache/baremaps/database/calcite/SqlDataTable.java index c2630af7..d98e102a 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/database/calcite/SqlDataTable.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/database/calcite/SqlDataTable.java @@ -13,78 +13,22 @@ package org.apache.baremaps.database.calcite; import java.util.Collection; -import java.util.EnumMap; import org.apache.baremaps.database.collection.DataCollectionAdapter; import org.apache.baremaps.database.schema.DataColumn; -import org.apache.baremaps.database.schema.DataColumn.Type; import org.apache.baremaps.database.schema.DataTable; import org.apache.calcite.DataContext; -import org.apache.calcite.jdbc.JavaTypeFactoryImpl; import org.apache.calcite.linq4j.Enumerable; import org.apache.calcite.linq4j.Linq4j; import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rel.type.RelDataTypeFactory; import org.apache.calcite.schema.ScannableTable; import org.apache.calcite.schema.impl.AbstractTable; -import org.apache.calcite.sql.type.SqlTypeName; -import org.locationtech.jts.geom.*; /** * A simple table based on a list. */ class SqlDataTable extends AbstractTable implements ScannableTable { - private static final EnumMap<Type, RelDataType> types = new EnumMap<>(Type.class); - - static { - types.put(Type.BYTE, new JavaTypeFactoryImpl() - .createSqlType(SqlTypeName.TINYINT)); - types.put(Type.BYTE_ARRAY, new JavaTypeFactoryImpl() - .createArrayType(new JavaTypeFactoryImpl().createSqlType(SqlTypeName.TINYINT), -1)); - types.put(Type.BOOLEAN, new JavaTypeFactoryImpl() - .createSqlType(SqlTypeName.BOOLEAN)); - types.put(Type.BOOLEAN_ARRAY, new JavaTypeFactoryImpl() - .createArrayType(new JavaTypeFactoryImpl().createSqlType(SqlTypeName.BOOLEAN), -1)); - types.put(Type.SHORT, new JavaTypeFactoryImpl() - .createSqlType(SqlTypeName.SMALLINT)); - types.put(Type.SHORT_ARRAY, new JavaTypeFactoryImpl() - .createArrayType(new JavaTypeFactoryImpl().createSqlType(SqlTypeName.SMALLINT), -1)); - types.put(Type.INTEGER, new JavaTypeFactoryImpl() - .createSqlType(SqlTypeName.INTEGER)); - types.put(Type.INTEGER_ARRAY, new JavaTypeFactoryImpl() - .createArrayType(new JavaTypeFactoryImpl().createSqlType(SqlTypeName.INTEGER), -1)); - types.put(Type.LONG, new JavaTypeFactoryImpl() - .createSqlType(SqlTypeName.BIGINT)); - types.put(Type.LONG_ARRAY, new JavaTypeFactoryImpl() - .createArrayType(new JavaTypeFactoryImpl().createSqlType(SqlTypeName.BIGINT), -1)); - types.put(Type.FLOAT, new JavaTypeFactoryImpl() - .createSqlType(SqlTypeName.FLOAT)); - types.put(Type.FLOAT_ARRAY, new JavaTypeFactoryImpl() - .createArrayType(new JavaTypeFactoryImpl().createSqlType(SqlTypeName.FLOAT), -1)); - types.put(Type.DOUBLE, new JavaTypeFactoryImpl() - .createSqlType(SqlTypeName.DOUBLE)); - types.put(Type.DOUBLE_ARRAY, new JavaTypeFactoryImpl() - .createArrayType(new JavaTypeFactoryImpl().createSqlType(SqlTypeName.DOUBLE), -1)); - types.put(Type.STRING, new JavaTypeFactoryImpl() - .createSqlType(SqlTypeName.VARCHAR)); - types.put(Type.GEOMETRY, new JavaTypeFactoryImpl() - .createJavaType(Geometry.class)); - types.put(Type.POINT, new JavaTypeFactoryImpl() - .createJavaType(Point.class)); - types.put(Type.LINESTRING, new JavaTypeFactoryImpl() - .createJavaType(LineString.class)); - types.put(Type.POLYGON, new JavaTypeFactoryImpl() - .createJavaType(Polygon.class)); - types.put(Type.MULTIPOINT, new JavaTypeFactoryImpl() - .createJavaType(MultiPoint.class)); - types.put(Type.MULTILINESTRING, new JavaTypeFactoryImpl() - .createJavaType(MultiLineString.class)); - types.put(Type.MULTIPOLYGON, new JavaTypeFactoryImpl() - .createJavaType(MultiPolygon.class)); - types.put(Type.GEOMETRYCOLLECTION, new JavaTypeFactoryImpl() - .createJavaType(GeometryCollection.class)); - } - private final DataTable table; SqlDataTable(DataTable table) { @@ -102,7 +46,7 @@ class SqlDataTable extends AbstractTable implements ScannableTable { public RelDataType getRowType(final RelDataTypeFactory typeFactory) { var rowType = new RelDataTypeFactory.Builder(typeFactory); for (DataColumn column : table.rowType().columns()) { - rowType.add(column.name(), types.get(column.type())); + rowType.add(column.name(), SqlTypeConversion.types.get(column.type())); } return rowType.build(); } diff --git a/baremaps-core/src/main/java/org/apache/baremaps/database/calcite/SqlDataTable.java b/baremaps-core/src/main/java/org/apache/baremaps/database/calcite/SqlTypeConversion.java similarity index 72% copy from baremaps-core/src/main/java/org/apache/baremaps/database/calcite/SqlDataTable.java copy to baremaps-core/src/main/java/org/apache/baremaps/database/calcite/SqlTypeConversion.java index c2630af7..b1efb1bf 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/database/calcite/SqlDataTable.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/database/calcite/SqlTypeConversion.java @@ -12,29 +12,16 @@ package org.apache.baremaps.database.calcite; -import java.util.Collection; import java.util.EnumMap; -import org.apache.baremaps.database.collection.DataCollectionAdapter; -import org.apache.baremaps.database.schema.DataColumn; import org.apache.baremaps.database.schema.DataColumn.Type; -import org.apache.baremaps.database.schema.DataTable; -import org.apache.calcite.DataContext; import org.apache.calcite.jdbc.JavaTypeFactoryImpl; -import org.apache.calcite.linq4j.Enumerable; -import org.apache.calcite.linq4j.Linq4j; import org.apache.calcite.rel.type.RelDataType; -import org.apache.calcite.rel.type.RelDataTypeFactory; -import org.apache.calcite.schema.ScannableTable; -import org.apache.calcite.schema.impl.AbstractTable; import org.apache.calcite.sql.type.SqlTypeName; import org.locationtech.jts.geom.*; -/** - * A simple table based on a list. - */ -class SqlDataTable extends AbstractTable implements ScannableTable { +public class SqlTypeConversion { - private static final EnumMap<Type, RelDataType> types = new EnumMap<>(Type.class); + static final EnumMap<Type, RelDataType> types = new EnumMap<>(Type.class); static { types.put(Type.BYTE, new JavaTypeFactoryImpl() @@ -84,26 +71,4 @@ class SqlDataTable extends AbstractTable implements ScannableTable { types.put(Type.GEOMETRYCOLLECTION, new JavaTypeFactoryImpl() .createJavaType(GeometryCollection.class)); } - - private final DataTable table; - - SqlDataTable(DataTable table) { - this.table = table; - } - - @Override - public Enumerable<Object[]> scan(final DataContext root) { - Collection<Object[]> collection = - new DataCollectionAdapter<>(table, row -> row.values().toArray()); - return Linq4j.asEnumerable(collection); - } - - @Override - public RelDataType getRowType(final RelDataTypeFactory typeFactory) { - var rowType = new RelDataTypeFactory.Builder(typeFactory); - for (DataColumn column : table.rowType().columns()) { - rowType.add(column.name(), types.get(column.type())); - } - return rowType.build(); - } } diff --git a/baremaps-core/src/main/java/org/apache/baremaps/storage/flatgeobuf/FlatGeoBufDataTable.java b/baremaps-core/src/main/java/org/apache/baremaps/storage/flatgeobuf/FlatGeoBufDataTable.java index a1b85d31..1401b29d 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/storage/flatgeobuf/FlatGeoBufDataTable.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/storage/flatgeobuf/FlatGeoBufDataTable.java @@ -26,7 +26,6 @@ import org.apache.baremaps.database.collection.AbstractDataCollection; import org.apache.baremaps.database.schema.AbstractDataTable; import org.apache.baremaps.database.schema.DataRow; import org.apache.baremaps.database.schema.DataRowType; -import org.apache.baremaps.storage.flatgeobuf.internal.TableConversions; import org.locationtech.jts.geom.*; import org.wololo.flatgeobuf.Constants; import org.wololo.flatgeobuf.GeometryConversions; @@ -81,7 +80,7 @@ public class FlatGeoBufDataTable extends AbstractDataTable { // try to read the row type from the file var buffer = ByteBuffer.allocate(1 << 20).order(ByteOrder.LITTLE_ENDIAN); HeaderMeta headerMeta = readHeaderMeta(channel, buffer); - return TableConversions.asRowType(headerMeta); + return FlatGeoBufTypeConversion.asRowType(headerMeta); } catch (IOException e) { return null; } @@ -164,7 +163,7 @@ public class FlatGeoBufDataTable extends AbstractDataTable { headerMeta.featuresCount = features instanceof AbstractDataCollection<DataRow>c ? c.sizeAsLong() : features.size(); headerMeta.name = rowType.name(); - headerMeta.columns = TableConversions.asColumns(rowType.columns()); + headerMeta.columns = FlatGeoBufTypeConversion.asColumns(rowType.columns()); HeaderMeta.write(headerMeta, outputStream, bufferBuilder); var indexSize = @@ -188,7 +187,7 @@ public class FlatGeoBufDataTable extends AbstractDataTable { var column = headerMeta.columns.get(i); var value = properties.get(i); propertiesBuffer.putShort((short) i); - TableConversions.writeValue(propertiesBuffer, column, value); + FlatGeoBufTypeConversion.writeValue(propertiesBuffer, column, value); } if (propertiesBuffer.position() > 0) { propertiesBuffer.flip(); @@ -268,7 +267,7 @@ public class FlatGeoBufDataTable extends AbstractDataTable { var featureSize = buffer.getInt(); var row = - TableConversions.asRow(headerMeta, rowType, Feature.getRootAsFeature(buffer)); + FlatGeoBufTypeConversion.asRow(headerMeta, rowType, Feature.getRootAsFeature(buffer)); buffer.position(Integer.BYTES + featureSize); buffer.compact(); diff --git a/baremaps-core/src/main/java/org/apache/baremaps/storage/flatgeobuf/internal/TableConversions.java b/baremaps-core/src/main/java/org/apache/baremaps/storage/flatgeobuf/FlatGeoBufTypeConversion.java similarity index 98% rename from baremaps-core/src/main/java/org/apache/baremaps/storage/flatgeobuf/internal/TableConversions.java rename to baremaps-core/src/main/java/org/apache/baremaps/storage/flatgeobuf/FlatGeoBufTypeConversion.java index 82f51a20..0974425a 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/storage/flatgeobuf/internal/TableConversions.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/storage/flatgeobuf/FlatGeoBufTypeConversion.java @@ -10,7 +10,7 @@ * the License. */ -package org.apache.baremaps.storage.flatgeobuf.internal; +package org.apache.baremaps.storage.flatgeobuf; import com.google.flatbuffers.FlatBufferBuilder; @@ -30,7 +30,7 @@ import org.wololo.flatgeobuf.generated.Crs; import org.wololo.flatgeobuf.generated.Feature; import org.wololo.flatgeobuf.generated.Header; -public class TableConversions { +public class FlatGeoBufTypeConversion { public static final EnumMap<Type, Integer> types = new EnumMap<>(Type.class); @@ -194,7 +194,7 @@ public class TableConversions { public static List<ColumnMeta> asColumns(List<DataColumn> columns) { return columns.stream() - .map(TableConversions::asColumn) + .map(FlatGeoBufTypeConversion::asColumn) .filter(Objects::nonNull) .collect(Collectors.toList()); } diff --git a/baremaps-core/src/main/java/org/apache/baremaps/storage/postgres/PostgresDataSchema.java b/baremaps-core/src/main/java/org/apache/baremaps/storage/postgres/PostgresDataSchema.java index 5a6e3a5d..e21703c4 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/storage/postgres/PostgresDataSchema.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/storage/postgres/PostgresDataSchema.java @@ -14,15 +14,9 @@ package org.apache.baremaps.storage.postgres; import de.bytefish.pgbulkinsert.pgsql.handlers.*; -import java.net.Inet4Address; -import java.net.Inet6Address; import java.sql.SQLException; -import java.time.LocalDate; -import java.time.LocalDateTime; -import java.time.LocalTime; import java.util.Collection; import java.util.List; -import java.util.Map; import java.util.stream.Collectors; import javax.sql.DataSource; import org.apache.baremaps.database.schema.*; @@ -31,7 +25,6 @@ import org.apache.baremaps.postgres.copy.CopyWriter; import org.apache.baremaps.postgres.copy.GeometryValueHandler; import org.apache.baremaps.postgres.metadata.DatabaseMetadata; import org.apache.baremaps.postgres.metadata.TableMetadata; -import org.locationtech.jts.geom.*; import org.postgresql.PGConnection; import org.postgresql.copy.PGCopyOutputStream; import org.slf4j.Logger; @@ -46,41 +39,6 @@ public class PostgresDataSchema implements DataSchema { private static final String[] TYPES = new String[] {"TABLE", "VIEW"}; - protected static final Map<Class, String> typeToName = Map.ofEntries( - Map.entry(String.class, "varchar"), - Map.entry(Short.class, "int2"), - Map.entry(Integer.class, "int4"), - Map.entry(Long.class, "int8"), - Map.entry(Float.class, "float4"), - Map.entry(Double.class, "float8"), - Map.entry(Geometry.class, "geometry"), - Map.entry(MultiPoint.class, "geometry"), - Map.entry(Point.class, "geometry"), - Map.entry(LineString.class, "geometry"), - Map.entry(MultiLineString.class, "geometry"), - Map.entry(Polygon.class, "geometry"), - Map.entry(MultiPolygon.class, "geometry"), - Map.entry(LinearRing.class, "geometry"), - Map.entry(GeometryCollection.class, "geometry"), - Map.entry(Inet4Address.class, "inet"), - Map.entry(Inet6Address.class, "inet"), - Map.entry(LocalDate.class, "date"), - Map.entry(LocalTime.class, "time"), - Map.entry(LocalDateTime.class, "timestamp")); - - protected static final Map<String, Type> nameToType = Map.ofEntries( - Map.entry("varchar", Type.STRING), - Map.entry("int2", Type.SHORT), - Map.entry("int4", Type.INTEGER), - Map.entry("int8", Type.LONG), - Map.entry("float4", Type.FLOAT), - Map.entry("float8", Type.DOUBLE), - Map.entry("geometry", Type.GEOMETRY), - Map.entry("inet", Type.INET6_ADDRESS), - Map.entry("date", Type.LOCAL_DATE), - Map.entry("time", Type.LOCAL_TIME), - Map.entry("timestamp", Type.LOCAL_DATE_TIME)); - private final DataSource dataSource; /** @@ -189,7 +147,8 @@ public class PostgresDataSchema implements DataSchema { protected static DataRowType createRowType(TableMetadata tableMetadata) { var name = tableMetadata.table().tableName(); var columns = tableMetadata.columns().stream() - .map(column -> new DataColumnImpl(column.columnName(), nameToType.get(column.typeName()))) + .map(column -> new DataColumnImpl(column.columnName(), + PostgresTypeConversion.nameToType.get(column.typeName()))) .map(DataColumn.class::cast) .toList(); return new DataRowTypeImpl(name, columns); @@ -204,7 +163,7 @@ public class PostgresDataSchema implements DataSchema { protected DataRowType adaptDataType(DataRowType rowType) { var name = rowType.name().replaceAll("[^a-zA-Z0-9]", "_"); var properties = rowType.columns().stream() - .filter(column -> typeToName.containsKey(column.type())) + .filter(column -> PostgresTypeConversion.typeToName.containsKey(column.type())) .map(column -> (DataColumn) new DataColumnImpl(column.name(), column.type())) .toList(); return new DataRowTypeImpl(name, properties); @@ -233,7 +192,7 @@ public class PostgresDataSchema implements DataSchema { builder.append("\" ("); builder.append(rowType.columns().stream() .map(column -> "\"" + column.name() - + "\" " + typeToName.get(column.type())) + + "\" " + PostgresTypeConversion.typeToName.get(column.type())) .collect(Collectors.joining(", "))); builder.append(")"); return builder.toString(); @@ -254,6 +213,7 @@ public class PostgresDataSchema implements DataSchema { .map(column -> "\"" + column.name() + "\"") .collect(Collectors.joining(", "))); builder.append(") FROM STDIN BINARY"); + System.out.println(builder.toString()); return builder.toString(); } @@ -320,6 +280,6 @@ public class PostgresDataSchema implements DataSchema { * @return true if the column type is supported */ protected boolean isSupported(DataColumn column) { - return typeToName.containsKey(column.type()); + return PostgresTypeConversion.typeToName.containsKey(column.type()); } } diff --git a/baremaps-core/src/main/java/org/apache/baremaps/storage/postgres/PostgresTypeConversion.java b/baremaps-core/src/main/java/org/apache/baremaps/storage/postgres/PostgresTypeConversion.java new file mode 100644 index 00000000..aa2080e4 --- /dev/null +++ b/baremaps-core/src/main/java/org/apache/baremaps/storage/postgres/PostgresTypeConversion.java @@ -0,0 +1,59 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + */ + +package org.apache.baremaps.storage.postgres; + +import java.util.EnumMap; +import java.util.Map; +import org.apache.baremaps.database.schema.DataColumn.Type; + +public class PostgresTypeConversion { + + public static final EnumMap<Type, String> typeToName = new EnumMap<>(Type.class); + + static { + typeToName.put(Type.STRING, "varchar"); + typeToName.put(Type.SHORT, "int2"); + typeToName.put(Type.INTEGER, "int4"); + typeToName.put(Type.LONG, "int8"); + typeToName.put(Type.FLOAT, "float4"); + typeToName.put(Type.DOUBLE, "float8"); + typeToName.put(Type.GEOMETRY, "geometry"); + typeToName.put(Type.POINT, "geometry"); + typeToName.put(Type.MULTIPOINT, "geometry"); + typeToName.put(Type.LINESTRING, "geometry"); + typeToName.put(Type.MULTILINESTRING, "geometry"); + typeToName.put(Type.POLYGON, "geometry"); + typeToName.put(Type.MULTIPOLYGON, "geometry"); + typeToName.put(Type.GEOMETRYCOLLECTION, "geometry"); + typeToName.put(Type.INET_ADDRESS, "inet"); + typeToName.put(Type.INET4_ADDRESS, "inet"); + typeToName.put(Type.INET6_ADDRESS, "inet"); + typeToName.put(Type.LOCAL_DATE, "date"); + typeToName.put(Type.LOCAL_TIME, "time"); + typeToName.put(Type.LOCAL_DATE_TIME, "timestamp"); + } + + protected static final Map<String, Type> nameToType = Map.ofEntries( + Map.entry("varchar", Type.STRING), + Map.entry("int2", Type.SHORT), + Map.entry("int4", Type.INTEGER), + Map.entry("int8", Type.LONG), + Map.entry("float4", Type.FLOAT), + Map.entry("float8", Type.DOUBLE), + Map.entry("geometry", Type.GEOMETRY), + Map.entry("inet", Type.INET6_ADDRESS), + Map.entry("date", Type.LOCAL_DATE), + Map.entry("time", Type.LOCAL_TIME), + Map.entry("timestamp", Type.LOCAL_DATE_TIME)); + +} diff --git a/baremaps-core/src/test/java/org/apache/baremaps/tilestore/postgres/PostgresTileDataSchemaTest.java b/baremaps-core/src/test/java/org/apache/baremaps/tilestore/postgres/PostgresTileStoreTest.java similarity index 99% rename from baremaps-core/src/test/java/org/apache/baremaps/tilestore/postgres/PostgresTileDataSchemaTest.java rename to baremaps-core/src/test/java/org/apache/baremaps/tilestore/postgres/PostgresTileStoreTest.java index b23954f8..ecfe479e 100644 --- a/baremaps-core/src/test/java/org/apache/baremaps/tilestore/postgres/PostgresTileDataSchemaTest.java +++ b/baremaps-core/src/test/java/org/apache/baremaps/tilestore/postgres/PostgresTileStoreTest.java @@ -19,7 +19,7 @@ import java.util.List; import org.apache.baremaps.tilestore.TileCoord; import org.junit.jupiter.api.Test; -class PostgresTileDataSchemaTest { +class PostgresTileStoreTest { @Test void sameQueries() {
