This is an automated email from the ASF dual-hosted git repository. bchapuis pushed a commit to branch 849-nested-type-and-jsonb in repository https://gitbox.apache.org/repos/asf/incubator-baremaps.git
commit 29a147869ba065ba9bca84ccee01771d495cf787 Author: Bertil Chapuis <[email protected]> AuthorDate: Fri May 31 23:14:58 2024 +0200 Add support for nested types in data table --- .../baremaps/database/copy/JsonbValueHandler.java | 60 ++++++++++++++++++ .../database/metadata/DatabaseMetadata.java | 36 +++++++---- .../baremaps/geocoder/GeonamesQueryBuilder.java | 2 +- .../org/apache/baremaps/iploc/IpLocMapper.java | 4 +- .../storage/flatgeobuf/FlatGeoBufDataTable.java | 7 ++- .../flatgeobuf/FlatGeoBufTypeConversion.java | 6 +- .../storage/geopackage/GeoPackageDataTable.java | 5 +- .../geoparquet/GeoParquetTypeConversion.java | 72 ++++++++++++++++------ .../storage/postgres/PostgresDataStore.java | 9 ++- .../storage/postgres/PostgresTypeConversion.java | 4 +- .../shapefile/internal/DbaseByteReader.java | 2 +- .../shapefile/internal/ShapefileByteReader.java | 7 ++- .../shapefile/internal/ShapefileInputStream.java | 4 +- .../shapefile/internal/ShapefileReader.java | 4 +- .../org/apache/baremaps/calcite/CalciteTest.java | 11 ++-- .../org/apache/baremaps/storage/MockDataTable.java | 11 ++-- .../geoparquet/GeoParquetToPostgresTest.java | 2 +- .../baremaps/data/calcite/SqlTypeConversion.java | 14 ----- .../apache/baremaps/data/storage/DataColumn.java | 20 +++--- .../{DataColumnImpl.java => DataColumnFixed.java} | 3 +- .../{DataColumnImpl.java => DataColumnNested.java} | 12 ++-- .../baremaps/data/type/DataTypeProvider.java | 35 ++++++----- .../geoparquet/data/GeoParquetGroupFactory.java | 3 +- .../geoparquet/data/GeoParquetGroupImpl.java | 20 +++--- 24 files changed, 239 insertions(+), 114 deletions(-) diff --git a/baremaps-core/src/main/java/org/apache/baremaps/database/copy/JsonbValueHandler.java b/baremaps-core/src/main/java/org/apache/baremaps/database/copy/JsonbValueHandler.java new file mode 100644 index 00000000..0bea44c4 --- /dev/null +++ b/baremaps-core/src/main/java/org/apache/baremaps/database/copy/JsonbValueHandler.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.baremaps.database.copy; + +import com.fasterxml.jackson.databind.ObjectMapper; +import de.bytefish.pgbulkinsert.pgsql.handlers.BaseValueHandler; +import java.io.DataOutputStream; + +public class JsonbValueHandler extends BaseValueHandler<Object> { + + private final int jsonbProtocolVersion; + + public JsonbValueHandler() { + this(1); + } + + public JsonbValueHandler(int jsonbProtocolVersion) { + this.jsonbProtocolVersion = jsonbProtocolVersion; + } + + private static byte[] asJson(Object object) { + try { + ObjectMapper objectMapper = new ObjectMapper(); + String value = objectMapper.writeValueAsString(object); + return value.getBytes("UTF-8"); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + @Override + protected void internalHandle(DataOutputStream buffer, Object value) throws Exception { + byte[] utf8Bytes = asJson(value); + buffer.writeInt(utf8Bytes.length + 1); + buffer.writeByte(jsonbProtocolVersion); + buffer.write(utf8Bytes); + } + + + @Override + public int getLength(Object value) { + byte[] utf8Bytes = asJson(value); + return utf8Bytes.length; + } +} diff --git a/baremaps-core/src/main/java/org/apache/baremaps/database/metadata/DatabaseMetadata.java b/baremaps-core/src/main/java/org/apache/baremaps/database/metadata/DatabaseMetadata.java index ff726ed3..c3b67575 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/database/metadata/DatabaseMetadata.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/database/metadata/DatabaseMetadata.java @@ -82,18 +82,30 @@ public class DatabaseMetadata { var resultSet = connection.getMetaData().getColumns(catalog, schemaPattern, tableNamePattern, columnNamePattern)) { while (resultSet.next()) { - tableColumns.add(new ColumnResult(resultSet.getString("TABLE_CAT"), - resultSet.getString("TABLE_SCHEM"), resultSet.getString("TABLE_NAME"), - resultSet.getString("COLUMN_NAME"), resultSet.getInt("DATA_TYPE"), - resultSet.getString("TYPE_NAME"), resultSet.getInt("COLUMN_SIZE"), - resultSet.getInt("DECIMAL_DIGITS"), resultSet.getInt("NUM_PREC_RADIX"), - resultSet.getInt("NULLABLE"), resultSet.getString("REMARKS"), - resultSet.getString("COLUMN_DEF"), resultSet.getInt("SQL_DATA_TYPE"), - resultSet.getInt("SQL_DATETIME_SUB"), resultSet.getInt("CHAR_OCTET_LENGTH"), - resultSet.getInt("ORDINAL_POSITION"), resultSet.getString("IS_NULLABLE"), - resultSet.getString("SCOPE_CATALOG"), resultSet.getString("SCOPE_SCHEMA"), - resultSet.getString("SCOPE_TABLE"), resultSet.getShort("SOURCE_DATA_TYPE"), - resultSet.getString("IS_AUTOINCREMENT"), resultSet.getString("IS_GENERATEDCOLUMN"))); + tableColumns.add(new ColumnResult( + resultSet.getString("TABLE_CAT"), + resultSet.getString("TABLE_SCHEM"), + resultSet.getString("TABLE_NAME"), + resultSet.getString("COLUMN_NAME"), + resultSet.getInt("DATA_TYPE"), + resultSet.getString("TYPE_NAME"), + resultSet.getInt("COLUMN_SIZE"), + resultSet.getInt("DECIMAL_DIGITS"), + resultSet.getInt("NUM_PREC_RADIX"), + resultSet.getInt("NULLABLE"), + resultSet.getString("REMARKS"), + resultSet.getString("COLUMN_DEF"), + resultSet.getInt("SQL_DATA_TYPE"), + resultSet.getInt("SQL_DATETIME_SUB"), + resultSet.getInt("CHAR_OCTET_LENGTH"), + resultSet.getInt("ORDINAL_POSITION"), + resultSet.getString("IS_NULLABLE"), + resultSet.getString("SCOPE_CATALOG"), + resultSet.getString("SCOPE_SCHEMA"), + resultSet.getString("SCOPE_TABLE"), + resultSet.getShort("SOURCE_DATA_TYPE"), + resultSet.getString("IS_AUTOINCREMENT"), + resultSet.getString("IS_GENERATEDCOLUMN"))); } } catch (SQLException e) { throw new RuntimeException(e); diff --git a/baremaps-core/src/main/java/org/apache/baremaps/geocoder/GeonamesQueryBuilder.java b/baremaps-core/src/main/java/org/apache/baremaps/geocoder/GeonamesQueryBuilder.java index 7cef357f..9c6f7ec4 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/geocoder/GeonamesQueryBuilder.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/geocoder/GeonamesQueryBuilder.java @@ -98,7 +98,7 @@ public class GeonamesQueryBuilder { if (queryText != null) { var queryTextEsc = QueryParser.escape(queryText); if (!queryTextEsc.isBlank()) { - // Changing the fields here might affect queries using queryText. + // Changing the columns here might affect queries using queryText. var fieldWeights = Map.of("name", 1f, "asciiname", 1f, "country", 1f, "countryCode", 1f); var parser = new SimpleQueryParser(analyzer, fieldWeights); if (andOperator) { diff --git a/baremaps-core/src/main/java/org/apache/baremaps/iploc/IpLocMapper.java b/baremaps-core/src/main/java/org/apache/baremaps/iploc/IpLocMapper.java index 5448c2b6..159dfc91 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/iploc/IpLocMapper.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/iploc/IpLocMapper.java @@ -99,14 +99,14 @@ public class IpLocMapper implements Function<NicObject, Optional<IpLocObject>> { } } - // If there is a country, we use that with a cherry-picked list of fields to query the + // If there is a country, we use that with a cherry-picked list of columns to query the // geocoder with confidence to find a relevant precise location, // in the worst case the error is within a country List<String> searchedFields = List.of("descr", "netname"); // at least one of a searchedField is present and the country is present. if (attributes.keySet().stream().anyMatch(searchedFields::contains) && attributes.containsKey("country")) { - // build a query text string out of the cherry-picked fields + // build a query text string out of the cherry-picked columns var queryTextBuilder = new StringBuilder(); for (String field : searchedFields) { if (!Strings.isNullOrEmpty(attributes.get(field))) { diff --git a/baremaps-core/src/main/java/org/apache/baremaps/storage/flatgeobuf/FlatGeoBufDataTable.java b/baremaps-core/src/main/java/org/apache/baremaps/storage/flatgeobuf/FlatGeoBufDataTable.java index a7c9c93c..f3598d28 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/storage/flatgeobuf/FlatGeoBufDataTable.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/storage/flatgeobuf/FlatGeoBufDataTable.java @@ -57,7 +57,12 @@ public class FlatGeoBufDataTable implements DataTable { this.schema = readSchema(file); } - + /** + * Reads the schema from a flatgeobuf file. + * + * @param file the path to the flatgeobuf file + * @return the schema of the table + */ private static DataSchema readSchema(Path file) { try (var channel = FileChannel.open(file, StandardOpenOption.READ)) { // try to read the schema from the file diff --git a/baremaps-core/src/main/java/org/apache/baremaps/storage/flatgeobuf/FlatGeoBufTypeConversion.java b/baremaps-core/src/main/java/org/apache/baremaps/storage/flatgeobuf/FlatGeoBufTypeConversion.java index 58c1074c..1a42148c 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/storage/flatgeobuf/FlatGeoBufTypeConversion.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/storage/flatgeobuf/FlatGeoBufTypeConversion.java @@ -26,6 +26,7 @@ import java.nio.charset.StandardCharsets; import java.util.*; import java.util.stream.Collectors; import org.apache.baremaps.data.storage.*; +import org.apache.baremaps.data.storage.DataColumn.Cardinality; import org.apache.baremaps.data.storage.DataColumn.Type; import org.wololo.flatgeobuf.ColumnMeta; import org.wololo.flatgeobuf.GeometryConversions; @@ -53,7 +54,10 @@ public class FlatGeoBufTypeConversion { public static DataSchema asSchema(HeaderMeta headerMeta) { var name = headerMeta.name; var columns = headerMeta.columns.stream() - .map(column -> new DataColumnImpl(column.name, Type.fromBinding(column.getBinding()))) + .map(column -> new DataColumnFixed( + column.name, + column.nullable ? Cardinality.OPTIONAL : Cardinality.REQUIRED, + Type.fromBinding(column.getBinding()))) .map(DataColumn.class::cast) .toList(); return new DataSchemaImpl(name, columns); diff --git a/baremaps-core/src/main/java/org/apache/baremaps/storage/geopackage/GeoPackageDataTable.java b/baremaps-core/src/main/java/org/apache/baremaps/storage/geopackage/GeoPackageDataTable.java index ed6bc89c..5518cb4b 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/storage/geopackage/GeoPackageDataTable.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/storage/geopackage/GeoPackageDataTable.java @@ -24,6 +24,7 @@ import mil.nga.geopackage.features.user.FeatureDao; import mil.nga.geopackage.features.user.FeatureResultSet; import mil.nga.geopackage.geom.GeoPackageGeometryData; import org.apache.baremaps.data.storage.*; +import org.apache.baremaps.data.storage.DataColumn.Cardinality; import org.apache.baremaps.data.storage.DataColumn.Type; import org.locationtech.jts.geom.*; @@ -50,7 +51,9 @@ public class GeoPackageDataTable implements DataTable { for (FeatureColumn column : featureDao.getColumns()) { var propertyName = column.getName(); var propertyType = classType(column); - columns.add(new DataColumnImpl(propertyName, propertyType)); + var propertyCardinality = column.isNotNull() ? Cardinality.REQUIRED : Cardinality.OPTIONAL; + columns.add(new DataColumnFixed( + propertyName, propertyCardinality, propertyType)); } schema = new DataSchemaImpl(name, columns); geometryFactory = new GeometryFactory(new PrecisionModel(), (int) featureDao.getSrs().getId()); diff --git a/baremaps-core/src/main/java/org/apache/baremaps/storage/geoparquet/GeoParquetTypeConversion.java b/baremaps-core/src/main/java/org/apache/baremaps/storage/geoparquet/GeoParquetTypeConversion.java index 19c09e41..8485c007 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/storage/geoparquet/GeoParquetTypeConversion.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/storage/geoparquet/GeoParquetTypeConversion.java @@ -18,14 +18,15 @@ package org.apache.baremaps.storage.geoparquet; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; -import org.apache.baremaps.data.storage.DataColumn; +import java.util.Map; +import org.apache.baremaps.data.storage.*; +import org.apache.baremaps.data.storage.DataColumn.Cardinality; import org.apache.baremaps.data.storage.DataColumn.Type; -import org.apache.baremaps.data.storage.DataColumnImpl; -import org.apache.baremaps.data.storage.DataSchema; -import org.apache.baremaps.data.storage.DataSchemaImpl; import org.apache.baremaps.geoparquet.data.GeoParquetGroup; import org.apache.baremaps.geoparquet.data.GeoParquetGroup.Field; +import org.apache.baremaps.geoparquet.data.GeoParquetGroup.GroupField; import org.apache.baremaps.geoparquet.data.GeoParquetGroup.Schema; public class GeoParquetTypeConversion { @@ -33,23 +34,33 @@ public class GeoParquetTypeConversion { private GeoParquetTypeConversion() {} public static DataSchema asSchema(String table, Schema schema) { - List<DataColumn> columns = schema.fields().stream() - .map(field -> (DataColumn) new DataColumnImpl(field.name(), asSchema(field.type()))) - .toList(); + List<DataColumn> columns = asDataColumns(schema); return new DataSchemaImpl(table, columns); } - public static Type asSchema(GeoParquetGroup.Type type) { - return switch (type) { - case BINARY -> Type.BYTE_ARRAY; - case BOOLEAN -> Type.BOOLEAN; - case INTEGER -> Type.INTEGER; - case INT96, LONG -> Type.LONG; - case FLOAT -> Type.FLOAT; - case DOUBLE -> Type.DOUBLE; - case STRING -> Type.STRING; - case GEOMETRY -> Type.GEOMETRY; - case GROUP -> null; + private static List<DataColumn> asDataColumns(Schema field) { + return field.fields().stream() + .map(GeoParquetTypeConversion::asDataColumn) + .toList(); + } + + private static DataColumn asDataColumn(Field field) { + Cardinality cardinality = switch (field.cardinality()) { + case REQUIRED -> Cardinality.REQUIRED; + case OPTIONAL -> Cardinality.OPTIONAL; + case REPEATED -> Cardinality.REPEATED; + }; + return switch (field.type()) { + case BINARY -> new DataColumnFixed(field.name(), cardinality, Type.BYTE); + case BOOLEAN -> new DataColumnFixed(field.name(), cardinality, Type.BOOLEAN); + case INTEGER -> new DataColumnFixed(field.name(), cardinality, Type.INTEGER); + case INT96, LONG -> new DataColumnFixed(field.name(), cardinality, Type.LONG); + case FLOAT -> new DataColumnFixed(field.name(), cardinality, Type.FLOAT); + case DOUBLE -> new DataColumnFixed(field.name(), cardinality, Type.DOUBLE); + case STRING -> new DataColumnFixed(field.name(), cardinality, Type.STRING); + case GEOMETRY -> new DataColumnFixed(field.name(), cardinality, Type.GEOMETRY); + case GROUP -> new DataColumnNested(field.name(), cardinality, + asDataColumns(((GroupField) field).schema())); }; } @@ -59,7 +70,6 @@ public class GeoParquetTypeConversion { List<Field> fields = schema.fields(); for (int i = 0; i < fields.size(); i++) { Field field = fields.get(i); - field.type(); switch (field.type()) { case BINARY -> values.add(group.getBinaryValue(i).getBytes()); case BOOLEAN -> values.add(group.getBooleanValue(i)); @@ -69,9 +79,31 @@ public class GeoParquetTypeConversion { case DOUBLE -> values.add(group.getDoubleValue(i)); case STRING -> values.add(group.getStringValue(i)); case GEOMETRY -> values.add(group.getGeometryValue(i)); - case GROUP -> values.add(null); // TODO: values.add(asDataRow(group.getGroupValue(i))); + case GROUP -> values.add(asNested(group.getGroupValue(i))); } } return values; } + + public static Map<String, Object> asNested(GeoParquetGroup group) { + Map<String, Object> nested = new HashMap<>(); + Schema schema = group.getSchema(); + List<Field> fields = schema.fields(); + for (int i = 0; i < fields.size(); i++) { + Field field = fields.get(i); + nested.put(field.name(), switch (field.type()) { + case BINARY -> group.getBinaryValue(i).getBytes(); + case BOOLEAN -> group.getBooleanValue(i); + case INTEGER -> group.getIntegerValue(i); + case INT96, LONG -> group.getLongValue(i); + case FLOAT -> group.getFloatValue(i); + case DOUBLE -> group.getDoubleValue(i); + case STRING -> group.getStringValue(i); + case GEOMETRY -> group.getGeometryValue(i); + case GROUP -> asNested(group.getGroupValue(i)); + }); + } + return nested; + } + } diff --git a/baremaps-core/src/main/java/org/apache/baremaps/storage/postgres/PostgresDataStore.java b/baremaps-core/src/main/java/org/apache/baremaps/storage/postgres/PostgresDataStore.java index eabd5c59..a3262847 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/storage/postgres/PostgresDataStore.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/storage/postgres/PostgresDataStore.java @@ -29,6 +29,7 @@ import org.apache.baremaps.data.storage.*; import org.apache.baremaps.data.storage.DataColumn.Type; import org.apache.baremaps.database.copy.CopyWriter; import org.apache.baremaps.database.copy.GeometryValueHandler; +import org.apache.baremaps.database.copy.JsonbValueHandler; import org.apache.baremaps.database.metadata.DatabaseMetadata; import org.apache.baremaps.database.metadata.TableMetadata; import org.postgresql.PGConnection; @@ -106,7 +107,7 @@ public class PostgresDataStore implements DataStore { if (PostgresTypeConversion.typeToName.containsKey(column.type())) { var columnName = column.name().replaceAll(REGEX, "_").toLowerCase(); mapping.put(columnName, column.name()); - properties.add(new DataColumnImpl(columnName, column.type())); + properties.add(new DataColumnFixed(columnName, column.cardinality(), column.type())); } } @@ -177,7 +178,10 @@ public class PostgresDataStore implements DataStore { protected static DataSchema createSchema(TableMetadata tableMetadata) { var name = tableMetadata.table().tableName(); var columns = tableMetadata.columns().stream() - .map(column -> new DataColumnImpl(column.columnName(), + .map(column -> new DataColumnFixed( + column.columnName(), + column.isNullable().equals("NO") ? DataColumn.Cardinality.REQUIRED + : DataColumn.Cardinality.OPTIONAL, PostgresTypeConversion.nameToType.get(column.typeName()))) .map(DataColumn.class::cast) .toList(); @@ -276,6 +280,7 @@ public class PostgresDataStore implements DataStore { case LOCAL_TIME -> new LocalTimeValueHandler(); case LOCAL_DATE_TIME -> new LocalDateTimeValueHandler(); case GEOMETRY, POINT, MULTIPOINT, LINESTRING, MULTILINESTRING, POLYGON, MULTIPOLYGON, GEOMETRYCOLLECTION -> new GeometryValueHandler(); + case NESTED -> new JsonbValueHandler(); default -> throw new IllegalArgumentException("Unsupported type: " + type); }; } diff --git a/baremaps-core/src/main/java/org/apache/baremaps/storage/postgres/PostgresTypeConversion.java b/baremaps-core/src/main/java/org/apache/baremaps/storage/postgres/PostgresTypeConversion.java index 4188ade5..33720ae7 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/storage/postgres/PostgresTypeConversion.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/storage/postgres/PostgresTypeConversion.java @@ -46,6 +46,7 @@ public class PostgresTypeConversion { typeToName.put(Type.LOCAL_DATE, "date"); typeToName.put(Type.LOCAL_TIME, "time"); typeToName.put(Type.LOCAL_DATE_TIME, "timestamp"); + typeToName.put(Type.NESTED, "jsonb"); } protected static final Map<String, Type> nameToType = Map.ofEntries( @@ -59,6 +60,7 @@ public class PostgresTypeConversion { Map.entry("inet", Type.INET6_ADDRESS), Map.entry("date", Type.LOCAL_DATE), Map.entry("time", Type.LOCAL_TIME), - Map.entry("timestamp", Type.LOCAL_DATE_TIME)); + Map.entry("timestamp", Type.LOCAL_DATE_TIME), + Map.entry("jsonb", Type.NESTED)); } diff --git a/baremaps-core/src/main/java/org/apache/baremaps/storage/shapefile/internal/DbaseByteReader.java b/baremaps-core/src/main/java/org/apache/baremaps/storage/shapefile/internal/DbaseByteReader.java index 60975998..1ccfa84a 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/storage/shapefile/internal/DbaseByteReader.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/storage/shapefile/internal/DbaseByteReader.java @@ -291,7 +291,7 @@ public class DbaseByteReader extends CommonByteReader implements AutoCloseable { } /** - * Returns the fields descriptors in their binary format. + * Returns the columns descriptors in their binary format. * * @return Fields descriptors. */ diff --git a/baremaps-core/src/main/java/org/apache/baremaps/storage/shapefile/internal/ShapefileByteReader.java b/baremaps-core/src/main/java/org/apache/baremaps/storage/shapefile/internal/ShapefileByteReader.java index d733067b..841c2d3e 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/storage/shapefile/internal/ShapefileByteReader.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/storage/shapefile/internal/ShapefileByteReader.java @@ -25,6 +25,7 @@ import java.nio.MappedByteBuffer; import java.nio.channels.FileChannel; import java.util.*; import org.apache.baremaps.data.storage.*; +import org.apache.baremaps.data.storage.DataColumn.Cardinality; import org.apache.baremaps.data.storage.DataColumn.Type; import org.locationtech.jts.algorithm.Orientation; import org.locationtech.jts.geom.Coordinate; @@ -90,7 +91,7 @@ public class ShapefileByteReader extends CommonByteReader { } /** - * Returns the DBase 3 fields descriptors. + * Returns the DBase 3 columns descriptors. * * @return Fields descriptors. */ @@ -148,11 +149,11 @@ public class ShapefileByteReader extends CommonByteReader { case TimeStamp -> Type.STRING; case DateTime -> Type.STRING; }; - columns.add(new DataColumnImpl(columnName, columnType)); + columns.add(new DataColumnFixed(columnName, Cardinality.OPTIONAL, columnType)); } // Add geometry column. - columns.add(new DataColumnImpl(GEOMETRY_NAME, Type.GEOMETRY)); + columns.add(new DataColumnFixed(GEOMETRY_NAME, Cardinality.OPTIONAL, Type.GEOMETRY)); return new DataSchemaImpl(name, columns); } diff --git a/baremaps-core/src/main/java/org/apache/baremaps/storage/shapefile/internal/ShapefileInputStream.java b/baremaps-core/src/main/java/org/apache/baremaps/storage/shapefile/internal/ShapefileInputStream.java index bd7a1e9a..f89aab7c 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/storage/shapefile/internal/ShapefileInputStream.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/storage/shapefile/internal/ShapefileInputStream.java @@ -148,9 +148,9 @@ public class ShapefileInputStream extends InputStream { } /** - * Returns the database fields descriptors. + * Returns the database columns descriptors. * - * @return List of fields descriptors. + * @return List of columns descriptors. */ public List<DBaseFieldDescriptor> getDatabaseFieldsDescriptors() { return this.shapefileReader.getFieldsDescriptors(); diff --git a/baremaps-core/src/main/java/org/apache/baremaps/storage/shapefile/internal/ShapefileReader.java b/baremaps-core/src/main/java/org/apache/baremaps/storage/shapefile/internal/ShapefileReader.java index 6d41aa7b..22acfcf1 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/storage/shapefile/internal/ShapefileReader.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/storage/shapefile/internal/ShapefileReader.java @@ -136,9 +136,9 @@ public class ShapefileReader { } /** - * Returns the database fields descriptors. + * Returns the database columns descriptors. * - * @return List of fields descriptors. + * @return List of columns descriptors. */ public List<DBaseFieldDescriptor> getDatabaseFieldsDescriptors() { return this.databaseFieldsDescriptors; diff --git a/baremaps-core/src/test/java/org/apache/baremaps/calcite/CalciteTest.java b/baremaps-core/src/test/java/org/apache/baremaps/calcite/CalciteTest.java index d59123ae..13cf44d0 100644 --- a/baremaps-core/src/test/java/org/apache/baremaps/calcite/CalciteTest.java +++ b/baremaps-core/src/test/java/org/apache/baremaps/calcite/CalciteTest.java @@ -26,6 +26,7 @@ import org.apache.baremaps.data.calcite.SqlDataTable; import org.apache.baremaps.data.collection.AppendOnlyLog; import org.apache.baremaps.data.collection.IndexedDataList; import org.apache.baremaps.data.storage.*; +import org.apache.baremaps.data.storage.DataColumn.Cardinality; import org.apache.baremaps.data.storage.DataColumn.Type; import org.apache.baremaps.data.type.RowDataType; import org.apache.baremaps.maplibre.vectortile.VectorTileFunctions; @@ -74,9 +75,9 @@ public class CalciteTest { // Create the city table DataSchema cityRowType = new DataSchemaImpl("city", List.of( - new DataColumnImpl("id", Type.INTEGER), - new DataColumnImpl("name", Type.STRING), - new DataColumnImpl("geometry", Type.GEOMETRY))); + new DataColumnFixed("id", Cardinality.OPTIONAL, Type.INTEGER), + new DataColumnFixed("name", Cardinality.OPTIONAL, Type.STRING), + new DataColumnFixed("geometry", Cardinality.OPTIONAL, Type.GEOMETRY))); DataTable cityDataTable = new DataTableImpl( cityRowType, new IndexedDataList<>(new AppendOnlyLog<>(new RowDataType(cityRowType)))); @@ -89,8 +90,8 @@ public class CalciteTest { // Create the population table DataSchema populationRowType = new DataSchemaImpl("population", List.of( - new DataColumnImpl("city_id", Type.INTEGER), - new DataColumnImpl("population", Type.INTEGER))); + new DataColumnFixed("city_id", Cardinality.OPTIONAL, Type.INTEGER), + new DataColumnFixed("population", Cardinality.OPTIONAL, Type.INTEGER))); DataTable populationDataTable = new DataTableImpl( populationRowType, new IndexedDataList<>(new AppendOnlyLog<>(new RowDataType(populationRowType)))); diff --git a/baremaps-core/src/test/java/org/apache/baremaps/storage/MockDataTable.java b/baremaps-core/src/test/java/org/apache/baremaps/storage/MockDataTable.java index f21bdd7f..86bcf176 100644 --- a/baremaps-core/src/test/java/org/apache/baremaps/storage/MockDataTable.java +++ b/baremaps-core/src/test/java/org/apache/baremaps/storage/MockDataTable.java @@ -22,6 +22,7 @@ import static org.apache.baremaps.database.repository.Constants.GEOMETRY_FACTORY import java.util.Iterator; import java.util.List; import org.apache.baremaps.data.storage.*; +import org.apache.baremaps.data.storage.DataColumn.Cardinality; import org.apache.baremaps.data.storage.DataColumn.Type; import org.locationtech.jts.geom.Coordinate; @@ -33,11 +34,11 @@ public class MockDataTable implements DataTable { public MockDataTable() { this.rowType = new DataSchemaImpl("mock", List.of( - new DataColumnImpl("string", Type.STRING), - new DataColumnImpl("integer", Type.INTEGER), - new DataColumnImpl("double", Type.DOUBLE), - new DataColumnImpl("float", Type.FLOAT), - new DataColumnImpl("geometry", Type.GEOMETRY))); + new DataColumnFixed("string", Cardinality.OPTIONAL, Type.STRING), + new DataColumnFixed("integer", Cardinality.OPTIONAL, Type.INTEGER), + new DataColumnFixed("double", Cardinality.OPTIONAL, Type.DOUBLE), + new DataColumnFixed("float", Cardinality.OPTIONAL, Type.FLOAT), + new DataColumnFixed("geometry", Cardinality.OPTIONAL, Type.GEOMETRY))); this.rows = List.of( new DataRowImpl(rowType, List.of("string", 1, 1.0, 1.0f, GEOMETRY_FACTORY.createPoint(new Coordinate(1, 1)))), diff --git a/baremaps-core/src/test/java/org/apache/baremaps/storage/geoparquet/GeoParquetToPostgresTest.java b/baremaps-core/src/test/java/org/apache/baremaps/storage/geoparquet/GeoParquetToPostgresTest.java index 198a63d2..12dabdfb 100644 --- a/baremaps-core/src/test/java/org/apache/baremaps/storage/geoparquet/GeoParquetToPostgresTest.java +++ b/baremaps-core/src/test/java/org/apache/baremaps/storage/geoparquet/GeoParquetToPostgresTest.java @@ -43,7 +43,7 @@ class GeoParquetToPostgresTest extends PostgresContainerTest { // Check the table in Postgres var postgresTable = postgresStore.get("geoparquet"); assertEquals("geoparquet", postgresTable.schema().name()); - assertEquals(3, postgresTable.schema().columns().size()); + assertEquals(4, postgresTable.schema().columns().size()); assertEquals(5L, postgresTable.size()); assertEquals(5L, postgresTable.stream().count()); } diff --git a/baremaps-data/src/main/java/org/apache/baremaps/data/calcite/SqlTypeConversion.java b/baremaps-data/src/main/java/org/apache/baremaps/data/calcite/SqlTypeConversion.java index e0116e59..78a6ee25 100644 --- a/baremaps-data/src/main/java/org/apache/baremaps/data/calcite/SqlTypeConversion.java +++ b/baremaps-data/src/main/java/org/apache/baremaps/data/calcite/SqlTypeConversion.java @@ -31,32 +31,18 @@ public class SqlTypeConversion { static { types.put(Type.BYTE, new JavaTypeFactoryImpl() .createSqlType(SqlTypeName.TINYINT)); - types.put(Type.BYTE_ARRAY, new JavaTypeFactoryImpl() - .createArrayType(new JavaTypeFactoryImpl().createSqlType(SqlTypeName.TINYINT), -1)); types.put(Type.BOOLEAN, new JavaTypeFactoryImpl() .createSqlType(SqlTypeName.BOOLEAN)); - types.put(Type.BOOLEAN_ARRAY, new JavaTypeFactoryImpl() - .createArrayType(new JavaTypeFactoryImpl().createSqlType(SqlTypeName.BOOLEAN), -1)); types.put(Type.SHORT, new JavaTypeFactoryImpl() .createSqlType(SqlTypeName.SMALLINT)); - types.put(Type.SHORT_ARRAY, new JavaTypeFactoryImpl() - .createArrayType(new JavaTypeFactoryImpl().createSqlType(SqlTypeName.SMALLINT), -1)); types.put(Type.INTEGER, new JavaTypeFactoryImpl() .createSqlType(SqlTypeName.INTEGER)); - types.put(Type.INTEGER_ARRAY, new JavaTypeFactoryImpl() - .createArrayType(new JavaTypeFactoryImpl().createSqlType(SqlTypeName.INTEGER), -1)); types.put(Type.LONG, new JavaTypeFactoryImpl() .createSqlType(SqlTypeName.BIGINT)); - types.put(Type.LONG_ARRAY, new JavaTypeFactoryImpl() - .createArrayType(new JavaTypeFactoryImpl().createSqlType(SqlTypeName.BIGINT), -1)); types.put(Type.FLOAT, new JavaTypeFactoryImpl() .createSqlType(SqlTypeName.FLOAT)); - types.put(Type.FLOAT_ARRAY, new JavaTypeFactoryImpl() - .createArrayType(new JavaTypeFactoryImpl().createSqlType(SqlTypeName.FLOAT), -1)); types.put(Type.DOUBLE, new JavaTypeFactoryImpl() .createSqlType(SqlTypeName.DOUBLE)); - types.put(Type.DOUBLE_ARRAY, new JavaTypeFactoryImpl() - .createArrayType(new JavaTypeFactoryImpl().createSqlType(SqlTypeName.DOUBLE), -1)); types.put(Type.STRING, new JavaTypeFactoryImpl() .createSqlType(SqlTypeName.VARCHAR)); types.put(Type.GEOMETRY, new JavaTypeFactoryImpl() diff --git a/baremaps-data/src/main/java/org/apache/baremaps/data/storage/DataColumn.java b/baremaps-data/src/main/java/org/apache/baremaps/data/storage/DataColumn.java index 9c70bd34..6fa554f3 100644 --- a/baremaps-data/src/main/java/org/apache/baremaps/data/storage/DataColumn.java +++ b/baremaps-data/src/main/java/org/apache/baremaps/data/storage/DataColumn.java @@ -23,6 +23,7 @@ import java.net.InetAddress; import java.time.LocalDate; import java.time.LocalDateTime; import java.time.LocalTime; +import java.util.Map; import org.locationtech.jts.geom.*; /** @@ -37,6 +38,15 @@ public interface DataColumn { */ String name(); + + Cardinality cardinality(); + + enum Cardinality { + REQUIRED, + OPTIONAL, + REPEATED + } + /** * Returns the type of the column. * @@ -49,19 +59,12 @@ public interface DataColumn { */ enum Type { BYTE(Byte.class), - BYTE_ARRAY(byte[].class), BOOLEAN(Boolean.class), - BOOLEAN_ARRAY(boolean[].class), SHORT(Short.class), - SHORT_ARRAY(short[].class), INTEGER(Integer.class), - INTEGER_ARRAY(int[].class), LONG(Long.class), - LONG_ARRAY(long[].class), FLOAT(Float.class), - FLOAT_ARRAY(float[].class), DOUBLE(Double.class), - DOUBLE_ARRAY(double[].class), STRING(String.class), COORDINATE(Coordinate.class), GEOMETRY(Geometry.class), @@ -77,7 +80,8 @@ public interface DataColumn { INET6_ADDRESS(Inet6Address.class), LOCAL_DATE(LocalDate.class), LOCAL_TIME(LocalTime.class), - LOCAL_DATE_TIME(LocalDateTime.class),; + LOCAL_DATE_TIME(LocalDateTime.class), + NESTED(Map.class); private final Class<?> binding; diff --git a/baremaps-data/src/main/java/org/apache/baremaps/data/storage/DataColumnImpl.java b/baremaps-data/src/main/java/org/apache/baremaps/data/storage/DataColumnFixed.java similarity index 89% copy from baremaps-data/src/main/java/org/apache/baremaps/data/storage/DataColumnImpl.java copy to baremaps-data/src/main/java/org/apache/baremaps/data/storage/DataColumnFixed.java index 400e6033..9b786e69 100644 --- a/baremaps-data/src/main/java/org/apache/baremaps/data/storage/DataColumnImpl.java +++ b/baremaps-data/src/main/java/org/apache/baremaps/data/storage/DataColumnFixed.java @@ -20,6 +20,7 @@ package org.apache.baremaps.data.storage; /** * A column in a table. */ -public record DataColumnImpl(String name, Type type) implements DataColumn { +public record DataColumnFixed(String name, Cardinality cardinality, + Type type) implements DataColumn { } diff --git a/baremaps-data/src/main/java/org/apache/baremaps/data/storage/DataColumnImpl.java b/baremaps-data/src/main/java/org/apache/baremaps/data/storage/DataColumnNested.java similarity index 80% rename from baremaps-data/src/main/java/org/apache/baremaps/data/storage/DataColumnImpl.java rename to baremaps-data/src/main/java/org/apache/baremaps/data/storage/DataColumnNested.java index 400e6033..179b5c8f 100644 --- a/baremaps-data/src/main/java/org/apache/baremaps/data/storage/DataColumnImpl.java +++ b/baremaps-data/src/main/java/org/apache/baremaps/data/storage/DataColumnNested.java @@ -17,9 +17,13 @@ package org.apache.baremaps.data.storage; -/** - * A column in a table. - */ -public record DataColumnImpl(String name, Type type) implements DataColumn { +import java.util.List; + +public record DataColumnNested(String name, Cardinality cardinality, + List<DataColumn> columns) implements DataColumn { + @Override + public Type type() { + return Type.NESTED; + } } diff --git a/baremaps-data/src/test/java/org/apache/baremaps/data/type/DataTypeProvider.java b/baremaps-data/src/test/java/org/apache/baremaps/data/type/DataTypeProvider.java index 0b2f98cf..eb0ec04d 100644 --- a/baremaps-data/src/test/java/org/apache/baremaps/data/type/DataTypeProvider.java +++ b/baremaps-data/src/test/java/org/apache/baremaps/data/type/DataTypeProvider.java @@ -22,6 +22,7 @@ import java.util.List; import java.util.Map; import java.util.stream.Stream; import org.apache.baremaps.data.storage.*; +import org.apache.baremaps.data.storage.DataColumn.Cardinality; import org.apache.baremaps.data.storage.DataColumn.Type; import org.apache.baremaps.data.storage.DataSchema; import org.apache.baremaps.data.storage.DataSchemaImpl; @@ -33,23 +34,23 @@ public class DataTypeProvider { private static final GeometryFactory geometryFactory = new GeometryFactory(); private static final DataSchema DATA_SCHEMA = new DataSchemaImpl("row", List.of( - new DataColumnImpl("byte", Type.BYTE), - new DataColumnImpl("boolean", Type.BOOLEAN), - new DataColumnImpl("short", Type.SHORT), - new DataColumnImpl("integer", Type.INTEGER), - new DataColumnImpl("long", Type.LONG), - new DataColumnImpl("float", Type.FLOAT), - new DataColumnImpl("double", Type.DOUBLE), - new DataColumnImpl("string", Type.STRING), - new DataColumnImpl("geometry", Type.GEOMETRY), - new DataColumnImpl("point", Type.POINT), - new DataColumnImpl("linestring", Type.LINESTRING), - new DataColumnImpl("polygon", Type.POLYGON), - new DataColumnImpl("multipoint", Type.MULTIPOINT), - new DataColumnImpl("multilinestring", Type.MULTILINESTRING), - new DataColumnImpl("multipolygon", Type.MULTIPOLYGON), - new DataColumnImpl("geometrycollection", Type.GEOMETRYCOLLECTION), - new DataColumnImpl("coordinate", Type.COORDINATE))); + new DataColumnFixed("byte", Cardinality.OPTIONAL, Type.BYTE), + new DataColumnFixed("boolean", Cardinality.OPTIONAL, Type.BOOLEAN), + new DataColumnFixed("short", Cardinality.OPTIONAL, Type.SHORT), + new DataColumnFixed("integer", Cardinality.OPTIONAL, Type.INTEGER), + new DataColumnFixed("long", Cardinality.OPTIONAL, Type.LONG), + new DataColumnFixed("float", Cardinality.OPTIONAL, Type.FLOAT), + new DataColumnFixed("double", Cardinality.OPTIONAL, Type.DOUBLE), + new DataColumnFixed("string", Cardinality.OPTIONAL, Type.STRING), + new DataColumnFixed("geometry", Cardinality.OPTIONAL, Type.GEOMETRY), + new DataColumnFixed("point", Cardinality.OPTIONAL, Type.POINT), + new DataColumnFixed("linestring", Cardinality.OPTIONAL, Type.LINESTRING), + new DataColumnFixed("polygon", Cardinality.OPTIONAL, Type.POLYGON), + new DataColumnFixed("multipoint", Cardinality.OPTIONAL, Type.MULTIPOINT), + new DataColumnFixed("multilinestring", Cardinality.OPTIONAL, Type.MULTILINESTRING), + new DataColumnFixed("multipolygon", Cardinality.OPTIONAL, Type.MULTIPOLYGON), + new DataColumnFixed("geometrycollection", Cardinality.OPTIONAL, Type.GEOMETRYCOLLECTION), + new DataColumnFixed("coordinate", Cardinality.OPTIONAL, Type.COORDINATE))); private static final DataRow DATA_ROW = DATA_SCHEMA.createRow() .with("byte", Byte.MAX_VALUE) diff --git a/baremaps-geoparquet/src/main/java/org/apache/baremaps/geoparquet/data/GeoParquetGroupFactory.java b/baremaps-geoparquet/src/main/java/org/apache/baremaps/geoparquet/data/GeoParquetGroupFactory.java index d89e0a03..14e4f081 100644 --- a/baremaps-geoparquet/src/main/java/org/apache/baremaps/geoparquet/data/GeoParquetGroupFactory.java +++ b/baremaps-geoparquet/src/main/java/org/apache/baremaps/geoparquet/data/GeoParquetGroupFactory.java @@ -64,8 +64,9 @@ public class GeoParquetGroupFactory { }; } else { GroupType groupType = field.asGroupType(); + GeoParquetGroup.Schema geoParquetSchema = createGeoParquetSchema(groupType, metadata); return (Field) new GeoParquetGroup.GroupField(groupType.getName(), - GeoParquetGroup.Cardinality.REQUIRED, createGeoParquetSchema(groupType, metadata)); + GeoParquetGroup.Cardinality.REQUIRED, geoParquetSchema); } }).toList(); return new GeoParquetGroup.Schema(schema.getName(), fields); diff --git a/baremaps-geoparquet/src/main/java/org/apache/baremaps/geoparquet/data/GeoParquetGroupImpl.java b/baremaps-geoparquet/src/main/java/org/apache/baremaps/geoparquet/data/GeoParquetGroupImpl.java index 2cd49058..7d140a62 100644 --- a/baremaps-geoparquet/src/main/java/org/apache/baremaps/geoparquet/data/GeoParquetGroupImpl.java +++ b/baremaps-geoparquet/src/main/java/org/apache/baremaps/geoparquet/data/GeoParquetGroupImpl.java @@ -38,7 +38,9 @@ public class GeoParquetGroupImpl implements GeoParquetGroup { private final List<?>[] data; - public GeoParquetGroupImpl(GroupType schema, GeoParquetMetadata metadata, + public GeoParquetGroupImpl( + GroupType schema, + GeoParquetMetadata metadata, Schema geoParquetSchema) { this.schema = schema; this.metadata = metadata; @@ -50,11 +52,9 @@ public class GeoParquetGroupImpl implements GeoParquetGroup { } public GeoParquetGroupImpl addGroup(int fieldIndex) { - GeoParquetGroupImpl g = - new GeoParquetGroupImpl(schema.getType(fieldIndex).asGroupType(), metadata, - geoParquetSchema); - add(fieldIndex, g); - return g; + GeoParquetGroupImpl group = createGroup(fieldIndex); + add(fieldIndex, group); + return group; } public GeoParquetGroupImpl addGroup(String field) { @@ -301,9 +301,11 @@ public class GeoParquetGroupImpl implements GeoParquetGroup { } @Override - public GeoParquetGroup createGroup(int fieldIndex) { - return new GeoParquetGroupImpl(schema.getType(fieldIndex).asGroupType(), metadata, - geoParquetSchema); + public GeoParquetGroupImpl createGroup(int fieldIndex) { + GroupField field = ((GroupField) geoParquetSchema.fields().get(fieldIndex)); + GeoParquetGroupImpl group = + new GeoParquetGroupImpl(schema.getType(fieldIndex).asGroupType(), metadata, field.schema()); + return group; } @Override
