This is an automated email from the ASF dual-hosted git repository. bchapuis pushed a commit to branch 619-transformation-workflow in repository https://gitbox.apache.org/repos/asf/incubator-baremaps.git
commit 07a5d4f7b08fb4b725dc773bfd47b41aec010614 Author: Bertil Chapuis <[email protected]> AuthorDate: Sat Apr 8 00:17:56 2023 +0200 Improve flatgeobuf serialization --- .../main/java/org/apache/baremaps/storage/Row.java | 2 +- .../storage/flatgeobuf/FlatGeoBufStore.java | 8 ++-- .../storage/flatgeobuf/FlatGeoBufTable.java | 52 ++++++++++++---------- .../flatgeobuf/internal/TableConversions.java | 4 +- 4 files changed, 38 insertions(+), 28 deletions(-) diff --git a/baremaps-core/src/main/java/org/apache/baremaps/storage/Row.java b/baremaps-core/src/main/java/org/apache/baremaps/storage/Row.java index 2af38e3f..3de90fbf 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/storage/Row.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/storage/Row.java @@ -31,7 +31,7 @@ public interface Row { * * @return the values of the columns in the row */ - List values(); + List<?> values(); /** * Returns the value of the specified column. diff --git a/baremaps-core/src/main/java/org/apache/baremaps/storage/flatgeobuf/FlatGeoBufStore.java b/baremaps-core/src/main/java/org/apache/baremaps/storage/flatgeobuf/FlatGeoBufStore.java index 2b0459ec..950597bd 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/storage/flatgeobuf/FlatGeoBufStore.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/storage/flatgeobuf/FlatGeoBufStore.java @@ -60,12 +60,14 @@ public class FlatGeoBufStore implements Store { */ @Override public void add(Table table) throws TableException { - var path = directory.resolve(table.schema().name()); + var filename = table.schema().name(); + filename = filename.endsWith(".fgb") ? filename : filename + ".fgb"; + var path = directory.resolve(filename); try { - Files.delete(path); + Files.deleteIfExists(path); Files.createFile(path); var flatGeoBufTable = new FlatGeoBufTable(path, table.schema()); - table.forEach(flatGeoBufTable::add); + flatGeoBufTable.write(table); } catch (IOException e) { throw new TableException(e); } diff --git a/baremaps-core/src/main/java/org/apache/baremaps/storage/flatgeobuf/FlatGeoBufTable.java b/baremaps-core/src/main/java/org/apache/baremaps/storage/flatgeobuf/FlatGeoBufTable.java index 5a13eeef..d428a220 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/storage/flatgeobuf/FlatGeoBufTable.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/storage/flatgeobuf/FlatGeoBufTable.java @@ -27,7 +27,7 @@ import org.apache.baremaps.storage.AbstractTable; import org.apache.baremaps.storage.Row; import org.apache.baremaps.storage.Schema; import org.apache.baremaps.storage.flatgeobuf.internal.TableConversions; -import org.locationtech.jts.geom.Geometry; +import org.locationtech.jts.geom.*; import org.wololo.flatgeobuf.Constants; import org.wololo.flatgeobuf.GeometryConversions; import org.wololo.flatgeobuf.HeaderMeta; @@ -160,6 +160,7 @@ public class FlatGeoBufTable extends AbstractTable { var headerMeta = new HeaderMeta(); headerMeta.geometryType = GeometryType.Unknown; headerMeta.indexNodeSize = 16; + headerMeta.srid = 3857; headerMeta.featuresCount = features instanceof AbstractDataCollection<Row>c ? c.sizeAsLong() : features.size(); headerMeta.name = schema.name(); @@ -175,38 +176,43 @@ public class FlatGeoBufTable extends AbstractTable { var iterator = features.iterator(); while (iterator.hasNext()) { + var featureBuilder = new FlatBufferBuilder(4096); + var row = iterator.next(); - var featureBuilder = new FlatBufferBuilder(); - var geometryOffset = 0; - var propertiesOffset = 0; + var propertiesBuffer = ByteBuffer.allocate(1 << 20).order(ByteOrder.LITTLE_ENDIAN); - var i = 0; - for (Object value : row.values()) { - if (value instanceof Geometry geometry) { - try { - geometryOffset = - GeometryConversions.serialize(featureBuilder, geometry, headerMeta.geometryType); - } catch (IOException e) { - throw new RuntimeException(e); - } - } else { - var column = headerMeta.columns.get(i); - propertiesBuffer.putShort((short) i); - TableConversions.writeValue(propertiesBuffer, column, value); - i++; - } + var properties = row.values().stream() + .filter(v -> !(v instanceof Geometry)) + .toList(); + for (int i = 0; i < properties.size(); i++) { + var column = headerMeta.columns.get(i); + var value = properties.get(i); + propertiesBuffer.putShort((short) i); + TableConversions.writeValue(propertiesBuffer, column, value); + } + if (propertiesBuffer.position() > 0) { + propertiesBuffer.flip(); } - propertiesBuffer.flip(); - propertiesOffset = org.wololo.flatgeobuf.generated.Feature + var propertiesOffset = org.wololo.flatgeobuf.generated.Feature .createPropertiesVector(featureBuilder, propertiesBuffer); + var geometry = row.values().stream() + .filter(v -> v instanceof Geometry) + .map(Geometry.class::cast) + .findFirst().get(); + var geometryOffset = geometry != null + ? GeometryConversions.serialize(featureBuilder, geometry, headerMeta.geometryType) + : 0; + var featureOffset = org.wololo.flatgeobuf.generated.Feature.createFeature(featureBuilder, geometryOffset, propertiesOffset, 0); - featureBuilder.finishSizePrefixed(featureOffset); - channel.write(featureBuilder.dataBuffer()); + ByteBuffer data = featureBuilder.dataBuffer(); + while (data.hasRemaining()) { + channel.write(data); + } } } } diff --git a/baremaps-core/src/main/java/org/apache/baremaps/storage/flatgeobuf/internal/TableConversions.java b/baremaps-core/src/main/java/org/apache/baremaps/storage/flatgeobuf/internal/TableConversions.java index 817dc6d0..5bc3fff9 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/storage/flatgeobuf/internal/TableConversions.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/storage/flatgeobuf/internal/TableConversions.java @@ -21,6 +21,7 @@ import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.stream.Collectors; import org.apache.baremaps.storage.*; import org.wololo.flatgeobuf.ColumnMeta; @@ -191,13 +192,14 @@ public class TableConversions { public static List<ColumnMeta> asColumns(List<Column> columns) { return columns.stream() .map(TableConversions::asColumn) + .filter(Objects::nonNull) .collect(Collectors.toList()); } public static ColumnMeta asColumn(Column column) { var type = types.get(column.type()); if (type == null) { - throw new IllegalArgumentException("Unsupported type " + type); + return null; } var columnMeta = new ColumnMeta(); columnMeta.name = column.name();
