This is an automated email from the ASF dual-hosted git repository. bchapuis pushed a commit to branch flatgeobuf in repository https://gitbox.apache.org/repos/asf/incubator-baremaps.git
commit 98d85e5c1df9c744eddff63338099aaaa59215ef Author: Bertil Chapuis <[email protected]> AuthorDate: Wed Jun 19 21:10:47 2024 +0200 Improve implementation --- .../org/apache/baremaps/flatgeobuf/FlatGeoBuf.java | 6 +- .../baremaps/flatgeobuf/FlatGeoBufMapper.java | 15 +- .../baremaps/flatgeobuf/FlatGeoBufReader.java | 11 +- .../baremaps/flatgeobuf/FlatGeoBufWriter.java | 317 ++++++++++++--------- .../apache/baremaps/flatgeobuf/FlatGeoBufTest.java | 61 ++++ 5 files changed, 251 insertions(+), 159 deletions(-) diff --git a/baremaps-flatgeobuf/src/main/java/org/apache/baremaps/flatgeobuf/FlatGeoBuf.java b/baremaps-flatgeobuf/src/main/java/org/apache/baremaps/flatgeobuf/FlatGeoBuf.java index 1712b321..a716ff1f 100644 --- a/baremaps-flatgeobuf/src/main/java/org/apache/baremaps/flatgeobuf/FlatGeoBuf.java +++ b/baremaps-flatgeobuf/src/main/java/org/apache/baremaps/flatgeobuf/FlatGeoBuf.java @@ -116,7 +116,7 @@ public class FlatGeoBuf { public record Header( String name, - double[] envelope, + List<Double> envelope, GeometryType geometryType, boolean hasZ, boolean hasM, @@ -134,8 +134,6 @@ public class FlatGeoBuf { } } - public record Feature( - Geometry geometry, - List<Object> properties) { + public record Feature(List<Object> properties, Geometry geometry) { } } diff --git a/baremaps-flatgeobuf/src/main/java/org/apache/baremaps/flatgeobuf/FlatGeoBufMapper.java b/baremaps-flatgeobuf/src/main/java/org/apache/baremaps/flatgeobuf/FlatGeoBufMapper.java index 81165398..f48d4d27 100644 --- a/baremaps-flatgeobuf/src/main/java/org/apache/baremaps/flatgeobuf/FlatGeoBufMapper.java +++ b/baremaps-flatgeobuf/src/main/java/org/apache/baremaps/flatgeobuf/FlatGeoBufMapper.java @@ -20,6 +20,7 @@ import com.google.flatbuffers.FlatBufferBuilder; import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; import java.util.ArrayList; +import java.util.List; import java.util.stream.IntStream; import org.apache.baremaps.flatgeobuf.generated.Column; import org.apache.baremaps.flatgeobuf.generated.Feature; @@ -56,7 +57,7 @@ public class FlatGeoBufMapper { } int envelopeOffset = 0; if (header.envelope() != null) { - envelopeOffset = Header.createEnvelopeVector(builder, header.envelope()); + envelopeOffset = Header.createEnvelopeVector(builder, header.envelope().stream().mapToDouble(d -> d).toArray()); } Header.startHeader(builder); Header.addGeometryType(builder, header.geometryType().getValue()); @@ -76,12 +77,11 @@ public class FlatGeoBufMapper { public static FlatGeoBuf.Header asHeaderRecord(Header header) { return new FlatGeoBuf.Header( header.name(), - new double[] { + List.of( header.envelope(0), header.envelope(1), header.envelope(2), - header.envelope(3) - }, + header.envelope(3)), FlatGeoBuf.GeometryType.values()[header.geometryType()], header.hasZ(), header.hasM(), @@ -128,12 +128,11 @@ public class FlatGeoBufMapper { } } return new FlatGeoBuf.Feature( - GeometryConversions.readGeometry(feature.geometry(), header.geometryType()), - values); + values, GeometryConversions.readGeometry(feature.geometry(), header.geometryType()) + ); } - private static Object readValue(ByteBuffer buffer, - Column column) { + private static Object readValue(ByteBuffer buffer, Column column) { return switch (FlatGeoBuf.ColumnType.values()[column.type()]) { case BYTE -> buffer.get(); case UBYTE -> buffer.get(); diff --git a/baremaps-flatgeobuf/src/main/java/org/apache/baremaps/flatgeobuf/FlatGeoBufReader.java b/baremaps-flatgeobuf/src/main/java/org/apache/baremaps/flatgeobuf/FlatGeoBufReader.java index 2b62cf3f..d7c390fd 100644 --- a/baremaps-flatgeobuf/src/main/java/org/apache/baremaps/flatgeobuf/FlatGeoBufReader.java +++ b/baremaps-flatgeobuf/src/main/java/org/apache/baremaps/flatgeobuf/FlatGeoBufReader.java @@ -49,10 +49,10 @@ public class FlatGeoBufReader { public static void skipIndex(ReadableByteChannel channel, Header header) throws IOException { - readIndexAsBuffer(channel, header); + readIndexBuffer(channel, header); } - public static ByteBuffer readIndexAsBuffer(ReadableByteChannel channel, Header header) + public static ByteBuffer readIndexBuffer(ReadableByteChannel channel, Header header) throws IOException { long indexSize = PackedRTree.calcSize(header.featuresCount(), header.indexNodeSize()); if (indexSize > 1L << 31) { @@ -60,11 +60,10 @@ public class FlatGeoBufReader { } ByteBuffer buffer = BufferUtil.createByteBuffer((int) indexSize, ByteOrder.LITTLE_ENDIAN); BufferUtil.readBytes(channel, buffer, (int) indexSize); - return buffer; } - public static InputStream readIndexAsStream(ReadableByteChannel channel, Header header) { + public static InputStream readIndexStream(ReadableByteChannel channel, Header header) { long indexSize = PackedRTree.calcSize(header.featuresCount(), header.indexNodeSize()); return new BoundedInputStream(Channels.newInputStream(channel), indexSize); } @@ -72,9 +71,9 @@ public class FlatGeoBufReader { public static Feature readFeature(ReadableByteChannel channel, ByteBuffer buffer) throws IOException { try { - ByteBuffer newBuffer = BufferUtil.readBytes(channel, buffer, 4); + ByteBuffer newBuffer = BufferUtil.readBytes(channel, buffer, 1<<16); int featureSize = newBuffer.getInt(); - newBuffer = BufferUtil.readBytes(channel, buffer, featureSize); + newBuffer = BufferUtil.readBytes(channel, newBuffer, featureSize); Feature feature = Feature.getRootAsFeature(newBuffer); buffer.position(buffer.position() + featureSize); return feature; diff --git a/baremaps-flatgeobuf/src/main/java/org/apache/baremaps/flatgeobuf/FlatGeoBufWriter.java b/baremaps-flatgeobuf/src/main/java/org/apache/baremaps/flatgeobuf/FlatGeoBufWriter.java index c1872857..a117d894 100644 --- a/baremaps-flatgeobuf/src/main/java/org/apache/baremaps/flatgeobuf/FlatGeoBufWriter.java +++ b/baremaps-flatgeobuf/src/main/java/org/apache/baremaps/flatgeobuf/FlatGeoBufWriter.java @@ -17,144 +17,179 @@ package org.apache.baremaps.flatgeobuf; -// -// import com.google.flatbuffers.FlatBufferBuilder; -// import java.io.IOException; -// import java.io.OutputStream; -// import java.nio.ByteBuffer; -// import java.nio.ByteOrder; -// import java.nio.channels.Channels; -// import java.nio.channels.WritableByteChannel; -// import java.nio.charset.StandardCharsets; -// -// import org.wololo.flatgeobuf.generated.*; -// -// public class FlatGeoBufWriter { -// public static void writeColumnValue(ByteBuffer buffer, ColumnMeta column, Object value) { -// switch (column.type()) { -// case ColumnType.Byte -> buffer.put((byte) value); -// case ColumnType.Bool -> buffer.put((byte) ((boolean) value ? 1 : 0)); -// case ColumnType.Short -> buffer.putShort((short) value); -// case ColumnType.Int -> buffer.putInt((int) value); -// case ColumnType.Long -> buffer.putLong((long) value); -// case ColumnType.Float -> buffer.putFloat((float) value); -// case ColumnType.Double -> buffer.putDouble((double) value); -// case ColumnType.String -> writeColumnString(buffer, value); -// case ColumnType.Json -> writeColumnJson(buffer, value); -// case ColumnType.DateTime -> writeColumnDateTime(buffer, value); -// case ColumnType.Binary -> writeColumnBinary(buffer, value); -// default -> { -// // Do nothing -// } -// }; -// } -// -// public static void writeColumnString(ByteBuffer propertiesBuffer, Object value) { -// var bytes = ((String) value).getBytes(StandardCharsets.UTF_8); -// propertiesBuffer.putInt(bytes.length); -// propertiesBuffer.put(bytes); -// } -// -// public static void writeColumnJson(ByteBuffer propertiesBuffer, Object value) { -// throw new UnsupportedOperationException(); -// } -// -// public static void writeColumnDateTime(ByteBuffer propertiesBuffer, Object value) { -// throw new UnsupportedOperationException(); -// } -// -// public static void writeColumnBinary(ByteBuffer propertiesBuffer, Object value) { -// throw new UnsupportedOperationException(); -// } -// -// public static void writeFeature( -// OutputStream outputStream, HeaderMeta headerMeta, -// FeatureMeta featureMeta) throws IOException { -// var featureBuilder = new FlatBufferBuilder(4096); -// -// // Write the properties -// var propertiesBuffer = ByteBuffer.allocate(1 << 20).order(ByteOrder.LITTLE_ENDIAN); -// var properties = featureMeta.properties(); -// for (int i = 0; i < properties.size(); i++) { -// var column = headerMeta.columns.get(i); -// var value = properties.get(i); -// propertiesBuffer.putShort((short) i); -// writeColumnValue(propertiesBuffer, column, value); -// } -// if (propertiesBuffer.position() > 0) { -// propertiesBuffer.flip(); -// } -// var propertiesOffset = Feature.createPropertiesVector(featureBuilder, propertiesBuffer); -// -// // Write the geometry -// var geometry = featureMeta.geometry(); -// var geometryOffset = 0; -// if (geometry != null) { -// geometryOffset = -// GeometryConversions.writeGeometry(featureBuilder, geometry, headerMeta.geometryType); -// } -// -// // Write the feature -// var featureOffset = Feature.createFeature(featureBuilder, geometryOffset, propertiesOffset, 0); -// featureBuilder.finishSizePrefixed(featureOffset); -// -// byte[] data = featureBuilder.sizedByteArray(); -// outputStream.write(data); -// } -// -// public static void write(HeaderMeta headerMeta, OutputStream to, FlatBufferBuilder builder) -// throws IOException { -// int[] columnsArray = headerMeta.columns.stream().mapToInt(c -> { -// int nameOffset = builder.createString(c.name()); -// int type = c.type(); -// return Column.createColumn( -// builder, -// nameOffset, -// type, -// 0, -// 0, -// c.width(), -// c.precision(), -// c.scale(), -// c.nullable(), -// c.unique(), -// c.primaryKey(), -// 0); -// }).toArray(); -// int columnsOffset = Header.createColumnsVector(builder, columnsArray); -// -// int nameOffset = 0; -// if (headerMeta.name != null) { -// nameOffset = builder.createString(headerMeta.name); -// } -// int crsOffset = 0; -// if (headerMeta.srid != 0) { -// Crs.startCrs(builder); -// Crs.addCode(builder, headerMeta.srid); -// crsOffset = Crs.endCrs(builder); -// } -// int envelopeOffset = 0; -// if (headerMeta.envelope != null) { -// envelopeOffset = Header.createEnvelopeVector(builder, -// new double[] {headerMeta.envelope.getMinX(), headerMeta.envelope.getMinY(), -// headerMeta.envelope.getMaxX(), headerMeta.envelope.getMaxY()}); -// } -// Header.startHeader(builder); -// Header.addGeometryType(builder, headerMeta.geometryType); -// Header.addIndexNodeSize(builder, headerMeta.indexNodeSize); -// Header.addColumns(builder, columnsOffset); -// Header.addEnvelope(builder, envelopeOffset); -// Header.addName(builder, nameOffset); -// Header.addCrs(builder, crsOffset); -// Header.addFeaturesCount(builder, headerMeta.featuresCount); -// int offset = Header.endHeader(builder); -// -// builder.finishSizePrefixed(offset); -// -// WritableByteChannel channel = Channels.newChannel(to); -// ByteBuffer dataBuffer = builder.dataBuffer(); -// while (dataBuffer.hasRemaining()) { -// channel.write(dataBuffer); -// } -// } -// } + +import com.google.flatbuffers.FlatBufferBuilder; +import org.apache.baremaps.flatgeobuf.generated.Column; +import org.apache.baremaps.flatgeobuf.generated.Crs; +import org.apache.baremaps.flatgeobuf.generated.Feature; +import org.apache.baremaps.flatgeobuf.generated.Header; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.channels.Channels; +import java.nio.channels.WritableByteChannel; +import java.nio.charset.StandardCharsets; + +public class FlatGeoBufWriter { + + public static void writeHeader(WritableByteChannel channel, Header header) throws IOException { + ByteBuffer headerBuffer = header.getByteBuffer(); + ByteBuffer startBuffer = ByteBuffer.allocate(12).order(ByteOrder.LITTLE_ENDIAN); + startBuffer.put(FlatGeoBuf.MAGIC_BYTES); + startBuffer.putInt(headerBuffer.remaining()); + startBuffer.flip(); + while (startBuffer.hasRemaining()) { + channel.write(startBuffer); + } + while (headerBuffer.hasRemaining()) { + channel.write(headerBuffer); + } + } + + public static void writeIndexStream(WritableByteChannel channel, InputStream inputStream) throws IOException { + try (OutputStream outputStream = Channels.newOutputStream(channel)) { + outputStream.write(inputStream.readAllBytes()); + } + } + + public static void writeIndexBuffer(WritableByteChannel channel, ByteBuffer buffer) throws IOException { + while (buffer.hasRemaining()) { + channel.write(buffer); + } + } + + public static void writeFeature(WritableByteChannel channel, Feature feature) throws IOException { + ByteBuffer featureBuffer = feature.getByteBuffer().duplicate(); + featureBuffer.flip(); + channel.write(featureBuffer); + while (featureBuffer.hasRemaining()) { + channel.write(featureBuffer); + } + } + + public static void writeColumnValue(ByteBuffer buffer, FlatGeoBuf.Column column, Object value) { + switch (column.type()) { + case BYTE -> buffer.put((byte) value); + case BOOL -> buffer.put((byte) ((boolean) value ? 1 : 0)); + case SHORT -> buffer.putShort((short) value); + case INT -> buffer.putInt((int) value); + case LONG -> buffer.putLong((long) value); + case FLOAT -> buffer.putFloat((float) value); + case DOUBLE -> buffer.putDouble((double) value); + case STRING -> writeColumnString(buffer, value); + case JSON -> writeColumnJson(buffer, value); + case DATETIME -> writeColumnDateTime(buffer, value); + case BINARY -> writeColumnBinary(buffer, value); + } + } + + public static void writeColumnString(ByteBuffer propertiesBuffer, Object value) { + var bytes = ((String) value).getBytes(StandardCharsets.UTF_8); + propertiesBuffer.putInt(bytes.length); + propertiesBuffer.put(bytes); + } + + public static void writeColumnJson(ByteBuffer propertiesBuffer, Object value) { + throw new UnsupportedOperationException(); + } + + public static void writeColumnDateTime(ByteBuffer propertiesBuffer, Object value) { + throw new UnsupportedOperationException(); + } + + public static void writeColumnBinary(ByteBuffer propertiesBuffer, Object value) { + throw new UnsupportedOperationException(); + } + + public static void writeFeature( + OutputStream outputStream, FlatGeoBuf.Header headerMeta, + FlatGeoBuf.Feature featureMeta) throws IOException { + var featureBuilder = new FlatBufferBuilder(4096); + + // Write the properties + var propertiesBuffer = ByteBuffer.allocate(1 << 20).order(ByteOrder.LITTLE_ENDIAN); + var properties = featureMeta.properties(); + for (int i = 0; i < properties.size(); i++) { + var column = headerMeta.columns().get(i); + var value = properties.get(i); + propertiesBuffer.putShort((short) i); + writeColumnValue(propertiesBuffer, column, value); + } + if (propertiesBuffer.position() > 0) { + propertiesBuffer.flip(); + } + var propertiesOffset = Feature.createPropertiesVector(featureBuilder, propertiesBuffer); + + // Write the geometry + var geometry = featureMeta.geometry(); + var geometryOffset = 0; + if (geometry != null) { + geometryOffset = + GeometryConversions.writeGeometry(featureBuilder, geometry, (byte) headerMeta.geometryType().getValue()); + } + + // Write the feature + var featureOffset = Feature.createFeature(featureBuilder, geometryOffset, propertiesOffset, 0); + featureBuilder.finishSizePrefixed(featureOffset); + + byte[] data = featureBuilder.sizedByteArray(); + outputStream.write(data); + } + + public static void write(FlatGeoBuf.Header headerMeta, OutputStream to, FlatBufferBuilder builder) + throws IOException { + int[] columnsArray = headerMeta.columns().stream().mapToInt(c -> { + int nameOffset = builder.createString(c.name()); + int type = c.type().ordinal(); + return Column.createColumn( + builder, + nameOffset, + type, + 0, + 0, + c.width(), + c.precision(), + c.scale(), + c.nullable(), + c.unique(), + c.primaryKey(), + 0); + }).toArray(); + int columnsOffset = Header.createColumnsVector(builder, columnsArray); + + int nameOffset = 0; + if (headerMeta.name() != null) { + nameOffset = builder.createString(headerMeta.name()); + } + int crsOffset = 0; + if (headerMeta.crs().code() != 0) { + Crs.startCrs(builder); + Crs.addCode(builder, headerMeta.crs().code()); + crsOffset = Crs.endCrs(builder); + } + int envelopeOffset = 0; + if (headerMeta.envelope() != null) { + envelopeOffset = Header.createEnvelopeVector(builder,headerMeta.envelope().stream().mapToDouble(d -> d).toArray()); + } + Header.startHeader(builder); + Header.addGeometryType(builder, headerMeta.geometryType().getValue()); + Header.addIndexNodeSize(builder, headerMeta.indexNodeSize()); + Header.addColumns(builder, columnsOffset); + Header.addEnvelope(builder, envelopeOffset); + Header.addName(builder, nameOffset); + Header.addCrs(builder, crsOffset); + Header.addFeaturesCount(builder, headerMeta.featuresCount()); + int offset = Header.endHeader(builder); + + builder.finishSizePrefixed(offset); + + WritableByteChannel channel = Channels.newChannel(to); + ByteBuffer dataBuffer = builder.dataBuffer(); + while (dataBuffer.hasRemaining()) { + channel.write(dataBuffer); + } + } +} diff --git a/baremaps-flatgeobuf/src/test/java/org/apache/baremaps/flatgeobuf/FlatGeoBufTest.java b/baremaps-flatgeobuf/src/test/java/org/apache/baremaps/flatgeobuf/FlatGeoBufTest.java index cbcb949d..7a2b6a67 100644 --- a/baremaps-flatgeobuf/src/test/java/org/apache/baremaps/flatgeobuf/FlatGeoBufTest.java +++ b/baremaps-flatgeobuf/src/test/java/org/apache/baremaps/flatgeobuf/FlatGeoBufTest.java @@ -23,6 +23,9 @@ import java.io.IOException; import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.nio.channels.FileChannel; +import java.nio.channels.ReadableByteChannel; +import java.nio.channels.WritableByteChannel; +import java.nio.file.Files; import java.nio.file.StandardOpenOption; import java.util.ArrayList; import java.util.List; @@ -91,4 +94,62 @@ public class FlatGeoBufTest { } } + + @Test + void readWrite() throws IOException { + var file = TestFiles.resolve("baremaps-testing/data/samples/countries.fgb"); + var tempFile = Files.createTempFile("countries", ".fgb"); + + FlatGeoBuf.Header headerRecord = null; + List<FlatGeoBuf.Feature> featureRecords = new ArrayList<>(); + + try (ReadableByteChannel channel = FileChannel.open(file, StandardOpenOption.READ); + WritableByteChannel tempChannel = FileChannel.open(tempFile, StandardOpenOption.WRITE)) { + + // Read the header + Header header = FlatGeoBufReader.readHeader(channel); + headerRecord = FlatGeoBufMapper.asHeaderRecord(header); + FlatGeoBufWriter.writeHeader(tempChannel, header); + + // Read the index + ByteBuffer indexBuffer = FlatGeoBufReader.readIndexBuffer(channel, header); + FlatGeoBufWriter.writeIndexBuffer(tempChannel, indexBuffer); + + var buffer = BufferUtil.createByteBuffer(1 << 16, ByteOrder.LITTLE_ENDIAN); + for (int i = 0; i < header.featuresCount(); i++) { + Feature feature = FlatGeoBufReader.readFeature(channel, buffer); + FlatGeoBuf.Feature featureRecord = FlatGeoBufMapper.asFeatureRecord(header, feature); + featureRecords.add(featureRecord); + FlatGeoBufWriter.writeFeature(tempChannel, feature); + } + } + + + try (var channel = FileChannel.open(tempFile, StandardOpenOption.READ)) { + + // Read the header + Header header = FlatGeoBufReader.readHeader(channel); + assertNotNull(header); + assertEquals(headerRecord, FlatGeoBufMapper.asHeaderRecord(header)); + + // Read the index + FlatGeoBufReader.skipIndex(channel, header); + + // Read the first feature + ByteBuffer buffer = BufferUtil.createByteBuffer(1 << 16, ByteOrder.LITTLE_ENDIAN); + + for (int i = 0; i < header.featuresCount(); i++) { + Feature feature = FlatGeoBufReader.readFeature(channel, buffer); + FlatGeoBuf.Feature featureRecord = FlatGeoBufMapper.asFeatureRecord(header, feature); + + System.out.println(featureRecord); + + assertNotNull(feature); + assertNotNull(featureRecord); + assertEquals(featureRecords.get(i), featureRecord); + } + + } + } + }
