This is an automated email from the ASF dual-hosted git repository. bchapuis pushed a commit to branch 590-flatgeobuf in repository https://gitbox.apache.org/repos/asf/incubator-baremaps.git
commit ac7bbbfdec5956c53111fc22a70981d2915984f9 Author: Bertil Chapuis <[email protected]> AuthorDate: Fri Feb 24 13:23:27 2023 +0100 Add support for FlatGeoBuf --- .../org/apache/baremaps/feature/FeatureImpl.java | 4 +- .../org/apache/baremaps/feature/FeatureType.java | 32 +--- .../{FeatureType.java => FeatureTypeImpl.java} | 20 +- .../baremaps/feature/ReflectiveFeatureAdapter.java | 90 --------- .../storage/FeatureSetProjectionTransform.java | 2 +- .../storage/flatgeobuf/FeatureConversions.java | 208 +++++++++++++++++++++ .../storage/flatgeobuf/FeatureIterator.java | 72 +++++++ .../storage/flatgeobuf/FlatGeoBufFeatureSet.java | 158 ++++++++++++++++ .../storage/geopackage/GeoPackageTable.java | 7 +- .../storage/postgres/PostgresDatabase.java | 12 +- .../shapefile/internal/ShapefileByteReader.java | 3 +- .../org/apache/baremaps/workflow/tasks/Entity.java | 11 +- .../workflow/tasks/TransformEntityCollection.java | 2 +- .../flatgeobuf/FlatGeoBufFeatureSetTest.java | 65 +++++++ baremaps-core/src/test/resources/countries.fgb | Bin 0 -> 205680 bytes 15 files changed, 541 insertions(+), 145 deletions(-) diff --git a/baremaps-core/src/main/java/org/apache/baremaps/feature/FeatureImpl.java b/baremaps-core/src/main/java/org/apache/baremaps/feature/FeatureImpl.java index 50f2605d..8e0fc0c2 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/feature/FeatureImpl.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/feature/FeatureImpl.java @@ -20,9 +20,9 @@ import java.util.Map; public final class FeatureImpl implements Feature { - private final FeatureType type; + private FeatureType type; - private final Map<String, Object> properties; + private Map<String, Object> properties; public FeatureImpl(FeatureType type) { this(type, new HashMap<>()); diff --git a/baremaps-core/src/main/java/org/apache/baremaps/feature/FeatureType.java b/baremaps-core/src/main/java/org/apache/baremaps/feature/FeatureType.java index 3829b16a..52c07fe0 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/feature/FeatureType.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/feature/FeatureType.java @@ -12,36 +12,14 @@ package org.apache.baremaps.feature; - - -import com.google.common.collect.ImmutableMap; import java.util.Map; -public class FeatureType { - - private final String name; - - private final Map<String, PropertyType> properties; - - public FeatureType(String name, Map<String, PropertyType> properties) { - this.name = name; - this.properties = properties; - } - - public String getName() { - return name; - } - - public Map<String, PropertyType> getProperties() { - return ImmutableMap.copyOf(properties); - } +public interface FeatureType { + String getName(); - public PropertyType getProperty(String name) { - return properties.get(name); - } + Map<String, PropertyType> getPropertyTypes(); - public Feature newInstance() { - return new FeatureImpl(this); - } + PropertyType getPropertyType(String name); + Feature newInstance(); } diff --git a/baremaps-core/src/main/java/org/apache/baremaps/feature/FeatureType.java b/baremaps-core/src/main/java/org/apache/baremaps/feature/FeatureTypeImpl.java similarity index 64% copy from baremaps-core/src/main/java/org/apache/baremaps/feature/FeatureType.java copy to baremaps-core/src/main/java/org/apache/baremaps/feature/FeatureTypeImpl.java index 3829b16a..1cda4086 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/feature/FeatureType.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/feature/FeatureTypeImpl.java @@ -17,29 +17,33 @@ package org.apache.baremaps.feature; import com.google.common.collect.ImmutableMap; import java.util.Map; -public class FeatureType { +public class FeatureTypeImpl implements FeatureType { private final String name; - private final Map<String, PropertyType> properties; + private final Map<String, PropertyType> propertyTypes; - public FeatureType(String name, Map<String, PropertyType> properties) { + public FeatureTypeImpl(String name, Map<String, PropertyType> propertyTypes) { this.name = name; - this.properties = properties; + this.propertyTypes = propertyTypes; } + @Override public String getName() { return name; } - public Map<String, PropertyType> getProperties() { - return ImmutableMap.copyOf(properties); + @Override + public Map<String, PropertyType> getPropertyTypes() { + return ImmutableMap.copyOf(propertyTypes); } - public PropertyType getProperty(String name) { - return properties.get(name); + @Override + public PropertyType getPropertyType(String name) { + return propertyTypes.get(name); } + @Override public Feature newInstance() { return new FeatureImpl(this); } diff --git a/baremaps-core/src/main/java/org/apache/baremaps/feature/ReflectiveFeatureAdapter.java b/baremaps-core/src/main/java/org/apache/baremaps/feature/ReflectiveFeatureAdapter.java deleted file mode 100644 index dc26c1e6..00000000 --- a/baremaps-core/src/main/java/org/apache/baremaps/feature/ReflectiveFeatureAdapter.java +++ /dev/null @@ -1,90 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except - * in compliance with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License - * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. See the License for the specific language governing permissions and limitations under - * the License. - */ - -package org.apache.baremaps.feature; - - - -import java.lang.reflect.Field; -import java.util.HashMap; -import java.util.Map; - -public class ReflectiveFeatureAdapter implements Feature { - - private final Object feature; - - public ReflectiveFeatureAdapter(Object feature) { - this.feature = feature; - } - - @Override - public FeatureType getType() { - var type = feature.getClass(); - var name = type.getSimpleName(); - var fields = type.getDeclaredFields(); - var properties = new HashMap<String, PropertyType>(); - for (Field field : fields) { - field.setAccessible(true); - var propertyName = field.getName(); - var propertyType = new PropertyType(propertyName, field.getType()); - properties.put(propertyName, propertyType); - } - return new FeatureType(name, properties); - } - - @Override - public Object getProperty(String name) { - try { - var field = this.getClass().getDeclaredField(name); - field.setAccessible(true); - return field.get(this); - } catch (IllegalAccessException e) { - throw new IllegalArgumentException(e); - } catch (NoSuchFieldException e) { - throw new IllegalArgumentException(e); - } - } - - @Override - public void setProperty(String name, Object value) throws IllegalArgumentException { - try { - var field = this.getClass().getDeclaredField(name); - field.setAccessible(true); - field.set(this, value); - } catch (IllegalAccessException e) { - throw new IllegalArgumentException(e); - } catch (NoSuchFieldException e) { - throw new IllegalArgumentException(e); - } - } - - - @Override - public Map<String, Object> getProperties() { - var type = feature.getClass(); - var fields = type.getDeclaredFields(); - var properties = new HashMap<String, Object>(); - for (Field field : fields) { - field.setAccessible(true); - var propertyName = field.getName(); - try { - var propertyValue = field.get(this); - properties.put(propertyName, propertyValue); - } catch (IllegalAccessException e) { - // Ignore the field - } - } - return properties; - } - - -} diff --git a/baremaps-core/src/main/java/org/apache/baremaps/storage/FeatureSetProjectionTransform.java b/baremaps-core/src/main/java/org/apache/baremaps/storage/FeatureSetProjectionTransform.java index b161732f..c02beb7c 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/storage/FeatureSetProjectionTransform.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/storage/FeatureSetProjectionTransform.java @@ -45,7 +45,7 @@ public class FeatureSetProjectionTransform implements ReadableFeatureSet { } public Feature transformProjection(Feature feature) { - for (var property : feature.getType().getProperties().values()) { + for (var property : feature.getType().getPropertyTypes().values()) { var name = property.getName(); var value = feature.getProperty(name); if (value instanceof Geometry geometry) { diff --git a/baremaps-core/src/main/java/org/apache/baremaps/storage/flatgeobuf/FeatureConversions.java b/baremaps-core/src/main/java/org/apache/baremaps/storage/flatgeobuf/FeatureConversions.java new file mode 100644 index 00000000..6e290804 --- /dev/null +++ b/baremaps-core/src/main/java/org/apache/baremaps/storage/flatgeobuf/FeatureConversions.java @@ -0,0 +1,208 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + */ + +package org.apache.baremaps.storage.flatgeobuf; + + +import com.google.flatbuffers.FlatBufferBuilder; +import org.apache.baremaps.feature.FeatureType; +import org.apache.baremaps.feature.FeatureTypeImpl; +import org.apache.baremaps.feature.PropertyType; +import org.wololo.flatgeobuf.ColumnMeta; +import org.wololo.flatgeobuf.GeometryConversions; +import org.wololo.flatgeobuf.HeaderMeta; +import org.wololo.flatgeobuf.generated.Column; +import org.wololo.flatgeobuf.generated.ColumnType; +import org.wololo.flatgeobuf.generated.Crs; +import org.wololo.flatgeobuf.generated.Header; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.channels.WritableByteChannel; +import java.nio.charset.StandardCharsets; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +public class FeatureConversions { + + public static FeatureType asFeatureType(HeaderMeta headerMeta) { + var name = headerMeta.name; + var properties = headerMeta.columns.stream().collect( + Collectors.toMap( + column -> column.name, + column -> new PropertyType(column.name, column.getBinding()))); + return new FeatureTypeImpl(name, properties); + } + + public static org.apache.baremaps.feature.Feature asFeature(HeaderMeta headerMeta, + FeatureType featureType, org.wololo.flatgeobuf.generated.Feature feature) { + var properties = new HashMap<String, Object>(); + + var geometryBuffer = feature.geometry(); + var geometry = GeometryConversions.deserialize(geometryBuffer, geometryBuffer.type()); + properties.put("geometry", geometry); + + if (feature.propertiesLength() > 0) { + var propertiesBuffer = feature.propertiesAsByteBuffer(); + while (propertiesBuffer.hasRemaining()) { + var type = propertiesBuffer.getShort(); + var column = headerMeta.columns.get(type); + var value = readValue(propertiesBuffer, column); + properties.put(column.name, value); + } + } + + return new org.apache.baremaps.feature.FeatureImpl(featureType, properties); + } + + public static void writeHeaderMeta(HeaderMeta headerMeta, WritableByteChannel channel, + FlatBufferBuilder builder) throws IOException { + int[] columnsArray = headerMeta.columns.stream().mapToInt(c -> { + int nameOffset = builder.createString(c.name); + int type = c.type; + return Column.createColumn(builder, nameOffset, type, 0, 0, c.width, c.precision, c.scale, c.nullable, c.unique, + c.primary_key, 0); + }).toArray(); + int columnsOffset = Header.createColumnsVector(builder, columnsArray); + + int nameOffset = 0; + if (headerMeta.name!=null) { + nameOffset = builder.createString(headerMeta.name); + } + int crsOffset = 0; + if (headerMeta.srid != 0) { + Crs.startCrs(builder); + Crs.addCode(builder, headerMeta.srid); + crsOffset = Crs.endCrs(builder); + } + int envelopeOffset = 0; + if (headerMeta.envelope != null) { + envelopeOffset = Header.createEnvelopeVector(builder, + new double[] { headerMeta.envelope.getMinX(), headerMeta.envelope.getMinY(), headerMeta.envelope.getMaxX(), headerMeta.envelope.getMaxY() }); + } + Header.startHeader(builder); + Header.addGeometryType(builder, headerMeta.geometryType); + Header.addIndexNodeSize(builder, headerMeta.indexNodeSize ); + Header.addColumns(builder, columnsOffset); + Header.addEnvelope(builder, envelopeOffset); + Header.addName(builder, nameOffset); + Header.addCrs(builder, crsOffset); + Header.addFeaturesCount(builder, headerMeta.featuresCount); + int offset = Header.endHeader(builder); + + builder.finishSizePrefixed(offset); + + ByteBuffer dataBuffer = builder.dataBuffer(); + while (dataBuffer.hasRemaining()) + channel.write(dataBuffer); + } + + public static Object readValue(ByteBuffer propertiesBuffer, ColumnMeta column) { + return switch (column.type) { + case ColumnType.Byte -> propertiesBuffer.get(); + case ColumnType.Bool -> propertiesBuffer.get() == 1; + case ColumnType.Short -> propertiesBuffer.getShort(); + case ColumnType.Int -> propertiesBuffer.getInt(); + case ColumnType.Long -> propertiesBuffer.getLong(); + case ColumnType.Float -> propertiesBuffer.getFloat(); + case ColumnType.Double -> propertiesBuffer.getDouble(); + case ColumnType.String -> readString(propertiesBuffer); + case ColumnType.Json -> readJson(propertiesBuffer); + case ColumnType.DateTime -> readDateTime(propertiesBuffer); + case ColumnType.Binary -> readBinary(propertiesBuffer); + default -> null; + }; + } + + public static void writeValue(ByteBuffer propertiesBuffer, ColumnMeta column, Object value) { + switch (column.type) { + case ColumnType.Byte -> propertiesBuffer.put((byte) value); + case ColumnType.Bool -> propertiesBuffer.put((byte) ((boolean) value ? 1 : 0)); + case ColumnType.Short -> propertiesBuffer.putShort((short) value); + case ColumnType.Int -> propertiesBuffer.putInt((int) value); + case ColumnType.Long -> propertiesBuffer.putLong((long) value); + case ColumnType.Float -> propertiesBuffer.putFloat((float) value); + case ColumnType.Double -> propertiesBuffer.putDouble((double) value); + case ColumnType.String -> writeString(propertiesBuffer, value); + case ColumnType.Json -> writeJson(propertiesBuffer, value); + case ColumnType.DateTime -> writeDateTime(propertiesBuffer, value); + case ColumnType.Binary -> writeBinary(propertiesBuffer, value); + default -> {} + }; + } + + public static void writeString(ByteBuffer propertiesBuffer, Object value) { + var bytes = ((String) value).getBytes(StandardCharsets.UTF_8); + propertiesBuffer.putInt(bytes.length); + propertiesBuffer.put(bytes); + } + + public static void writeJson(ByteBuffer propertiesBuffer, Object value) { + throw new UnsupportedOperationException(); + } + + public static void writeDateTime(ByteBuffer propertiesBuffer, Object value) { + throw new UnsupportedOperationException(); + } + + public static void writeBinary(ByteBuffer propertiesBuffer, Object value) { + throw new UnsupportedOperationException(); + } + + public static Object readString(ByteBuffer buffer) { + var length = buffer.getInt(); + var bytes = new byte[length]; + buffer.get(bytes); + return new String(bytes, StandardCharsets.UTF_8); + } + + public static Object readJson(ByteBuffer buffer) { + throw new UnsupportedOperationException(); + } + + public static Object readDateTime(ByteBuffer buffer) { + throw new UnsupportedOperationException(); + } + + public static Object readBinary(ByteBuffer buffer) { + throw new UnsupportedOperationException(); + } + + public static final Map<Class, Integer> types = Map.of( + Byte.class, ColumnType.Byte, + Boolean.class, ColumnType.Bool, + Short.class, ColumnType.Short, + Integer.class, ColumnType.Int, + Long.class, ColumnType.Long, + Float.class, ColumnType.Float, + Double.class, ColumnType.Double, + String.class, ColumnType.String); + + public static List<ColumnMeta> asColumns(Map<String, PropertyType> propertyTypes) { + return propertyTypes.values().stream() + .map(FeatureConversions::asColumn) + .collect(Collectors.toList()); + } + + public static ColumnMeta asColumn(PropertyType propertyType) { + var type = types.get(propertyType.getType()); + if (type == null) { + throw new IllegalArgumentException("Unsupported type " + propertyType); + } + var columnMeta = new ColumnMeta(); + columnMeta.name = propertyType.getName(); + columnMeta.type = type.byteValue(); + return columnMeta; + } +} diff --git a/baremaps-core/src/main/java/org/apache/baremaps/storage/flatgeobuf/FeatureIterator.java b/baremaps-core/src/main/java/org/apache/baremaps/storage/flatgeobuf/FeatureIterator.java new file mode 100644 index 00000000..a5b5affd --- /dev/null +++ b/baremaps-core/src/main/java/org/apache/baremaps/storage/flatgeobuf/FeatureIterator.java @@ -0,0 +1,72 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + */ + +package org.apache.baremaps.storage.flatgeobuf; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.channels.SeekableByteChannel; +import java.util.Iterator; +import java.util.NoSuchElementException; +import java.util.Spliterator; +import java.util.function.Consumer; +import org.apache.baremaps.feature.FeatureType; +import org.wololo.flatgeobuf.HeaderMeta; +import org.wololo.flatgeobuf.generated.Feature; + +public class FeatureIterator implements Iterator<org.apache.baremaps.feature.Feature> { + + private final HeaderMeta headerMeta; + + private final FeatureType featureType; + + private final SeekableByteChannel channel; + + private final ByteBuffer buffer; + + private long cursor = 0; + + public FeatureIterator(SeekableByteChannel channel, HeaderMeta headerMeta, + FeatureType featureType) { + this.channel = channel; + this.headerMeta = headerMeta; + this.featureType = featureType; + this.buffer = ByteBuffer.allocate(1 << 20).order(ByteOrder.LITTLE_ENDIAN); + } + + @Override + public boolean hasNext() { + return cursor < headerMeta.featuresCount; + } + + @Override + public org.apache.baremaps.feature.Feature next() { + try { + channel.read(buffer); + buffer.flip(); + + var featureSize = buffer.getInt(); + var feature = + FeatureConversions.asFeature(headerMeta, featureType, Feature.getRootAsFeature(buffer)); + + buffer.position(Integer.BYTES + featureSize); + buffer.compact(); + + cursor++; + + return feature; + } catch (IOException e) { + throw new NoSuchElementException(e); + } + } +} diff --git a/baremaps-core/src/main/java/org/apache/baremaps/storage/flatgeobuf/FlatGeoBufFeatureSet.java b/baremaps-core/src/main/java/org/apache/baremaps/storage/flatgeobuf/FlatGeoBufFeatureSet.java new file mode 100644 index 00000000..b9b5edc9 --- /dev/null +++ b/baremaps-core/src/main/java/org/apache/baremaps/storage/flatgeobuf/FlatGeoBufFeatureSet.java @@ -0,0 +1,158 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + */ + +package org.apache.baremaps.storage.flatgeobuf; + +import com.google.flatbuffers.FlatBufferBuilder; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.channels.Channels; +import java.nio.channels.SeekableByteChannel; +import java.util.Collection; +import java.util.Iterator; +import java.util.Map.Entry; + +import org.apache.baremaps.collection.DataCollection; +import org.apache.baremaps.feature.Feature; +import org.apache.baremaps.feature.FeatureType; +import org.locationtech.jts.geom.Geometry; +import org.wololo.flatgeobuf.Constants; +import org.wololo.flatgeobuf.GeometryConversions; +import org.wololo.flatgeobuf.HeaderMeta; +import org.wololo.flatgeobuf.PackedRTree; +import org.wololo.flatgeobuf.generated.GeometryType; + +public class FlatGeoBufFeatureSet { + + private final SeekableByteChannel channel; + + private HeaderMeta headerMeta; + + private FeatureType featureType; + + public FlatGeoBufFeatureSet(SeekableByteChannel channel) { + this.channel = channel; + } + + public FlatGeoBufFeatureSet(SeekableByteChannel channel, FeatureType featureType) { + this.channel = channel; + this.featureType = featureType; + } + + public FeatureType getType() throws IOException { + readHeaderMeta(); + return featureType; + } + + public Collection<Feature> read() throws IOException { + readHeaderMeta(); + + return new DataCollection<>() { + + @Override + public Iterator<Feature> iterator() { + try { + channel.position(headerMeta.offset); + + // skip the index + var indexSize = + (int) PackedRTree.calcSize((int) headerMeta.featuresCount, headerMeta.indexNodeSize); + channel.position(headerMeta.offset + indexSize); + + // create the feature stream + return new FeatureIterator(channel, headerMeta, featureType); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + @Override + public long sizeAsLong() { + return headerMeta.featuresCount; + } + + }; + } + + public void readHeaderMeta() throws IOException { + channel.position(0); + var headerMetaBuffer = ByteBuffer.allocate(1 << 20).order(ByteOrder.LITTLE_ENDIAN); + channel.read(headerMetaBuffer); + headerMetaBuffer.flip(); + headerMeta = HeaderMeta.read(headerMetaBuffer); + featureType = FeatureConversions.asFeatureType(headerMeta); + } + + public void write(Collection<Feature> features) throws IOException { + channel.position(0); + + var outputStream = Channels.newOutputStream(channel); + + outputStream.write(Constants.MAGIC_BYTES); + + var bufferBuilder = new FlatBufferBuilder(); + + var headerMeta = new HeaderMeta(); + headerMeta.geometryType = GeometryType.Unknown; + headerMeta.indexNodeSize = 16; + headerMeta.featuresCount = features instanceof DataCollection<Feature> c ? c.sizeAsLong() : features.size(); + headerMeta.name = featureType.getName(); + headerMeta.columns = FeatureConversions.asColumns(featureType.getPropertyTypes()); + + HeaderMeta.write(headerMeta, outputStream, bufferBuilder); + + var indexSize = + (int) PackedRTree.calcSize((int) headerMeta.featuresCount, headerMeta.indexNodeSize); + + for (int i = 0; i < indexSize; i++) { + outputStream.write(0); + } + + var iterator = features.iterator(); + while (iterator.hasNext()) { + var feature = iterator.next(); + + var featureBuilder = new FlatBufferBuilder(); + var geometryOffset = 0; + var propertiesOffset = 0; + var propertiesBuffer = ByteBuffer.allocate(1 << 20).order(ByteOrder.LITTLE_ENDIAN); + var i = 0; + for (Entry<String, Object> entry : feature.getProperties().entrySet()) { + if (entry.getValue() instanceof Geometry) { + try { + var geometry = (Geometry) entry.getValue(); + geometryOffset = GeometryConversions.serialize(featureBuilder, geometry, headerMeta.geometryType); + } catch (IOException e) { + throw new RuntimeException(e); + } + } else { + var column = headerMeta.columns.get(i); + var value = feature.getProperty(column.name); + propertiesBuffer.putShort((short) i); + FeatureConversions.writeValue(propertiesBuffer, column, value); + i++; + } + } + propertiesBuffer.flip(); + propertiesOffset = org.wololo.flatgeobuf.generated.Feature.createPropertiesVector(featureBuilder, propertiesBuffer); + + var featureOffset = + org.wololo.flatgeobuf.generated.Feature.createFeature(featureBuilder, geometryOffset, propertiesOffset, 0); + + featureBuilder.finishSizePrefixed(featureOffset); + + channel.write(featureBuilder.dataBuffer()); + } + } +} diff --git a/baremaps-core/src/main/java/org/apache/baremaps/storage/geopackage/GeoPackageTable.java b/baremaps-core/src/main/java/org/apache/baremaps/storage/geopackage/GeoPackageTable.java index cb15df41..9fb0e186 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/storage/geopackage/GeoPackageTable.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/storage/geopackage/GeoPackageTable.java @@ -22,10 +22,7 @@ import mil.nga.geopackage.features.user.FeatureColumn; import mil.nga.geopackage.features.user.FeatureDao; import mil.nga.geopackage.features.user.FeatureResultSet; import mil.nga.geopackage.geom.GeoPackageGeometryData; -import org.apache.baremaps.feature.Feature; -import org.apache.baremaps.feature.FeatureType; -import org.apache.baremaps.feature.PropertyType; -import org.apache.baremaps.feature.ReadableFeatureSet; +import org.apache.baremaps.feature.*; import org.locationtech.jts.geom.Coordinate; import org.locationtech.jts.geom.Geometry; import org.locationtech.jts.geom.GeometryCollection; @@ -56,7 +53,7 @@ public class GeoPackageTable implements ReadableFeatureSet { var propertyType = classType(column); properties.put(propertyName, new PropertyType(propertyName, propertyType)); } - featureType = new FeatureType(name, properties); + featureType = new FeatureTypeImpl(name, properties); geometryFactory = new GeometryFactory(new PrecisionModel(), (int) featureDao.getSrs().getId()); } diff --git a/baremaps-core/src/main/java/org/apache/baremaps/storage/postgres/PostgresDatabase.java b/baremaps-core/src/main/java/org/apache/baremaps/storage/postgres/PostgresDatabase.java index 0d2436cb..f7214c27 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/storage/postgres/PostgresDatabase.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/storage/postgres/PostgresDatabase.java @@ -34,6 +34,7 @@ import org.postgresql.copy.PGCopyOutputStream; public class PostgresDatabase implements WritableAggregate { + // TODO: Instantiate with Map.of() private static Map<Class, String> typeToName = new HashMap<>(); static { @@ -58,6 +59,7 @@ public class PostgresDatabase implements WritableAggregate { typeToName.put(LocalDateTime.class, "timestamp"); } + // TODO: Instantiate with Map.of() private static Map<Class, BaseValueHandler> typeToHandler = new HashMap<>(); static { @@ -90,10 +92,10 @@ public class PostgresDatabase implements WritableAggregate { private FeatureType createFeatureType(FeatureType featureType) { var name = featureType.getName().replaceAll("[^a-zA-Z0-9]", "_"); - var properties = featureType.getProperties().values().stream() + var properties = featureType.getPropertyTypes().values().stream() .filter(type -> typeToName.containsKey(type.getType())) .collect(Collectors.toMap(k -> k.getName(), v -> v)); - return new FeatureType(name, properties); + return new FeatureTypeImpl(name, properties); } @Override @@ -142,7 +144,7 @@ public class PostgresDatabase implements WritableAggregate { } private List<PropertyType> getAttributes(FeatureType featureType) { - return featureType.getProperties().values().stream() + return featureType.getPropertyTypes().values().stream() .filter(this::isSupported).collect(Collectors.toList()); } @@ -155,7 +157,7 @@ public class PostgresDatabase implements WritableAggregate { builder.append("CREATE TABLE "); builder.append(featureType.getName()); builder.append(" ("); - builder.append(featureType.getProperties().values().stream() + builder.append(featureType.getPropertyTypes().values().stream() .map(attributeType -> attributeType.getName() + " " + typeToName.get(attributeType.getType())) .collect(Collectors.joining(", "))); @@ -168,7 +170,7 @@ public class PostgresDatabase implements WritableAggregate { builder.append("COPY "); builder.append(featureType.getName()); builder.append(" ("); - builder.append(featureType.getProperties().values().stream() + builder.append(featureType.getPropertyTypes().values().stream() .map(propertyType -> propertyType.getName()) .collect(Collectors.joining(", "))); builder.append(") FROM STDIN BINARY"); diff --git a/baremaps-core/src/main/java/org/apache/baremaps/storage/shapefile/internal/ShapefileByteReader.java b/baremaps-core/src/main/java/org/apache/baremaps/storage/shapefile/internal/ShapefileByteReader.java index d56824af..8dbb3fbd 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/storage/shapefile/internal/ShapefileByteReader.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/storage/shapefile/internal/ShapefileByteReader.java @@ -21,6 +21,7 @@ import java.nio.channels.FileChannel; import java.util.*; import org.apache.baremaps.feature.Feature; import org.apache.baremaps.feature.FeatureType; +import org.apache.baremaps.feature.FeatureTypeImpl; import org.apache.baremaps.feature.PropertyType; import org.locationtech.jts.algorithm.Orientation; import org.locationtech.jts.geom.Coordinate; @@ -152,7 +153,7 @@ public class ShapefileByteReader extends CommonByteReader { // Add geometry field. properties.put(GEOMETRY_NAME, new PropertyType(GEOMETRY_NAME, Geometry.class)); - return new FeatureType(name, properties); + return new FeatureTypeImpl(name, properties); } /** Load shapefile descriptor. */ diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/Entity.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/Entity.java index 0eb3ce82..17a8d336 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/Entity.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/Entity.java @@ -16,16 +16,17 @@ import java.util.HashMap; import java.util.Map; import org.apache.baremaps.feature.Feature; import org.apache.baremaps.feature.FeatureType; +import org.apache.baremaps.feature.FeatureTypeImpl; import org.apache.baremaps.feature.PropertyType; import org.locationtech.jts.geom.Geometry; public class Entity implements Feature { - private final long id; + private long id; - private final Map<String, String> tags; + private Map<String, String> tags; - private final Geometry geometry; + private Geometry geometry; public Entity(long id, Map<String, String> tags, Geometry geometry) { this.id = id; @@ -42,12 +43,12 @@ public class Entity implements Feature { map.put(entry.getKey(), new PropertyType<>(entry.getKey(), String.class)); } map.put("geometry", new PropertyType<>("geometry", Geometry.class)); - return new FeatureType("entity", map); + return new FeatureTypeImpl("entity", map); } @Override public void setProperty(String name, Object value) { - throw new UnsupportedOperationException(); + tags.put(name, value.toString()); } @Override diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/TransformEntityCollection.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/TransformEntityCollection.java index 94aeaa29..b39a06f9 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/TransformEntityCollection.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/TransformEntityCollection.java @@ -53,7 +53,7 @@ public record TransformEntityCollection(Path collection, String database, public void execute(WorkflowContext context) throws Exception { logger.info("Transform {} with {}", collection, recipe); - var featureType = new FeatureType(recipe.name, propertyTypes()); + var featureType = new FeatureTypeImpl(recipe.name, propertyTypes()); var groups = new AppendOnlyBuffer<>(new EntityDataType(), new MemoryMappedFile(collection)) .stream() diff --git a/baremaps-core/src/test/java/org/apache/baremaps/storage/flatgeobuf/FlatGeoBufFeatureSetTest.java b/baremaps-core/src/test/java/org/apache/baremaps/storage/flatgeobuf/FlatGeoBufFeatureSetTest.java new file mode 100644 index 00000000..3c17ea83 --- /dev/null +++ b/baremaps-core/src/test/java/org/apache/baremaps/storage/flatgeobuf/FlatGeoBufFeatureSetTest.java @@ -0,0 +1,65 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + */ + +package org.apache.baremaps.storage.flatgeobuf; + +import static org.junit.jupiter.api.Assertions.*; + +import java.io.IOException; +import java.nio.channels.FileChannel; +import java.nio.file.Files; +import java.nio.file.StandardOpenOption; + +import org.apache.baremaps.testing.TestFiles; +import org.junit.jupiter.api.Test; + +class FlatGeoBufFeatureSetTest { + + @Test + void getType() throws IOException { + try (var channel = FileChannel.open(TestFiles.resolve("countries.fgb"))) { + var featureSet = new FlatGeoBufFeatureSet(channel); + var featureType = featureSet.getType(); + assertEquals(featureType.getName(), null); + assertEquals(featureType.getPropertyTypes().size(), 2); + } + } + + @Test + void read() throws IOException { + try (var channel = FileChannel.open(TestFiles.resolve("countries.fgb"))) { + var featureSet = new FlatGeoBufFeatureSet(channel); + assertEquals(179, featureSet.read().size()); + assertEquals(179, featureSet.read().stream().count()); + } + } + + @Test + void write() throws IOException { + var file = Files.createTempFile("countries", ".fgb"); + file.toFile().deleteOnExit(); + try (var channel1 = FileChannel.open(TestFiles.resolve("countries.fgb")); + var channel2 = FileChannel.open(file, StandardOpenOption.WRITE)) { + var featureSet1 = new FlatGeoBufFeatureSet(channel1); + var featureList = featureSet1.read().stream().toList(); + var featureSet2 = new FlatGeoBufFeatureSet(channel2, featureSet1.getType()); + featureSet2.write(featureList); + } + + try (var channel = FileChannel.open(file, StandardOpenOption.READ)) { + var featureSet = new FlatGeoBufFeatureSet(channel); + assertEquals(179, featureSet.read().size()); + } + } + + +} diff --git a/baremaps-core/src/test/resources/countries.fgb b/baremaps-core/src/test/resources/countries.fgb new file mode 100644 index 00000000..970fa634 Binary files /dev/null and b/baremaps-core/src/test/resources/countries.fgb differ
