This is an automated email from the ASF dual-hosted git repository. bchapuis pushed a commit to branch 590-flatgeobuf in repository https://gitbox.apache.org/repos/asf/incubator-baremaps.git
commit 045c48f3dc6739d48b68c3f18e13159eca1ea6aa Author: Bertil Chapuis <[email protected]> AuthorDate: Mon Feb 27 21:47:32 2023 +0100 Improve naming --- ...wConversions.java => DataFrameConversions.java} | 4 +- .../storage/flatgeobuf/FlatGeoBufDataFrame.java | 231 +++++++++++++-------- .../baremaps/storage/flatgeobuf/RowIterator.java | 71 ------- .../storage/shapefile/ShapefileDataFrame.java | 14 +- .../shapefile/internal/DbaseByteReader.java | 13 +- .../shapefile/internal/ShapefileByteReader.java | 58 +++--- .../shapefile/internal/ShapefileDescriptor.java | 10 +- .../{ShapeType.java => ShapefileGeometryType.java} | 11 +- ...eatureStream.java => ShapefileInputStream.java} | 9 +- .../shapefile/internal/ShapefileReader.java | 8 +- .../baremaps/workflow/tasks/ImportGeoPackage.java | 7 +- .../flatgeobuf/FlatGeoBufDataFrameTest.java | 38 ++-- 12 files changed, 225 insertions(+), 249 deletions(-) diff --git a/baremaps-core/src/main/java/org/apache/baremaps/storage/flatgeobuf/RowConversions.java b/baremaps-core/src/main/java/org/apache/baremaps/storage/flatgeobuf/DataFrameConversions.java similarity index 98% rename from baremaps-core/src/main/java/org/apache/baremaps/storage/flatgeobuf/RowConversions.java rename to baremaps-core/src/main/java/org/apache/baremaps/storage/flatgeobuf/DataFrameConversions.java index 84c940d7..1acfe9a8 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/storage/flatgeobuf/RowConversions.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/storage/flatgeobuf/DataFrameConversions.java @@ -31,7 +31,7 @@ import org.wololo.flatgeobuf.generated.Crs; import org.wololo.flatgeobuf.generated.Feature; import org.wololo.flatgeobuf.generated.Header; -public class RowConversions { +public class DataFrameConversions { public static Schema asFeatureType(HeaderMeta headerMeta) { var name = headerMeta.name; @@ -190,7 +190,7 @@ public class RowConversions { public static List<ColumnMeta> asColumns(List<Column> columns) { return columns.stream() - .map(RowConversions::asColumn) + .map(DataFrameConversions::asColumn) .collect(Collectors.toList()); } diff --git a/baremaps-core/src/main/java/org/apache/baremaps/storage/flatgeobuf/FlatGeoBufDataFrame.java b/baremaps-core/src/main/java/org/apache/baremaps/storage/flatgeobuf/FlatGeoBufDataFrame.java index f0d2788f..a31e0b26 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/storage/flatgeobuf/FlatGeoBufDataFrame.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/storage/flatgeobuf/FlatGeoBufDataFrame.java @@ -16,10 +16,12 @@ import com.google.flatbuffers.FlatBufferBuilder; import java.io.IOException; import java.nio.ByteBuffer; import java.nio.ByteOrder; -import java.nio.channels.Channels; -import java.nio.channels.SeekableByteChannel; +import java.nio.channels.*; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; import java.util.Collection; import java.util.Iterator; +import java.util.NoSuchElementException; import org.apache.baremaps.collection.AbstractDataCollection; import org.apache.baremaps.dataframe.Row; import org.apache.baremaps.dataframe.Schema; @@ -28,129 +30,182 @@ import org.wololo.flatgeobuf.Constants; import org.wololo.flatgeobuf.GeometryConversions; import org.wololo.flatgeobuf.HeaderMeta; import org.wololo.flatgeobuf.PackedRTree; +import org.wololo.flatgeobuf.generated.Feature; import org.wololo.flatgeobuf.generated.GeometryType; -public class FlatGeoBufDataFrame { +public class FlatGeoBufDataFrame extends AbstractDataCollection<Row> { - private final SeekableByteChannel channel; - - private HeaderMeta headerMeta; + private final Path file; private Schema schema; - public FlatGeoBufDataFrame(SeekableByteChannel channel) { - this.channel = channel; + public FlatGeoBufDataFrame(Path file) { + this.file = file; } - public FlatGeoBufDataFrame(SeekableByteChannel channel, Schema schema) { - this.channel = channel; + public FlatGeoBufDataFrame(Path file, Schema schema) { + this.file = file; this.schema = schema; } public Schema getSchema() throws IOException { - readHeaderMeta(); - return schema; + try (var channel = FileChannel.open(file, StandardOpenOption.READ)) { + var buffer = ByteBuffer.allocate(1 << 20).order(ByteOrder.LITTLE_ENDIAN); + HeaderMeta headerMeta = readHeaderMeta(channel, buffer); + return DataFrameConversions.asFeatureType(headerMeta); + } } - public Collection<Row> read() throws IOException { - readHeaderMeta(); + @Override + public Iterator<Row> iterator() { + try { + var channel = FileChannel.open(file, StandardOpenOption.READ); - return new AbstractDataCollection<>() { + var buffer = ByteBuffer.allocate(1 << 20).order(ByteOrder.LITTLE_ENDIAN); + HeaderMeta headerMeta = readHeaderMeta(channel, buffer); + channel.position(headerMeta.offset); - @Override - public Iterator<Row> iterator() { - try { - channel.position(headerMeta.offset); + // skip the index + var indexSize = + (int) PackedRTree.calcSize((int) headerMeta.featuresCount, headerMeta.indexNodeSize); + channel.position(headerMeta.offset + indexSize); - // skip the index - var indexSize = - (int) PackedRTree.calcSize((int) headerMeta.featuresCount, headerMeta.indexNodeSize); - channel.position(headerMeta.offset + indexSize); + buffer.clear(); - // create the feature stream - return new RowIterator(channel, headerMeta, schema); - } catch (IOException e) { - throw new RuntimeException(e); - } - } + // create the feature stream + return new RowIterator(channel, headerMeta, schema, buffer); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + + private HeaderMeta readHeaderMeta(SeekableByteChannel channel, ByteBuffer buffer) + throws IOException { + channel.read(buffer); + buffer.flip(); + return HeaderMeta.read(buffer); + } + + public void write(Collection<Row> features) throws IOException { + try ( + var channel = FileChannel.open(file, StandardOpenOption.CREATE, StandardOpenOption.WRITE)) { + + var outputStream = Channels.newOutputStream(channel); + outputStream.write(Constants.MAGIC_BYTES); - @Override - public long sizeAsLong() { - return headerMeta.featuresCount; + var bufferBuilder = new FlatBufferBuilder(); + + var headerMeta = new HeaderMeta(); + headerMeta.geometryType = GeometryType.Unknown; + headerMeta.indexNodeSize = 16; + headerMeta.featuresCount = + features instanceof AbstractDataCollection<Row>c ? c.sizeAsLong() : features.size(); + headerMeta.name = schema.name(); + headerMeta.columns = DataFrameConversions.asColumns(schema.columns()); + HeaderMeta.write(headerMeta, outputStream, bufferBuilder); + + var indexSize = + (int) PackedRTree.calcSize((int) headerMeta.featuresCount, headerMeta.indexNodeSize); + + for (int i = 0; i < indexSize; i++) { + outputStream.write(0); } - }; + var iterator = features.iterator(); + while (iterator.hasNext()) { + var row = iterator.next(); + var featureBuilder = new FlatBufferBuilder(); + var geometryOffset = 0; + var propertiesOffset = 0; + var propertiesBuffer = ByteBuffer.allocate(1 << 20).order(ByteOrder.LITTLE_ENDIAN); + var i = 0; + for (Object value : row.values()) { + if (value instanceof Geometry geometry) { + try { + geometryOffset = + GeometryConversions.serialize(featureBuilder, geometry, headerMeta.geometryType); + } catch (IOException e) { + throw new RuntimeException(e); + } + } else { + var column = headerMeta.columns.get(i); + propertiesBuffer.putShort((short) i); + DataFrameConversions.writeValue(propertiesBuffer, column, value); + i++; + } + } + propertiesBuffer.flip(); + propertiesOffset = org.wololo.flatgeobuf.generated.Feature + .createPropertiesVector(featureBuilder, propertiesBuffer); + + var featureOffset = + org.wololo.flatgeobuf.generated.Feature.createFeature(featureBuilder, geometryOffset, + propertiesOffset, 0); + + featureBuilder.finishSizePrefixed(featureOffset); + + channel.write(featureBuilder.dataBuffer()); + } + } } - public void readHeaderMeta() throws IOException { - channel.position(0); - var headerMetaBuffer = ByteBuffer.allocate(1 << 20).order(ByteOrder.LITTLE_ENDIAN); - channel.read(headerMetaBuffer); - headerMetaBuffer.flip(); - headerMeta = HeaderMeta.read(headerMetaBuffer); - schema = RowConversions.asFeatureType(headerMeta); + @Override + public long sizeAsLong() { + try (var channel = FileChannel.open(file, StandardOpenOption.READ)) { + var buffer = ByteBuffer.allocate(1 << 20).order(ByteOrder.LITTLE_ENDIAN); + HeaderMeta headerMeta = readHeaderMeta(channel, buffer); + return headerMeta.featuresCount; + } catch (IOException e) { + throw new RuntimeException(e); + } } - public void write(Collection<Row> features) throws IOException { - channel.position(0); + public static class RowIterator implements Iterator<Row> { - var outputStream = Channels.newOutputStream(channel); + private final HeaderMeta headerMeta; - outputStream.write(Constants.MAGIC_BYTES); + private final Schema dataType; - var bufferBuilder = new FlatBufferBuilder(); + private final SeekableByteChannel channel; - var headerMeta = new HeaderMeta(); - headerMeta.geometryType = GeometryType.Unknown; - headerMeta.indexNodeSize = 16; - headerMeta.featuresCount = - features instanceof AbstractDataCollection<Row>c ? c.sizeAsLong() : features.size(); - headerMeta.name = schema.name(); - headerMeta.columns = RowConversions.asColumns(schema.columns()); + private final ByteBuffer buffer; - HeaderMeta.write(headerMeta, outputStream, bufferBuilder); + private long cursor = 0; - var indexSize = - (int) PackedRTree.calcSize((int) headerMeta.featuresCount, headerMeta.indexNodeSize); + public RowIterator(SeekableByteChannel channel, HeaderMeta headerMeta, + Schema dataType, ByteBuffer buffer) { + this.channel = channel; + this.headerMeta = headerMeta; + this.dataType = dataType; + this.buffer = buffer; + } - for (int i = 0; i < indexSize; i++) { - outputStream.write(0); + @Override + public boolean hasNext() { + return cursor < headerMeta.featuresCount; } - var iterator = features.iterator(); - while (iterator.hasNext()) { - var row = iterator.next(); - var featureBuilder = new FlatBufferBuilder(); - var geometryOffset = 0; - var propertiesOffset = 0; - var propertiesBuffer = ByteBuffer.allocate(1 << 20).order(ByteOrder.LITTLE_ENDIAN); - var i = 0; - for (Object value : row.values()) { - if (value instanceof Geometry geometry) { - try { - geometryOffset = - GeometryConversions.serialize(featureBuilder, geometry, headerMeta.geometryType); - } catch (IOException e) { - throw new RuntimeException(e); - } - } else { - var column = headerMeta.columns.get(i); - propertiesBuffer.putShort((short) i); - RowConversions.writeValue(propertiesBuffer, column, value); - i++; - } - } - propertiesBuffer.flip(); - propertiesOffset = org.wololo.flatgeobuf.generated.Feature - .createPropertiesVector(featureBuilder, propertiesBuffer); + @Override + public Row next() { + try { + channel.read(buffer); + buffer.flip(); - var featureOffset = - org.wololo.flatgeobuf.generated.Feature.createFeature(featureBuilder, geometryOffset, - propertiesOffset, 0); + var featureSize = buffer.getInt(); + var row = + DataFrameConversions.asRow(headerMeta, dataType, Feature.getRootAsFeature(buffer)); - featureBuilder.finishSizePrefixed(featureOffset); + buffer.position(Integer.BYTES + featureSize); + buffer.compact(); - channel.write(featureBuilder.dataBuffer()); + cursor++; + + return row; + } catch (IOException e) { + throw new NoSuchElementException(e); + } } } + } diff --git a/baremaps-core/src/main/java/org/apache/baremaps/storage/flatgeobuf/RowIterator.java b/baremaps-core/src/main/java/org/apache/baremaps/storage/flatgeobuf/RowIterator.java deleted file mode 100644 index f1b611ad..00000000 --- a/baremaps-core/src/main/java/org/apache/baremaps/storage/flatgeobuf/RowIterator.java +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except - * in compliance with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License - * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. See the License for the specific language governing permissions and limitations under - * the License. - */ - -package org.apache.baremaps.storage.flatgeobuf; - -import java.io.IOException; -import java.nio.ByteBuffer; -import java.nio.ByteOrder; -import java.nio.channels.SeekableByteChannel; -import java.util.Iterator; -import java.util.NoSuchElementException; -import org.apache.baremaps.dataframe.Row; -import org.apache.baremaps.dataframe.Schema; -import org.wololo.flatgeobuf.HeaderMeta; -import org.wololo.flatgeobuf.generated.Feature; - -public class RowIterator implements Iterator<Row> { - - private final HeaderMeta headerMeta; - - private final Schema dataType; - - private final SeekableByteChannel channel; - - private final ByteBuffer buffer; - - private long cursor = 0; - - public RowIterator(SeekableByteChannel channel, HeaderMeta headerMeta, - Schema dataType) { - this.channel = channel; - this.headerMeta = headerMeta; - this.dataType = dataType; - this.buffer = ByteBuffer.allocate(1 << 20).order(ByteOrder.LITTLE_ENDIAN); - } - - @Override - public boolean hasNext() { - return cursor < headerMeta.featuresCount; - } - - @Override - public Row next() { - try { - channel.read(buffer); - buffer.flip(); - - var featureSize = buffer.getInt(); - var row = - RowConversions.asRow(headerMeta, dataType, Feature.getRootAsFeature(buffer)); - - buffer.position(Integer.BYTES + featureSize); - buffer.compact(); - - cursor++; - - return row; - } catch (IOException e) { - throw new NoSuchElementException(e); - } - } -} diff --git a/baremaps-core/src/main/java/org/apache/baremaps/storage/shapefile/ShapefileDataFrame.java b/baremaps-core/src/main/java/org/apache/baremaps/storage/shapefile/ShapefileDataFrame.java index a762de5d..2cc2f250 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/storage/shapefile/ShapefileDataFrame.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/storage/shapefile/ShapefileDataFrame.java @@ -22,7 +22,7 @@ import org.apache.baremaps.dataframe.DataFrame; import org.apache.baremaps.dataframe.DataFrameException; import org.apache.baremaps.dataframe.Row; import org.apache.baremaps.dataframe.Schema; -import org.apache.baremaps.storage.shapefile.internal.InputFeatureStream; +import org.apache.baremaps.storage.shapefile.internal.ShapefileInputStream; import org.apache.baremaps.storage.shapefile.internal.ShapefileReader; public class ShapefileDataFrame extends AbstractDataCollection<Row> @@ -64,22 +64,23 @@ public class ShapefileDataFrame extends AbstractDataCollection<Row> static class RowIterator implements Iterator<Row> { - private final InputFeatureStream inputFeatureStream; + private final ShapefileInputStream shapefileInputStream; private Row next; - public RowIterator(InputFeatureStream inputFeatureStream) { - this.inputFeatureStream = inputFeatureStream; + public RowIterator(ShapefileInputStream shapefileInputStream) { + this.shapefileInputStream = shapefileInputStream; } @Override public boolean hasNext() { try { if (next == null) { - next = inputFeatureStream.readRow(); + next = shapefileInputStream.readRow(); } return next != null; } catch (IOException exception) { + shapefileInputStream.close(); return false; } } @@ -88,12 +89,13 @@ public class ShapefileDataFrame extends AbstractDataCollection<Row> public Row next() { try { if (next == null) { - next = inputFeatureStream.readRow(); + next = shapefileInputStream.readRow(); } Row current = next; next = null; return current; } catch (Exception e) { + shapefileInputStream.close(); throw new NoSuchElementException(); } } diff --git a/baremaps-core/src/main/java/org/apache/baremaps/storage/shapefile/internal/DbaseByteReader.java b/baremaps-core/src/main/java/org/apache/baremaps/storage/shapefile/internal/DbaseByteReader.java index e48a1e0d..275d4e63 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/storage/shapefile/internal/DbaseByteReader.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/storage/shapefile/internal/DbaseByteReader.java @@ -108,8 +108,8 @@ public class DbaseByteReader extends CommonByteReader implements AutoCloseable { var check = nextRowAvailable(); - for (DBaseFieldDescriptor fd : this.fieldsDescriptors) { - byte[] data = new byte[fd.getLength()]; + for (DBaseFieldDescriptor fieldDescriptor : this.fieldsDescriptors) { + byte[] data = new byte[fieldDescriptor.getLength()]; getByteBuffer().get(data); int length = data.length; @@ -120,9 +120,9 @@ public class DbaseByteReader extends CommonByteReader implements AutoCloseable { String value = new String(data, 0, length); // TODO: move somewhere else - Object object = switch (fd.getType()) { + Object object = switch (fieldDescriptor.getType()) { case Character -> value; - case Number -> getNumber(fd, value); + case Number -> getNumber(fieldDescriptor, value); case Currency -> Double.parseDouble(value.trim()); case Integer -> Integer.parseInt(value.trim()); case Double -> Double.parseDouble(value.trim()); @@ -138,7 +138,7 @@ public class DbaseByteReader extends CommonByteReader implements AutoCloseable { case DateTime -> value; }; - row.set(fd.getName(), object); + row.set(fieldDescriptor.getName(), object); } } @@ -163,9 +163,6 @@ public class DbaseByteReader extends CommonByteReader implements AutoCloseable { // 2) Check that the immediate next byte read isn't the EOF signal. byte eofCheck = getByteBuffer().get(); - - boolean isEOF = (eofCheck == 0x1A); - if (eofCheck == 0x1A) { return false; } else { diff --git a/baremaps-core/src/main/java/org/apache/baremaps/storage/shapefile/internal/ShapefileByteReader.java b/baremaps-core/src/main/java/org/apache/baremaps/storage/shapefile/internal/ShapefileByteReader.java index dbc44ade..0b729913 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/storage/shapefile/internal/ShapefileByteReader.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/storage/shapefile/internal/ShapefileByteReader.java @@ -44,7 +44,7 @@ public class ShapefileByteReader extends CommonByteReader { /** Database Field descriptors. */ private List<DBaseFieldDescriptor> databaseFieldsDescriptors; - /** Schema of the features contained in this shapefile. */ + /** Schema of the rows contained in this shapefile. */ private Schema schema; /** Shapefile index. */ @@ -111,13 +111,13 @@ public class ShapefileByteReader extends CommonByteReader { } /** - * Create a feature descriptor. + * Create a row descriptor. * * @param name Name of the field. - * @return The feature type. + * @return The row type. */ private Schema getSchema(final String name) { - Objects.requireNonNull(name, "The feature name cannot be null."); + Objects.requireNonNull(name, "The row name cannot be null."); var columns = new ArrayList<Column>(); for (int i = 0; i < databaseFieldsDescriptors.size(); i++) { @@ -222,7 +222,7 @@ public class ShapefileByteReader extends CommonByteReader { } /** - * Direct access to a feature by its record number. + * Direct access to a row by its record number. * * @param recordNumber Record number. */ @@ -252,63 +252,63 @@ public class ShapefileByteReader extends CommonByteReader { } /** - * Complete a feature with shapefile content. + * Complete a row with shapefile content. * - * @param feature Feature to complete. + * @param row the row to complete */ - public void completeFeature(Row feature) throws ShapefileException { + public void completeRow(Row row) throws ShapefileException { // insert points into some type of list int RecordNumber = getByteBuffer().getInt(); int ContentLength = getByteBuffer().getInt(); getByteBuffer().order(ByteOrder.LITTLE_ENDIAN); - int iShapeType = getByteBuffer().getInt(); + int shapeTypeId = getByteBuffer().getInt(); - ShapeType type = ShapeType.get(iShapeType); + ShapefileGeometryType shapefileGeometryType = ShapefileGeometryType.get(shapeTypeId); - if (type == null) { + if (shapefileGeometryType == null) { throw new ShapefileException( - "The shapefile feature type doesn''t match to any known feature type."); + "The shapefile row type doesn''t match to any known row type."); } - switch (type) { + switch (shapefileGeometryType) { case Point: - loadPointFeature(feature); + loadPointRow(row); break; case Polygon: - loadPolygonFeature(feature); + loadPolygonRow(row); break; case PolyLine: - loadPolylineFeature(feature); + loadPolylineRow(row); break; default: - throw new ShapefileException("Unsupported shapefile type: " + iShapeType); + throw new ShapefileException("Unsupported shapefile type: " + shapeTypeId); } getByteBuffer().order(ByteOrder.BIG_ENDIAN); } /** - * Load point feature. + * Load point row. * - * @param feature Feature to fill. + * @param row the row to fill. */ - private void loadPointFeature(Row feature) { + private void loadPointRow(Row row) { double x = getByteBuffer().getDouble(); double y = getByteBuffer().getDouble(); Point pnt = geometryFactory.createPoint(new Coordinate(x, y)); - feature.set(GEOMETRY_NAME, pnt); + row.set(GEOMETRY_NAME, pnt); } /** - * Load polygon feature. + * Load polygon row. * - * @param feature Feature to fill. + * @param row the row to fill. */ - private void loadPolygonFeature(Row feature) { + private void loadPolygonRow(Row row) { double xmin = getByteBuffer().getDouble(); double ymin = getByteBuffer().getDouble(); double xmax = getByteBuffer().getDouble(); @@ -319,7 +319,7 @@ public class ShapefileByteReader extends CommonByteReader { Geometry multiPolygon = readMultiplePolygon(numParts, numPoints); - feature.set(GEOMETRY_NAME, multiPolygon); + row.set(GEOMETRY_NAME, multiPolygon); } /** @@ -380,11 +380,11 @@ public class ShapefileByteReader extends CommonByteReader { } /** - * Load polyline feature. + * Load polyline row. * - * @param feature Feature to fill. + * @param row the row to fill. */ - private void loadPolylineFeature(Row feature) { + private void loadPolylineRow(Row row) { /* double xmin = */ getByteBuffer().getDouble(); /* double ymin = */ getByteBuffer().getDouble(); /* double xmax = */ getByteBuffer().getDouble(); @@ -416,7 +416,7 @@ public class ShapefileByteReader extends CommonByteReader { } } - feature.set(GEOMETRY_NAME, + row.set(GEOMETRY_NAME, geometryFactory.createLineString(coordinates.toCoordinateArray())); } } diff --git a/baremaps-core/src/main/java/org/apache/baremaps/storage/shapefile/internal/ShapefileDescriptor.java b/baremaps-core/src/main/java/org/apache/baremaps/storage/shapefile/internal/ShapefileDescriptor.java index 0e231518..d4d26a1b 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/storage/shapefile/internal/ShapefileDescriptor.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/storage/shapefile/internal/ShapefileDescriptor.java @@ -35,7 +35,7 @@ public class ShapefileDescriptor { private int version; // little /** Shapefile type. */ - private ShapeType shapeType; // little + private ShapefileGeometryType shapefileGeometryType; // little /** X Min. */ private double xmin; // little @@ -77,7 +77,7 @@ public class ShapefileDescriptor { byteBuffer.order(ByteOrder.LITTLE_ENDIAN); this.version = byteBuffer.getInt(); - this.shapeType = ShapeType.get(byteBuffer.getInt()); + this.shapefileGeometryType = ShapefileGeometryType.get(byteBuffer.getInt()); this.xmin = byteBuffer.getDouble(); this.ymin = byteBuffer.getDouble(); this.xmax = byteBuffer.getDouble(); @@ -105,8 +105,8 @@ public class ShapefileDescriptor { * * @return Shape type. */ - public ShapeType getShapeType() { - return this.shapeType; + public ShapefileGeometryType getShapeType() { + return this.shapefileGeometryType; } /** @@ -190,7 +190,7 @@ public class ShapefileDescriptor { s.append("FileCode: ").append(this.fileCode).append(lineSeparator); s.append("FileLength: ").append(this.fileLength).append(lineSeparator); s.append("Version: ").append(this.version).append(lineSeparator); - s.append("ShapeType: ").append(this.shapeType).append(lineSeparator); + s.append("ShapeType: ").append(this.shapefileGeometryType).append(lineSeparator); s.append("xmin: ").append(this.xmin).append(lineSeparator); s.append("ymin: ").append(this.ymin).append(lineSeparator); s.append("xmax: ").append(this.xmax).append(lineSeparator); diff --git a/baremaps-core/src/main/java/org/apache/baremaps/storage/shapefile/internal/ShapeType.java b/baremaps-core/src/main/java/org/apache/baremaps/storage/shapefile/internal/ShapefileGeometryType.java similarity index 81% rename from baremaps-core/src/main/java/org/apache/baremaps/storage/shapefile/internal/ShapeType.java rename to baremaps-core/src/main/java/org/apache/baremaps/storage/shapefile/internal/ShapefileGeometryType.java index 2fdefdf0..904da21f 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/storage/shapefile/internal/ShapeType.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/storage/shapefile/internal/ShapefileGeometryType.java @@ -29,7 +29,7 @@ import java.util.Map; * @see <a href="http://www.esri.com/library/whitepapers/pdfs/shapefile.pdf";>ESRI Shapefile * Specification</a> */ -public enum ShapeType { +public enum ShapefileGeometryType { NullShape(0), Point(1), PolyLine(3), Polygon(5), MultiPoint(8), PointZ(11), PolyLineZ( 13), PolygonZ(15), MultiPointZ( 18), PointM(21), PolyLineM(23), PolygonM(25), MultiPointM(28), MultiPatch(31); @@ -37,7 +37,7 @@ public enum ShapeType { // used for initializing the enumeration private int value; - private ShapeType(int value) { + private ShapefileGeometryType(int value) { this.value = value; } @@ -45,15 +45,16 @@ public enum ShapeType { return value; } - private static final Map<Integer, ShapeType> lookup = new HashMap<Integer, ShapeType>(); + private static final Map<Integer, ShapefileGeometryType> lookup = + new HashMap<Integer, ShapefileGeometryType>(); static { - for (ShapeType ste : EnumSet.allOf(ShapeType.class)) { + for (ShapefileGeometryType ste : EnumSet.allOf(ShapefileGeometryType.class)) { lookup.put(ste.getValue(), ste); } } - public static ShapeType get(int value) { + public static ShapefileGeometryType get(int value) { return lookup.get(value); } } diff --git a/baremaps-core/src/main/java/org/apache/baremaps/storage/shapefile/internal/InputFeatureStream.java b/baremaps-core/src/main/java/org/apache/baremaps/storage/shapefile/internal/ShapefileInputStream.java similarity index 93% rename from baremaps-core/src/main/java/org/apache/baremaps/storage/shapefile/internal/InputFeatureStream.java rename to baremaps-core/src/main/java/org/apache/baremaps/storage/shapefile/internal/ShapefileInputStream.java index 4a0b4758..28cc36b3 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/storage/shapefile/internal/InputFeatureStream.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/storage/shapefile/internal/ShapefileInputStream.java @@ -30,7 +30,7 @@ import org.apache.baremaps.dataframe.Schema; * * @author Marc Le Bihan */ -public class InputFeatureStream extends InputStream { +public class ShapefileInputStream extends InputStream { private DbaseByteReader dbaseReader; @@ -59,7 +59,8 @@ public class InputFeatureStream extends InputStream { * @param dbaseFile Database file. * @param shpfileIndex Shapefile index, null if none provided, will be checked for existence. */ - public InputFeatureStream(File shapefile, File dbaseFile, File shpfileIndex) throws IOException { + public ShapefileInputStream(File shapefile, File dbaseFile, File shpfileIndex) + throws IOException { this.shapefile = shapefile; this.databaseFile = dbaseFile; this.dbaseReader = new DbaseByteReader(dbaseFile, null); @@ -83,7 +84,7 @@ public class InputFeatureStream extends InputStream { * @param shpfile Shapefile. * @param dbaseFile Database file. */ - public InputFeatureStream(File shpfile, File dbaseFile) throws IOException { + public ShapefileInputStream(File shpfile, File dbaseFile) throws IOException { this(shpfile, dbaseFile, null); } @@ -119,7 +120,7 @@ public class InputFeatureStream extends InputStream { Row row = this.schema.createRow(); this.dbaseReader.loadRow(row); this.shapefileReader.setRowNum(this.dbaseReader.getRowNum()); - this.shapefileReader.completeFeature(row); + this.shapefileReader.completeRow(row); return row; } diff --git a/baremaps-core/src/main/java/org/apache/baremaps/storage/shapefile/internal/ShapefileReader.java b/baremaps-core/src/main/java/org/apache/baremaps/storage/shapefile/internal/ShapefileReader.java index 3b3d5c52..5ad785f9 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/storage/shapefile/internal/ShapefileReader.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/storage/shapefile/internal/ShapefileReader.java @@ -171,9 +171,9 @@ public class ShapefileReader { * * @return Features */ - public InputFeatureStream read() throws IOException { - InputFeatureStream is = - new InputFeatureStream(this.shapefile, this.databaseFile, this.shapeFileIndex); + public ShapefileInputStream read() throws IOException { + ShapefileInputStream is = + new ShapefileInputStream(this.shapefile, this.databaseFile, this.shapeFileIndex); this.schema = is.getSchema(); this.shapefileDescriptor = is.getShapefileDescriptor(); this.databaseFieldsDescriptors = is.getDatabaseFieldsDescriptors(); @@ -186,7 +186,7 @@ public class ShapefileReader { */ public void loadDescriptors() throws IOException { // Doing a simple query will init the internal descriptors. - try (InputFeatureStream is = read()) { + try (ShapefileInputStream is = read()) { } } } diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportGeoPackage.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportGeoPackage.java index 784d093e..a2988501 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportGeoPackage.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportGeoPackage.java @@ -33,12 +33,13 @@ public record ImportGeoPackage(Path file, String database, Integer sourceSRID, I public void execute(WorkflowContext context) throws Exception { logger.info("Importing {} into {}", file, database); var path = file.toAbsolutePath(); - try (var geoPackageStore = new GeoPackageDatabase(path)) { + try (var geoPackageDatabase = new GeoPackageDatabase(path)) { var dataSource = context.getDataSource(database); var postgresDatabase = new PostgresDatabase(dataSource); - for (var dataFrame : geoPackageStore.list()) { + for (var name : geoPackageDatabase.list()) { var transformer = new ProjectionTransformer(sourceSRID, targetSRID); - var transformedDataFrame = new DataFrameGeometryTransformer(dataFrame, transformer); + var transformedDataFrame = + new DataFrameGeometryTransformer(geoPackageDatabase.get(name), transformer); postgresDatabase.add(transformedDataFrame); } logger.info("Finished importing {} into {}", file, database); diff --git a/baremaps-core/src/test/java/org/apache/baremaps/storage/flatgeobuf/FlatGeoBufDataFrameTest.java b/baremaps-core/src/test/java/org/apache/baremaps/storage/flatgeobuf/FlatGeoBufDataFrameTest.java index 32b8fab2..580f91df 100644 --- a/baremaps-core/src/test/java/org/apache/baremaps/storage/flatgeobuf/FlatGeoBufDataFrameTest.java +++ b/baremaps-core/src/test/java/org/apache/baremaps/storage/flatgeobuf/FlatGeoBufDataFrameTest.java @@ -15,9 +15,7 @@ package org.apache.baremaps.storage.flatgeobuf; import static org.junit.jupiter.api.Assertions.*; import java.io.IOException; -import java.nio.channels.FileChannel; import java.nio.file.Files; -import java.nio.file.StandardOpenOption; import org.apache.baremaps.testing.TestFiles; import org.junit.jupiter.api.Test; @@ -25,39 +23,31 @@ class FlatGeoBufDataFrameTest { @Test void getType() throws IOException { - try (var channel = FileChannel.open(TestFiles.resolve("countries.fgb"))) { - var featureSet = new FlatGeoBufDataFrame(channel); - var featureType = featureSet.getSchema(); - assertEquals(featureType.name(), null); - assertEquals(featureType.columns().size(), 2); - } + var featureSet = new FlatGeoBufDataFrame(TestFiles.resolve("countries.fgb")); + var featureType = featureSet.getSchema(); + assertEquals(featureType.name(), null); + assertEquals(featureType.columns().size(), 2); } @Test void read() throws IOException { - try (var channel = FileChannel.open(TestFiles.resolve("countries.fgb"))) { - var featureSet = new FlatGeoBufDataFrame(channel); - assertEquals(179, featureSet.read().size()); - assertEquals(179, featureSet.read().stream().count()); - } + var featureSet = new FlatGeoBufDataFrame(TestFiles.resolve("countries.fgb")); + assertEquals(179, featureSet.sizeAsLong()); + assertEquals(179, featureSet.stream().count()); + } @Test void write() throws IOException { var file = Files.createTempFile("countries", ".fgb"); file.toFile().deleteOnExit(); - try (var channel1 = FileChannel.open(TestFiles.resolve("countries.fgb")); - var channel2 = FileChannel.open(file, StandardOpenOption.WRITE)) { - var featureSet1 = new FlatGeoBufDataFrame(channel1); - var featureList = featureSet1.read().stream().toList(); - var featureSet2 = new FlatGeoBufDataFrame(channel2, featureSet1.getSchema()); - featureSet2.write(featureList); - } + var featureSet1 = new FlatGeoBufDataFrame(TestFiles.resolve("countries.fgb")); + var featureList = featureSet1.stream().toList(); + var featureSet2 = new FlatGeoBufDataFrame(file, featureSet1.getSchema()); + featureSet2.write(featureList); - try (var channel = FileChannel.open(file, StandardOpenOption.READ)) { - var featureSet = new FlatGeoBufDataFrame(channel); - assertEquals(179, featureSet.read().size()); - } + var featureSet = new FlatGeoBufDataFrame(file); + assertEquals(179, featureSet.stream().count()); }
