This is an automated email from the ASF dual-hosted git repository. bchapuis pushed a commit to branch openstreetmap-datatable in repository https://gitbox.apache.org/repos/asf/incubator-baremaps.git
commit aeca0ec792eb8596b6270a05889e67faab97fbc3 Author: Bertil Chapuis <[email protected]> AuthorDate: Thu Nov 7 10:03:22 2024 +0100 Add a data table for openstreetmap data --- .../openstreetmap/OpenStreetMapDataTable.java | 134 +++++++++++++++++++++ .../openstreetmap/OpenStreetMapDataTableTest.java | 51 ++++++++ .../baremaps/data/collection/DataCollection.java | 2 +- .../baremaps/openstreetmap/OpenStreetMap.java | 6 +- .../baremaps/openstreetmap/pbf/PbfBlockReader.java | 2 +- .../openstreetmap/pbf/PbfEntityReader.java | 2 +- .../baremaps/openstreetmap/state/StateReader.java | 26 ++-- .../openstreetmap/xml/XmlEntityReader.java | 2 +- 8 files changed, 207 insertions(+), 18 deletions(-) diff --git a/baremaps-core/src/main/java/org/apache/baremaps/storage/openstreetmap/OpenStreetMapDataTable.java b/baremaps-core/src/main/java/org/apache/baremaps/storage/openstreetmap/OpenStreetMapDataTable.java new file mode 100644 index 000000000..1fb4bbd43 --- /dev/null +++ b/baremaps-core/src/main/java/org/apache/baremaps/storage/openstreetmap/OpenStreetMapDataTable.java @@ -0,0 +1,134 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.baremaps.storage.openstreetmap; + +import java.io.InputStream; +import java.util.*; +import java.util.stream.Stream; +import org.apache.baremaps.data.storage.*; +import org.apache.baremaps.openstreetmap.OpenStreetMap.EntityReader; +import org.apache.baremaps.openstreetmap.model.*; + +/** + * A DataTable implementation for OpenStreetMap data. + */ +public class OpenStreetMapDataTable implements DataTable { + + private final DataSchema schema; + private final EntityReader<Entity> entityReader; + private final InputStream inputStream; + + /** + * Constructs an OpenStreetMapDataTable with the specified parameters. + * + * @param entityReader the EntityReader + * @param inputStream the input stream + */ + public OpenStreetMapDataTable(EntityReader<Entity> entityReader, InputStream inputStream) { + this.entityReader = entityReader; + this.inputStream = inputStream; + this.schema = createSchema(); + } + + /** + * Creates the schema for the OpenStreetMap data. + * + * @return the DataSchema + */ + private DataSchema createSchema() { + List<DataColumn> columns = List.of( + new DataColumnFixed("id", DataColumn.Cardinality.REQUIRED, DataColumn.Type.LONG), + new DataColumnFixed("type", DataColumn.Cardinality.REQUIRED, DataColumn.Type.STRING), + new DataColumnFixed("version", DataColumn.Cardinality.OPTIONAL, DataColumn.Type.INTEGER), + new DataColumnFixed("timestamp", DataColumn.Cardinality.OPTIONAL, + DataColumn.Type.LOCAL_DATE_TIME), + new DataColumnFixed("uid", DataColumn.Cardinality.OPTIONAL, DataColumn.Type.INTEGER), + new DataColumnFixed("user", DataColumn.Cardinality.OPTIONAL, DataColumn.Type.STRING), + new DataColumnFixed("changeset", DataColumn.Cardinality.OPTIONAL, DataColumn.Type.LONG), + new DataColumnFixed("tags", DataColumn.Cardinality.OPTIONAL, DataColumn.Type.NESTED), + new DataColumnFixed("geometry", DataColumn.Cardinality.OPTIONAL, DataColumn.Type.GEOMETRY)); + return new DataSchemaImpl("osm_data", columns); + } + + @Override + public DataSchema schema() { + return schema; + } + + @Override + public boolean add(DataRow row) { + throw new UnsupportedOperationException( + "Add operation is not supported for OpenStreetMapDataTable."); + } + + @Override + public void clear() { + throw new UnsupportedOperationException( + "Clear operation is not supported for OpenStreetMapDataTable."); + } + + @Override + public long size() { + // Unknown size + return Long.MAX_VALUE; + } + + @Override + public Iterator<DataRow> iterator() { + Stream<Element> elementStream = entityReader.read(inputStream) + .filter(element -> element instanceof Element) + .map(element -> (Element) element); + return elementStream.map(this::elementToDataRow).iterator(); + } + + /** + * Converts an Element to a DataRow. + * + * @param element the OSM Element + * @return the corresponding DataRow + */ + private DataRow elementToDataRow(Element element) { + DataRow row = schema.createRow(); + row.set("id", element.getId()); + row.set("type", elementTypeToString(element)); + row.set("version", element.getInfo().getVersion()); + row.set("timestamp", element.getInfo().getTimestamp()); + row.set("uid", element.getInfo().getUid()); + row.set("changeset", element.getInfo().getChangeset()); + row.set("tags", element.getTags()); + row.set("geometry", element.getGeometry()); + return row; + } + + /** + * Converts the element type to a String. + * + * @param element the OSM Element + * @return the element type as String + */ + private String elementTypeToString(Element element) { + if (element instanceof Node) + return "node"; + else if (element instanceof Way) + return "way"; + else if (element instanceof Relation) + return "relation"; + else + return "unknown"; + } +} diff --git a/baremaps-core/src/test/java/org/apache/baremaps/storage/openstreetmap/OpenStreetMapDataTableTest.java b/baremaps-core/src/test/java/org/apache/baremaps/storage/openstreetmap/OpenStreetMapDataTableTest.java new file mode 100644 index 000000000..4bf3adcbe --- /dev/null +++ b/baremaps-core/src/test/java/org/apache/baremaps/storage/openstreetmap/OpenStreetMapDataTableTest.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.baremaps.storage.openstreetmap; + +import static org.junit.jupiter.api.Assertions.*; + +import java.io.IOException; +import java.nio.file.Files; +import org.apache.baremaps.openstreetmap.pbf.PbfEntityReader; +import org.apache.baremaps.testing.TestFiles; +import org.junit.jupiter.api.Test; + +class OpenStreetMapDataTableTest { + + @Test + void schema() throws IOException { + var uri = TestFiles.resolve("baremaps-testing/data/osm-sample/sample.osm.pbf"); + try (var inputStream = Files.newInputStream(uri)) { + var table = new OpenStreetMapDataTable(new PbfEntityReader(), inputStream); + var rowType = table.schema(); + assertEquals(rowType.name(), "osm_data"); + assertEquals(9, rowType.columns().size()); + } + } + + @Test + void read() throws IOException { + var uri = TestFiles.resolve("baremaps-testing/data/osm-sample/sample.osm.pbf"); + try (var inputStream = Files.newInputStream(uri)) { + var table = new OpenStreetMapDataTable(new PbfEntityReader(), inputStream); + assertEquals(Long.MAX_VALUE, table.size()); + assertEquals(36, table.stream().count()); + } + } + +} diff --git a/baremaps-data/src/main/java/org/apache/baremaps/data/collection/DataCollection.java b/baremaps-data/src/main/java/org/apache/baremaps/data/collection/DataCollection.java index 7602ccec9..1189cd846 100644 --- a/baremaps-data/src/main/java/org/apache/baremaps/data/collection/DataCollection.java +++ b/baremaps-data/src/main/java/org/apache/baremaps/data/collection/DataCollection.java @@ -58,7 +58,7 @@ public interface DataCollection<E> extends Iterable<E> { */ @Override default Spliterator<E> spliterator() { - return Spliterators.spliterator(iterator(), size(), Spliterator.ORDERED); + return Spliterators.spliteratorUnknownSize(iterator(), Spliterator.ORDERED); } /** diff --git a/baremaps-openstreetmap/src/main/java/org/apache/baremaps/openstreetmap/OpenStreetMap.java b/baremaps-openstreetmap/src/main/java/org/apache/baremaps/openstreetmap/OpenStreetMap.java index 379488375..a9c2a3d3a 100644 --- a/baremaps-openstreetmap/src/main/java/org/apache/baremaps/openstreetmap/OpenStreetMap.java +++ b/baremaps-openstreetmap/src/main/java/org/apache/baremaps/openstreetmap/OpenStreetMap.java @@ -17,10 +17,10 @@ package org.apache.baremaps.openstreetmap; -import java.io.IOException; import java.io.InputStream; import java.util.List; import java.util.Map; +import java.util.stream.Stream; import org.apache.baremaps.openstreetmap.pbf.PbfBlockReader; import org.apache.baremaps.openstreetmap.pbf.PbfEntityReader; import org.apache.baremaps.openstreetmap.state.StateReader; @@ -87,7 +87,7 @@ public class OpenStreetMap { */ public interface Reader<T> { - T read(InputStream inputStream) throws IOException; + T read(InputStream inputStream); } @@ -96,7 +96,7 @@ public class OpenStreetMap { * * @param <T> the type of the object */ - public interface EntityReader<T> extends Reader<T> { + public interface EntityReader<T> extends Reader<Stream<T>> { /** * Gets the flag enabling the generation of geometries. diff --git a/baremaps-openstreetmap/src/main/java/org/apache/baremaps/openstreetmap/pbf/PbfBlockReader.java b/baremaps-openstreetmap/src/main/java/org/apache/baremaps/openstreetmap/pbf/PbfBlockReader.java index a031eccd5..52e5e26a9 100644 --- a/baremaps-openstreetmap/src/main/java/org/apache/baremaps/openstreetmap/pbf/PbfBlockReader.java +++ b/baremaps-openstreetmap/src/main/java/org/apache/baremaps/openstreetmap/pbf/PbfBlockReader.java @@ -28,7 +28,7 @@ import org.apache.baremaps.openstreetmap.stream.StreamUtils; import org.locationtech.jts.geom.Coordinate; /** A utility class for reading an OpenStreetMap pbf file. */ -public class PbfBlockReader implements PbfReader<Stream<Block>> { +public class PbfBlockReader implements PbfReader<Block> { private int buffer = Runtime.getRuntime().availableProcessors(); diff --git a/baremaps-openstreetmap/src/main/java/org/apache/baremaps/openstreetmap/pbf/PbfEntityReader.java b/baremaps-openstreetmap/src/main/java/org/apache/baremaps/openstreetmap/pbf/PbfEntityReader.java index dc23e8fc7..0f5abc7b3 100644 --- a/baremaps-openstreetmap/src/main/java/org/apache/baremaps/openstreetmap/pbf/PbfEntityReader.java +++ b/baremaps-openstreetmap/src/main/java/org/apache/baremaps/openstreetmap/pbf/PbfEntityReader.java @@ -30,7 +30,7 @@ import org.apache.baremaps.openstreetmap.stream.StreamException; import org.locationtech.jts.geom.Coordinate; /** A utility class for flattening the blocks streamed by a {@link PbfBlockReader}. */ -public class PbfEntityReader implements PbfReader<Stream<Entity>> { +public class PbfEntityReader implements PbfReader<Entity> { private final PbfBlockReader reader; diff --git a/baremaps-openstreetmap/src/main/java/org/apache/baremaps/openstreetmap/state/StateReader.java b/baremaps-openstreetmap/src/main/java/org/apache/baremaps/openstreetmap/state/StateReader.java index 85ab571ab..25f0a3cd1 100644 --- a/baremaps-openstreetmap/src/main/java/org/apache/baremaps/openstreetmap/state/StateReader.java +++ b/baremaps-openstreetmap/src/main/java/org/apache/baremaps/openstreetmap/state/StateReader.java @@ -88,19 +88,23 @@ public class StateReader implements Reader<State> { * @return the state */ @Override - public State read(InputStream input) throws IOException { - InputStreamReader reader = new InputStreamReader(input, StandardCharsets.UTF_8); - Map<String, String> map = new HashMap<>(); - for (String line : CharStreams.readLines(reader)) { - String[] array = line.split("="); - if (array.length == 2) { - map.put(array[0], array[1]); + public State read(InputStream input) { + try { + InputStreamReader reader = new InputStreamReader(input, StandardCharsets.UTF_8); + Map<String, String> map = new HashMap<>(); + for (String line : CharStreams.readLines(reader)) { + String[] array = line.split("="); + if (array.length == 2) { + map.put(array[0], array[1]); + } } + long sequenceNumber = Long.parseLong(map.get("sequenceNumber")); + DateTimeFormatter format = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss'Z'"); + LocalDateTime timestamp = LocalDateTime.parse(map.get("timestamp").replace("\\", ""), format); + return new State(sequenceNumber, timestamp); + } catch (IOException e) { + throw new RuntimeException(e); } - long sequenceNumber = Long.parseLong(map.get("sequenceNumber")); - DateTimeFormatter format = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss'Z'"); - LocalDateTime timestamp = LocalDateTime.parse(map.get("timestamp").replace("\\", ""), format); - return new State(sequenceNumber, timestamp); } /** diff --git a/baremaps-openstreetmap/src/main/java/org/apache/baremaps/openstreetmap/xml/XmlEntityReader.java b/baremaps-openstreetmap/src/main/java/org/apache/baremaps/openstreetmap/xml/XmlEntityReader.java index d79895515..2fa297122 100644 --- a/baremaps-openstreetmap/src/main/java/org/apache/baremaps/openstreetmap/xml/XmlEntityReader.java +++ b/baremaps-openstreetmap/src/main/java/org/apache/baremaps/openstreetmap/xml/XmlEntityReader.java @@ -35,7 +35,7 @@ import org.apache.baremaps.openstreetmap.model.Entity; import org.locationtech.jts.geom.Coordinate; /** A utility class for parsing an OpenStreetMap XML file. */ -public class XmlEntityReader implements EntityReader<Stream<Entity>> { +public class XmlEntityReader implements EntityReader<Entity> { private boolean geometry = false;
