This is an automated email from the ASF dual-hosted git repository. bchapuis pushed a commit to branch 745-daylight in repository https://gitbox.apache.org/repos/asf/incubator-baremaps.git
commit 582469a1742ad1d75e5e7d3a1a52b01b7880c7a7 Author: Bertil Chapuis <[email protected]> AuthorDate: Thu Sep 28 21:43:12 2023 +0200 Assume that osm change files are standalone --- .../baremaps/workflow/tasks/ImportOsmChange.java | 57 ++++++++++++++++++---- .../baremaps/workflow/tasks/ImportOsmPbf.java | 41 ++++------------ 2 files changed, 58 insertions(+), 40 deletions(-) diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportOsmChange.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportOsmChange.java index b6112682..5bc0c085 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportOsmChange.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportOsmChange.java @@ -17,15 +17,22 @@ import static org.apache.baremaps.stream.ConsumerUtils.consumeThenReturn; import java.io.BufferedInputStream; import java.nio.file.Files; import java.nio.file.Path; -import org.apache.baremaps.openstreetmap.function.ChangeEntitiesHandler; -import org.apache.baremaps.openstreetmap.function.EntityGeometryBuilder; -import org.apache.baremaps.openstreetmap.function.EntityProjectionTransformer; +import java.nio.file.Paths; + +import org.apache.baremaps.database.collection.*; +import org.apache.baremaps.database.memory.MemoryMappedDirectory; +import org.apache.baremaps.database.type.LongDataType; +import org.apache.baremaps.database.type.LongListDataType; +import org.apache.baremaps.database.type.PairDataType; +import org.apache.baremaps.database.type.geometry.LonLatDataType; +import org.apache.baremaps.openstreetmap.function.*; import org.apache.baremaps.openstreetmap.postgres.*; import org.apache.baremaps.openstreetmap.repository.ChangeImporter; import org.apache.baremaps.openstreetmap.xml.XmlChangeReader; import org.apache.baremaps.utils.Compression; import org.apache.baremaps.workflow.Task; import org.apache.baremaps.workflow.WorkflowContext; +import org.locationtech.jts.geom.Coordinate; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -41,22 +48,54 @@ public record ImportOsmChange(Path file, String database, Integer srid, @Override public void execute(WorkflowContext context) throws Exception { var datasource = context.getDataSource(database); + var path = file.toAbsolutePath(); + + var cacheDir = Files.createTempDirectory(Paths.get("."), "cache_"); + + DataMap<Long, Coordinate> coordinateMap; + if (Files.size(path) > 1 << 30) { + var coordinateDir = Files.createDirectories(cacheDir.resolve("coordinate_keys")); + coordinateMap = new MemoryAlignedDataMap<>( + new LonLatDataType(), + new MemoryMappedDirectory(coordinateDir)); + } else { + var coordinateKeysDir = Files.createDirectories(cacheDir.resolve("coordinate_keys")); + var coordinateValuesDir = Files.createDirectories(cacheDir.resolve("coordinate_vals")); + coordinateMap = + new MonotonicDataMap<>( + new MemoryAlignedDataList<>( + new PairDataType<>(new LongDataType(), new LongDataType()), + new MemoryMappedDirectory(coordinateKeysDir)), + new AppendOnlyBuffer<>( + new LonLatDataType(), + new MemoryMappedDirectory(coordinateValuesDir))); + } - var coordinateMap = new PostgresCoordinateMap(datasource); - var referenceMap = new PostgresReferenceMap(datasource); + var referenceKeysDir = Files.createDirectory(cacheDir.resolve("reference_keys")); + var referenceValuesDir = Files.createDirectory(cacheDir.resolve("reference_vals")); + var referenceMap = + new MonotonicDataMap<>( + new MemoryAlignedDataList<>( + new PairDataType<>(new LongDataType(), new LongDataType()), + new MemoryMappedDirectory(referenceKeysDir)), + new AppendOnlyBuffer<>( + new LongListDataType(), + new MemoryMappedDirectory(referenceValuesDir))); var nodeRepository = new PostgresNodeRepository(datasource); var wayRepository = new PostgresWayRepository(datasource); var relationRepository = new PostgresRelationRepository(datasource); - var createGeometry = new EntityGeometryBuilder(coordinateMap, referenceMap); + var coordinateMapBuilder = new CoordinateMapBuilder(coordinateMap); + var referenceMapBuilder = new ReferenceMapBuilder(referenceMap); + var buildGeometry = new EntityGeometryBuilder(coordinateMap, referenceMap); var reprojectGeometry = new EntityProjectionTransformer(4326, srid); - var prepareGeometries = new ChangeEntitiesHandler(createGeometry.andThen(reprojectGeometry)); - var prepareChange = consumeThenReturn(prepareGeometries); + var prepareGeometries = coordinateMapBuilder.andThen(referenceMapBuilder).andThen(buildGeometry).andThen(reprojectGeometry); + var prepareChange = consumeThenReturn(new ChangeEntitiesHandler(prepareGeometries)); var importChange = new ChangeImporter(nodeRepository, wayRepository, relationRepository); try (var changeInputStream = - new BufferedInputStream(compression.decompress(Files.newInputStream(file)))) { + new BufferedInputStream(compression.decompress(Files.newInputStream(path)))) { new XmlChangeReader().stream(changeInputStream).map(prepareChange).forEach(importChange); } } diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportOsmPbf.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportOsmPbf.java index 662cd7ca..5ec5c7b7 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportOsmPbf.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportOsmPbf.java @@ -99,46 +99,25 @@ public record ImportOsmPbf(Path file, Object database, Integer databaseSrid) new LongListDataType(), new MemoryMappedDirectory(referenceValuesDir))); - execute( - path, - coordinateMap, - referenceMap, - headerRepository, - nodeRepository, - wayRepository, - relationRepository, - databaseSrid); - - FileUtils.deleteRecursively(cacheDir); - } - - public static void execute( - Path path, - DataMap<Long, Coordinate> coordinateMap, - DataMap<Long, List<Long>> referenceMap, - HeaderRepository headerRepository, - Repository<Long, Node> nodeRepository, - Repository<Long, Way> wayRepository, - Repository<Long, Relation> relationRepository, - Integer databaseSrid) throws IOException { - // configure the block reader var reader = new PbfBlockReader() - .geometries(true) - .projection(databaseSrid) - .coordinateMap(coordinateMap) - .referenceMap(referenceMap); + .geometries(true) + .projection(databaseSrid) + .coordinateMap(coordinateMap) + .referenceMap(referenceMap); // configure the block importer var importer = new BlockImporter( - headerRepository, - nodeRepository, - wayRepository, - relationRepository); + headerRepository, + nodeRepository, + wayRepository, + relationRepository); // Stream and process the blocks try (var input = Files.newInputStream(path)) { StreamUtils.batch(reader.stream(input)).forEach(importer); } + + FileUtils.deleteRecursively(cacheDir); } }
