This is an automated email from the ASF dual-hosted git repository.
bchapuis pushed a commit to branch 745-daylight
in repository https://gitbox.apache.org/repos/asf/incubator-baremaps.git
The following commit(s) were added to refs/heads/745-daylight by this push:
new 74241cb0 Assume that osm change files are standalone
74241cb0 is described below
commit 74241cb0b202bee95a2f794b3433098e6e654ee8
Author: Bertil Chapuis <[email protected]>
AuthorDate: Thu Sep 28 21:43:12 2023 +0200
Assume that osm change files are standalone
---
.../baremaps/workflow/tasks/ImportOsmChange.java | 57 ++++++++++++++++++----
.../baremaps/workflow/tasks/ImportOsmPbf.java | 41 ++++------------
2 files changed, 58 insertions(+), 40 deletions(-)
diff --git
a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportOsmChange.java
b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportOsmChange.java
index b6112682..5bc0c085 100644
---
a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportOsmChange.java
+++
b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportOsmChange.java
@@ -17,15 +17,22 @@ import static
org.apache.baremaps.stream.ConsumerUtils.consumeThenReturn;
import java.io.BufferedInputStream;
import java.nio.file.Files;
import java.nio.file.Path;
-import org.apache.baremaps.openstreetmap.function.ChangeEntitiesHandler;
-import org.apache.baremaps.openstreetmap.function.EntityGeometryBuilder;
-import org.apache.baremaps.openstreetmap.function.EntityProjectionTransformer;
+import java.nio.file.Paths;
+
+import org.apache.baremaps.database.collection.*;
+import org.apache.baremaps.database.memory.MemoryMappedDirectory;
+import org.apache.baremaps.database.type.LongDataType;
+import org.apache.baremaps.database.type.LongListDataType;
+import org.apache.baremaps.database.type.PairDataType;
+import org.apache.baremaps.database.type.geometry.LonLatDataType;
+import org.apache.baremaps.openstreetmap.function.*;
import org.apache.baremaps.openstreetmap.postgres.*;
import org.apache.baremaps.openstreetmap.repository.ChangeImporter;
import org.apache.baremaps.openstreetmap.xml.XmlChangeReader;
import org.apache.baremaps.utils.Compression;
import org.apache.baremaps.workflow.Task;
import org.apache.baremaps.workflow.WorkflowContext;
+import org.locationtech.jts.geom.Coordinate;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -41,22 +48,54 @@ public record ImportOsmChange(Path file, String database,
Integer srid,
@Override
public void execute(WorkflowContext context) throws Exception {
var datasource = context.getDataSource(database);
+ var path = file.toAbsolutePath();
+
+ var cacheDir = Files.createTempDirectory(Paths.get("."), "cache_");
+
+ DataMap<Long, Coordinate> coordinateMap;
+ if (Files.size(path) > 1 << 30) {
+ var coordinateDir =
Files.createDirectories(cacheDir.resolve("coordinate_keys"));
+ coordinateMap = new MemoryAlignedDataMap<>(
+ new LonLatDataType(),
+ new MemoryMappedDirectory(coordinateDir));
+ } else {
+ var coordinateKeysDir =
Files.createDirectories(cacheDir.resolve("coordinate_keys"));
+ var coordinateValuesDir =
Files.createDirectories(cacheDir.resolve("coordinate_vals"));
+ coordinateMap =
+ new MonotonicDataMap<>(
+ new MemoryAlignedDataList<>(
+ new PairDataType<>(new LongDataType(), new
LongDataType()),
+ new MemoryMappedDirectory(coordinateKeysDir)),
+ new AppendOnlyBuffer<>(
+ new LonLatDataType(),
+ new MemoryMappedDirectory(coordinateValuesDir)));
+ }
- var coordinateMap = new PostgresCoordinateMap(datasource);
- var referenceMap = new PostgresReferenceMap(datasource);
+ var referenceKeysDir =
Files.createDirectory(cacheDir.resolve("reference_keys"));
+ var referenceValuesDir =
Files.createDirectory(cacheDir.resolve("reference_vals"));
+ var referenceMap =
+ new MonotonicDataMap<>(
+ new MemoryAlignedDataList<>(
+ new PairDataType<>(new LongDataType(), new
LongDataType()),
+ new MemoryMappedDirectory(referenceKeysDir)),
+ new AppendOnlyBuffer<>(
+ new LongListDataType(),
+ new MemoryMappedDirectory(referenceValuesDir)));
var nodeRepository = new PostgresNodeRepository(datasource);
var wayRepository = new PostgresWayRepository(datasource);
var relationRepository = new PostgresRelationRepository(datasource);
- var createGeometry = new EntityGeometryBuilder(coordinateMap,
referenceMap);
+ var coordinateMapBuilder = new CoordinateMapBuilder(coordinateMap);
+ var referenceMapBuilder = new ReferenceMapBuilder(referenceMap);
+ var buildGeometry = new EntityGeometryBuilder(coordinateMap, referenceMap);
var reprojectGeometry = new EntityProjectionTransformer(4326, srid);
- var prepareGeometries = new
ChangeEntitiesHandler(createGeometry.andThen(reprojectGeometry));
- var prepareChange = consumeThenReturn(prepareGeometries);
+ var prepareGeometries =
coordinateMapBuilder.andThen(referenceMapBuilder).andThen(buildGeometry).andThen(reprojectGeometry);
+ var prepareChange = consumeThenReturn(new
ChangeEntitiesHandler(prepareGeometries));
var importChange = new ChangeImporter(nodeRepository, wayRepository,
relationRepository);
try (var changeInputStream =
- new
BufferedInputStream(compression.decompress(Files.newInputStream(file)))) {
+ new
BufferedInputStream(compression.decompress(Files.newInputStream(path)))) {
new
XmlChangeReader().stream(changeInputStream).map(prepareChange).forEach(importChange);
}
}
diff --git
a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportOsmPbf.java
b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportOsmPbf.java
index 662cd7ca..5ec5c7b7 100644
---
a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportOsmPbf.java
+++
b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportOsmPbf.java
@@ -99,46 +99,25 @@ public record ImportOsmPbf(Path file, Object database,
Integer databaseSrid)
new LongListDataType(),
new MemoryMappedDirectory(referenceValuesDir)));
- execute(
- path,
- coordinateMap,
- referenceMap,
- headerRepository,
- nodeRepository,
- wayRepository,
- relationRepository,
- databaseSrid);
-
- FileUtils.deleteRecursively(cacheDir);
- }
-
- public static void execute(
- Path path,
- DataMap<Long, Coordinate> coordinateMap,
- DataMap<Long, List<Long>> referenceMap,
- HeaderRepository headerRepository,
- Repository<Long, Node> nodeRepository,
- Repository<Long, Way> wayRepository,
- Repository<Long, Relation> relationRepository,
- Integer databaseSrid) throws IOException {
-
// configure the block reader
var reader = new PbfBlockReader()
- .geometries(true)
- .projection(databaseSrid)
- .coordinateMap(coordinateMap)
- .referenceMap(referenceMap);
+ .geometries(true)
+ .projection(databaseSrid)
+ .coordinateMap(coordinateMap)
+ .referenceMap(referenceMap);
// configure the block importer
var importer = new BlockImporter(
- headerRepository,
- nodeRepository,
- wayRepository,
- relationRepository);
+ headerRepository,
+ nodeRepository,
+ wayRepository,
+ relationRepository);
// Stream and process the blocks
try (var input = Files.newInputStream(path)) {
StreamUtils.batch(reader.stream(input)).forEach(importer);
}
+
+ FileUtils.deleteRecursively(cacheDir);
}
}
