This is an automated email from the ASF dual-hosted git repository. bchapuis pushed a commit to branch 745-daylight in repository https://gitbox.apache.org/repos/asf/incubator-baremaps.git
commit e4bda42ce9c748a21d2b99117890ed9f5905fa21 Author: Bertil Chapuis <[email protected]> AuthorDate: Mon Aug 28 17:03:47 2023 +0200 wqRename import osm task and add osc task --- .../org/apache/baremaps/cli/database/Database.java | 2 +- ...{ImportOpenStreetMap.java => ImportOsmPbf.java} | 8 +- .../java/org/apache/baremaps/workflow/Task.java | 2 +- .../apache/baremaps/workflow/tasks/ImportOsc.java | 91 ++++++++++++++++++++++ ...{ImportOpenStreetMap.java => ImportOsmPbf.java} | 2 +- .../apache/baremaps/workflow/ObjectMapperTest.java | 8 +- .../org/apache/baremaps/workflow/WorkflowTest.java | 4 +- .../baremaps/workflow/tasks/ImportMonacoTest.java | 2 +- ...rtOpenStreetMapTest.java => ImportPbfTest.java} | 4 +- .../workflow/tasks/ImportUpdateDataTest.java | 2 +- .../tasks/ImportUpdateLiechtensteinTest.java | 2 +- basemap/daylight/workflow.js | 2 +- basemap/workflow.js | 2 +- examples/extrusion/workflow.json | 2 +- examples/openstreetmap/workflow.json | 2 +- 15 files changed, 114 insertions(+), 21 deletions(-) diff --git a/baremaps-cli/src/main/java/org/apache/baremaps/cli/database/Database.java b/baremaps-cli/src/main/java/org/apache/baremaps/cli/database/Database.java index dc83f45b..3d2df584 100644 --- a/baremaps-cli/src/main/java/org/apache/baremaps/cli/database/Database.java +++ b/baremaps-cli/src/main/java/org/apache/baremaps/cli/database/Database.java @@ -18,7 +18,7 @@ import picocli.CommandLine; import picocli.CommandLine.Command; @Command(name = "database", description = "Database commands.", - subcommands = {ExecuteSql.class, ImportOpenStreetMap.class, UpdateOpenStreetMap.class}, + subcommands = {ExecuteSql.class, ImportOsmPbf.class, UpdateOpenStreetMap.class}, sortOptions = false) public class Database implements Runnable { diff --git a/baremaps-cli/src/main/java/org/apache/baremaps/cli/database/ImportOpenStreetMap.java b/baremaps-cli/src/main/java/org/apache/baremaps/cli/database/ImportOsmPbf.java similarity index 87% rename from baremaps-cli/src/main/java/org/apache/baremaps/cli/database/ImportOpenStreetMap.java rename to baremaps-cli/src/main/java/org/apache/baremaps/cli/database/ImportOsmPbf.java index b64cb565..dbfc68e8 100644 --- a/baremaps-cli/src/main/java/org/apache/baremaps/cli/database/ImportOpenStreetMap.java +++ b/baremaps-cli/src/main/java/org/apache/baremaps/cli/database/ImportOsmPbf.java @@ -23,7 +23,7 @@ import picocli.CommandLine.Mixin; import picocli.CommandLine.Option; @Command(name = "import-osm", description = "Import OpenStreetMap data in Postgres.") -public class ImportOpenStreetMap implements Callable<Integer> { +public class ImportOsmPbf implements Callable<Integer> { @Mixin private Options options; @@ -42,8 +42,10 @@ public class ImportOpenStreetMap implements Callable<Integer> { @Override public Integer call() throws Exception { - new org.apache.baremaps.workflow.tasks.ImportOpenStreetMap(file.toAbsolutePath(), - database, srid).execute(new WorkflowContext()); + new org.apache.baremaps.workflow.tasks.ImportOsmPbf( + file.toAbsolutePath(), + database, + srid).execute(new WorkflowContext()); return 0; } } diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/Task.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/Task.java index 3fb601b1..58a0ca71 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/Task.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/Task.java @@ -33,7 +33,7 @@ import org.apache.baremaps.workflow.tasks.*; @JsonSubTypes.Type(value = ExecuteSqlScript.class, name = "ExecuteSqlScript"), @JsonSubTypes.Type(value = ExportVectorTiles.class, name = "ExportVectorTiles"), @JsonSubTypes.Type(value = ImportGeoPackage.class, name = "ImportGeoPackage"), - @JsonSubTypes.Type(value = ImportOpenStreetMap.class, name = "ImportOpenStreetMap"), + @JsonSubTypes.Type(value = ImportOsmPbf.class, name = "ImportOsmPbf"), @JsonSubTypes.Type(value = ImportShapefile.class, name = "ImportShapefile"), @JsonSubTypes.Type(value = LogMessage.class, name = "LogMessage"), @JsonSubTypes.Type(value = UnzipFile.class, name = "UnzipFile"), diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportOsc.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportOsc.java new file mode 100644 index 00000000..f0aa8285 --- /dev/null +++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportOsc.java @@ -0,0 +1,91 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + */ + +package org.apache.baremaps.workflow.tasks; + +import static org.apache.baremaps.stream.ConsumerUtils.consumeThenReturn; + +import java.io.BufferedInputStream; +import java.net.MalformedURLException; +import java.net.URI; +import java.net.URL; +import java.nio.file.Path; +import java.util.zip.GZIPInputStream; +import org.apache.baremaps.openstreetmap.function.ChangeEntitiesHandler; +import org.apache.baremaps.openstreetmap.function.EntityGeometryBuilder; +import org.apache.baremaps.openstreetmap.function.EntityProjectionTransformer; +import org.apache.baremaps.openstreetmap.model.Header; +import org.apache.baremaps.openstreetmap.model.Node; +import org.apache.baremaps.openstreetmap.model.Relation; +import org.apache.baremaps.openstreetmap.model.Way; +import org.apache.baremaps.openstreetmap.postgres.*; +import org.apache.baremaps.openstreetmap.repository.ChangeImporter; +import org.apache.baremaps.openstreetmap.repository.Repository; +import org.apache.baremaps.openstreetmap.state.StateReader; +import org.apache.baremaps.openstreetmap.xml.XmlChangeReader; +import org.apache.baremaps.workflow.Task; +import org.apache.baremaps.workflow.WorkflowContext; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public record ImportOsc(Path file, String database, Integer srid) implements Task { + + private static final Logger logger = LoggerFactory.getLogger(ImportOsc.class); + + @Override + public void execute(WorkflowContext context) throws Exception { + var datasource = context.getDataSource(database); + + Repository<Long, Node> nodeRepository = new PostgresNodeRepository(datasource); + Repository<Long, Way> wayRepository = new PostgresWayRepository(datasource); + Repository<Long, Relation> relationRepository = new PostgresRelationRepository(datasource); + execute( + nodeRepository, + wayRepository, + relationRepository, + srid); + } + + public static void execute( + Repository<Long, Node> nodeRepository, + Repository<Long, Way> wayRepository, + Repository<Long, Relation> relationRepository, + int srid) throws Exception { + + var createGeometry = new EntityGeometryBuilder(coordinateMap, referenceMap); + var reprojectGeometry = new EntityProjectionTransformer(4326, srid); + var prepareGeometries = new ChangeEntitiesHandler(createGeometry.andThen(reprojectGeometry)); + var prepareChange = consumeThenReturn(prepareGeometries); + var saveChange = new ChangeImporter(nodeRepository, wayRepository, relationRepository); + + var changeUrl = resolve(replicationUrl, sequenceNumber, "osc.gz"); + try (var changeInputStream = + new GZIPInputStream(new BufferedInputStream(changeUrl.openStream()))) { + new XmlChangeReader().stream(changeInputStream).map(prepareChange).forEach(saveChange); + } + + var stateUrl = resolve(replicationUrl, sequenceNumber, "state.txt"); + try (var stateInputStream = new BufferedInputStream(stateUrl.openStream())) { + var state = new StateReader().state(stateInputStream); + headerRepository.put(new Header(state.getSequenceNumber(), state.getTimestamp(), + header.getReplicationUrl(), header.getSource(), header.getWritingProgram())); + } + } + + public static URL resolve(String replicationUrl, Long sequenceNumber, String extension) + throws MalformedURLException { + var s = String.format("%09d", sequenceNumber); + var uri = String.format("%s/%s/%s/%s.%s", replicationUrl, s.substring(0, 3), s.substring(3, 6), + s.substring(6, 9), extension); + return URI.create(uri).toURL(); + } +} diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportOpenStreetMap.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportOsmPbf.java similarity index 99% rename from baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportOpenStreetMap.java rename to baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportOsmPbf.java index c387fe0c..37c6de61 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportOpenStreetMap.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportOsmPbf.java @@ -45,7 +45,7 @@ public record ImportOpenStreetMap(Path file, Object database, Integer databaseSr implements Task { - private static final Logger logger = LoggerFactory.getLogger(ImportOpenStreetMap.class); + private static final Logger logger = LoggerFactory.getLogger(ImportOsmPbf.class); @Override public void execute(WorkflowContext context) throws Exception { diff --git a/baremaps-core/src/test/java/org/apache/baremaps/workflow/ObjectMapperTest.java b/baremaps-core/src/test/java/org/apache/baremaps/workflow/ObjectMapperTest.java index 24d34d69..ac62e875 100644 --- a/baremaps-core/src/test/java/org/apache/baremaps/workflow/ObjectMapperTest.java +++ b/baremaps-core/src/test/java/org/apache/baremaps/workflow/ObjectMapperTest.java @@ -19,7 +19,7 @@ import java.io.IOException; import java.nio.file.Paths; import java.util.List; import org.apache.baremaps.workflow.tasks.DownloadUrl; -import org.apache.baremaps.workflow.tasks.ImportOpenStreetMap; +import org.apache.baremaps.workflow.tasks.ImportOsmPbf; import org.junit.Test; public class ObjectMapperTest { @@ -36,16 +36,16 @@ public class ObjectMapperTest { "https://download.geofabrik.de/europe/liechtenstein-latest.osm.pbf";, Paths.get("liechtenstein-latest.osm.pbf")))), new Step("import", List.of("download"), - List.of(new ImportOpenStreetMap(Paths.get("liechtenstein-latest.osm.pbf"), + List.of(new ImportOsmPbf(Paths.get("liechtenstein-latest.osm.pbf"), "jdbc:postgresql://localhost:5432/baremaps?&user=baremaps&password=baremaps", 3857))))); var json = mapper.writeValueAsString(workflow1); assertTrue(json.contains(DownloadUrl.class.getSimpleName())); - assertTrue(json.contains(ImportOpenStreetMap.class.getSimpleName())); + assertTrue(json.contains(ImportOsmPbf.class.getSimpleName())); // deserialize the workflow var workflow2 = mapper.readValue(json, Workflow.class); assertTrue(workflow2.getSteps().get(0).getTasks().get(0) instanceof DownloadUrl); - assertTrue(workflow2.getSteps().get(1).getTasks().get(0) instanceof ImportOpenStreetMap); + assertTrue(workflow2.getSteps().get(1).getTasks().get(0) instanceof ImportOsmPbf); } } diff --git a/baremaps-core/src/test/java/org/apache/baremaps/workflow/WorkflowTest.java b/baremaps-core/src/test/java/org/apache/baremaps/workflow/WorkflowTest.java index 57bb1fb1..4dcd682a 100644 --- a/baremaps-core/src/test/java/org/apache/baremaps/workflow/WorkflowTest.java +++ b/baremaps-core/src/test/java/org/apache/baremaps/workflow/WorkflowTest.java @@ -19,7 +19,7 @@ import java.util.List; import org.apache.baremaps.testing.PostgresContainerTest; import org.apache.baremaps.workflow.tasks.DownloadUrl; import org.apache.baremaps.workflow.tasks.ImportGeoPackage; -import org.apache.baremaps.workflow.tasks.ImportOpenStreetMap; +import org.apache.baremaps.workflow.tasks.ImportOsmPbf; import org.apache.baremaps.workflow.tasks.ImportShapefile; import org.apache.baremaps.workflow.tasks.UnzipFile; import org.junit.jupiter.api.Disabled; @@ -98,7 +98,7 @@ class WorkflowTest extends PostgresContainerTest { List.of(new DownloadUrl("https://tiles.baremaps.com/samples/liechtenstein.osm.pbf";, Paths.get("downloads/liechtenstein.osm.pbf")))), new Step("import-osmpbf", List.of("fetch-osmpbf"), - List.of(new ImportOpenStreetMap(Paths.get("downloads/liechtenstein.osm.pbf"), jdbcUrl(), + List.of(new ImportOsmPbf(Paths.get("downloads/liechtenstein.osm.pbf"), jdbcUrl(), 3857))), new Step("fetch-shapefile", List.of(), List.of(new DownloadUrl( "https://osmdata.openstreetmap.de/download/simplified-water-polygons-split-3857.zip";, diff --git a/baremaps-core/src/test/java/org/apache/baremaps/workflow/tasks/ImportMonacoTest.java b/baremaps-core/src/test/java/org/apache/baremaps/workflow/tasks/ImportMonacoTest.java index aa5d26b5..0a1d05db 100644 --- a/baremaps-core/src/test/java/org/apache/baremaps/workflow/tasks/ImportMonacoTest.java +++ b/baremaps-core/src/test/java/org/apache/baremaps/workflow/tasks/ImportMonacoTest.java @@ -52,7 +52,7 @@ class ImportMonacoTest extends PostgresRepositoryTest { new IndexedDataMap<>(new AppendOnlyBuffer<>(new LongListDataType(), new OnHeapMemory())); // Import data - ImportOpenStreetMap.execute(TestFiles.resolve("monaco/monaco-210801.osm.pbf"), coordinateMap, + ImportOsmPbf.execute(TestFiles.resolve("monaco/monaco-210801.osm.pbf"), coordinateMap, referenceMap, headerRepository, nodeRepository, wayRepository, relationRepository, 3857); assertEquals(3047l, headerRepository.selectLatest().getReplicationSequenceNumber()); diff --git a/baremaps-core/src/test/java/org/apache/baremaps/workflow/tasks/ImportOpenStreetMapTest.java b/baremaps-core/src/test/java/org/apache/baremaps/workflow/tasks/ImportPbfTest.java similarity index 89% rename from baremaps-core/src/test/java/org/apache/baremaps/workflow/tasks/ImportOpenStreetMapTest.java rename to baremaps-core/src/test/java/org/apache/baremaps/workflow/tasks/ImportPbfTest.java index ba457f2f..60761564 100644 --- a/baremaps-core/src/test/java/org/apache/baremaps/workflow/tasks/ImportOpenStreetMapTest.java +++ b/baremaps-core/src/test/java/org/apache/baremaps/workflow/tasks/ImportPbfTest.java @@ -20,7 +20,7 @@ import org.apache.baremaps.workflow.WorkflowContext; import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -class ImportOpenStreetMapTest extends PostgresContainerTest { +class ImportOsmPbfTest extends PostgresContainerTest { @Test @Tag("integration") @@ -28,7 +28,7 @@ class ImportOpenStreetMapTest extends PostgresContainerTest { var file = TestFiles.resolve("data.osm.pbf"); var jdbcUrl = jdbcUrl(); var srid = 3857; - var task = new ImportOpenStreetMap(file, jdbcUrl, srid); + var task = new ImportOsmPbf(file, jdbcUrl, srid); task.execute(new WorkflowContext()); } } diff --git a/baremaps-core/src/test/java/org/apache/baremaps/workflow/tasks/ImportUpdateDataTest.java b/baremaps-core/src/test/java/org/apache/baremaps/workflow/tasks/ImportUpdateDataTest.java index ff23d891..e1e49dfc 100644 --- a/baremaps-core/src/test/java/org/apache/baremaps/workflow/tasks/ImportUpdateDataTest.java +++ b/baremaps-core/src/test/java/org/apache/baremaps/workflow/tasks/ImportUpdateDataTest.java @@ -56,7 +56,7 @@ class ImportUpdateDataTest extends PostgresRepositoryTest { new IndexedDataMap<>(new AppendOnlyBuffer<>(new LongListDataType(), new OnHeapMemory())); // Import data - ImportOpenStreetMap.execute(SIMPLE_DATA_OSM_PBF, coordinateMap, referenceMap, headerRepository, + ImportOsmPbf.execute(SIMPLE_DATA_OSM_PBF, coordinateMap, referenceMap, headerRepository, nodeRepository, wayRepository, relationRepository, 3857); headerRepository.put(new Header(0l, LocalDateTime.of(2020, 1, 1, 0, 0, 0, 0), diff --git a/baremaps-core/src/test/java/org/apache/baremaps/workflow/tasks/ImportUpdateLiechtensteinTest.java b/baremaps-core/src/test/java/org/apache/baremaps/workflow/tasks/ImportUpdateLiechtensteinTest.java index fd1e23c6..83990894 100644 --- a/baremaps-core/src/test/java/org/apache/baremaps/workflow/tasks/ImportUpdateLiechtensteinTest.java +++ b/baremaps-core/src/test/java/org/apache/baremaps/workflow/tasks/ImportUpdateLiechtensteinTest.java @@ -53,7 +53,7 @@ class ImportUpdateLiechtensteinTest extends PostgresRepositoryTest { new IndexedDataMap<>(new AppendOnlyBuffer<>(new LongListDataType(), new OnHeapMemory())); // Import data - ImportOpenStreetMap.execute(LIECHTENSTEIN_OSM_PBF, coordinateMap, referenceMap, + ImportOsmPbf.execute(LIECHTENSTEIN_OSM_PBF, coordinateMap, referenceMap, headerRepository, nodeRepository, wayRepository, relationRepository, 3857); diff --git a/basemap/daylight/workflow.js b/basemap/daylight/workflow.js index 258dd3f1..abc12550 100644 --- a/basemap/daylight/workflow.js +++ b/basemap/daylight/workflow.js @@ -26,7 +26,7 @@ export default { // "path": "data/data.osm.pbf" // }, // { - // "type": "ImportOpenStreetMap", + // "type": "ImportOsmPbf", // "file": "data/data.osm.pbf", // "database": config.database, // "databaseSrid": 3857 diff --git a/basemap/workflow.js b/basemap/workflow.js index 5ff3b1ec..80eea670 100644 --- a/basemap/workflow.js +++ b/basemap/workflow.js @@ -122,7 +122,7 @@ export default { "path": "data/data.osm.pbf" }, { - "type": "ImportOpenStreetMap", + "type": "ImportOsmPbf", "file": "data/data.osm.pbf", "database": config.database, "databaseSrid": 3857 diff --git a/examples/extrusion/workflow.json b/examples/extrusion/workflow.json index de11af58..039ad0d1 100644 --- a/examples/extrusion/workflow.json +++ b/examples/extrusion/workflow.json @@ -18,7 +18,7 @@ ], "tasks": [ { - "type": "ImportOpenStreetMap", + "type": "ImportOsmPbf", "file": "greater-london-latest.osm.pbf", "database": "jdbc:postgresql://localhost:5432/baremaps?&user=baremaps&password=baremaps", "databaseSrid": 3857 diff --git a/examples/openstreetmap/workflow.json b/examples/openstreetmap/workflow.json index 9ebd677a..d488cb4b 100644 --- a/examples/openstreetmap/workflow.json +++ b/examples/openstreetmap/workflow.json @@ -18,7 +18,7 @@ ], "tasks": [ { - "type": "ImportOpenStreetMap", + "type": "ImportOsmPbf", "file": "liechtenstein-latest.osm.pbf", "database": "jdbc:postgresql://localhost:5432/baremaps?&user=baremaps&password=baremaps", "databaseSrid": 3857
