This is an automated email from the ASF dual-hosted git repository. bchapuis pushed a commit to branch 745-daylight in repository https://gitbox.apache.org/repos/asf/incubator-baremaps.git
commit 05c2ae01e2afc91660b8d78934f809f19e0d4feb Author: Bertil Chapuis <[email protected]> AuthorDate: Sat Aug 26 14:20:42 2023 +0200 Create inital workflow for daylight --- .../java/org/apache/baremaps/workflow/Task.java | 1 + .../baremaps/workflow/tasks/DecompressBZip2.java | 41 +++ basemap/daylight.js | 408 +++++++++++++++++++++ 3 files changed, 450 insertions(+) diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/Task.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/Task.java index 901e130d..013df2bb 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/Task.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/Task.java @@ -38,6 +38,7 @@ import org.apache.baremaps.workflow.tasks.*; @JsonSubTypes.Type(value = LogMessage.class, name = "LogMessage"), @JsonSubTypes.Type(value = UnzipFile.class, name = "UnzipFile"), @JsonSubTypes.Type(value = UngzipFile.class, name = "UngzipFile"), + @JsonSubTypes.Type(value = DecompressBZip2.class, name = "DecompressBZip2"), @JsonSubTypes.Type(value = UpdateOpenStreetMap.class, name = "UpdateOpenStreetMap"), @JsonSubTypes.Type(value = CreateGeonamesIndex.class, name = "CreateGeonamesIndex"), @JsonSubTypes.Type(value = CreateIplocIndex.class, name = "CreateIplocIndex")}) diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/DecompressBZip2.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/DecompressBZip2.java new file mode 100644 index 00000000..3eb95012 --- /dev/null +++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/DecompressBZip2.java @@ -0,0 +1,41 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + */ + +package org.apache.baremaps.workflow.tasks; + +import java.io.BufferedInputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import org.apache.baremaps.workflow.Task; +import org.apache.baremaps.workflow.WorkflowContext; +import org.apache.baremaps.workflow.WorkflowException; +import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream; + +public record DecompressBZip2(Path source, Path target) implements Task { + + @Override + public void execute(WorkflowContext context) throws Exception { + var sourcePath = source.toAbsolutePath(); + try (var zis = + new BZip2CompressorInputStream(new BufferedInputStream(Files.newInputStream(sourcePath)))) { + var targetPath = target.toAbsolutePath(); + if (!Files.exists(targetPath)) { + Files.createDirectories(targetPath.getParent()); + Files.createFile(targetPath); + } + Files.copy(zis, targetPath, StandardCopyOption.REPLACE_EXISTING); + } catch (Exception e) { + throw new WorkflowException(e); + } + } +} diff --git a/basemap/daylight.js b/basemap/daylight.js new file mode 100644 index 00000000..008336f9 --- /dev/null +++ b/basemap/daylight.js @@ -0,0 +1,408 @@ +/** + Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + in compliance with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software distributed under the License + is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + or implied. See the License for the specific language governing permissions and limitations under + the License. + **/ + +let config = { + "database": "jdbc:postgresql://localhost:5432/daylight?user=daylight&password=daylight" +}; + +export default { + "steps": [ + { + "id": "openstreetmap-data", + "needs": [], + "tasks": [ + { + "type": "DownloadUrl", + "url": "https://daylight-map-distribution.s3.us-west-1.amazonaws.com/release/v1.29/planet-v1.29.osm.pbf";, + "path": "data/data.osm.pbf" + }, + { + "type": "ImportOpenStreetMap", + "file": "data/data.osm.pbf", + "database": config.database, + "databaseSrid": 3857 + }, + ] + }, + { + "id": "openstreetmap-roads", + "needs": ["openstreetmap-data"], + "tasks": [ + { + "type": "DownloadUrl", + "url": "https://daylight-map-distribution.s3.us-west-1.amazonaws.com/release/v1.29/fb-ml-roads-v1.29.osc.bz2";, + "path": "data/roads.osc.bz2" + }, + { + "type": "DecompressBZip2", + "file": "data/roads.osc.bz2", + "directory": "data/roads.osc" + }, + ] + }, + { + "id": "openstreetmap-admin", + "needs": ["openstreetmap-data"], + "tasks": [ + { + "type": "DownloadUrl", + "url": "https://daylight-map-distribution.s3.us-west-1.amazonaws.com/release/v1.29/admin-v1.29.osc.bz2";, + "path": "data/admin.osc.bz2" + }, + { + "type": "DecompressBZip2", + "file": "data/admin.osc.bz2", + "directory": "data/admin.osc" + }, + ] + }, + { + "id": "openstreetmap-coastlines", + "needs": ["openstreetmap-data"], + "tasks": [ + { + "type": "DownloadUrl", + "url": "https://daylight-map-distribution.s3.us-west-1.amazonaws.com/release/v1.29/coastlines-v1.29.tgz";, + "path": "data/coastlines.osc.bz2" + }, + { + "type": "DecompressBZip2", + "file": "data/coastlines.osc.bz2", + "directory": "data/coastlines.osc" + }, + ] + }, + { + "id": "openstreetmap-preferred-localization", + "needs": ["openstreetmap-data"], + "tasks": [ + { + "type": "DownloadUrl", + "url": "https://daylight-map-distribution.s3.us-west-1.amazonaws.com/release/v1.29/preferred-localization-v1.29.tsv";, + "path": "data/preferred-localization.tsv" + }, + ] + }, + { + "id": "openstreetmap-important-features", + "needs": ["openstreetmap-data"], + "tasks": [ + { + "type": "DownloadUrl", + "url": "https://daylight-map-distribution.s3.us-west-1.amazonaws.com/release/v1.29/important-features-v1.29.json";, + "path": "data/important-features.json" + }, + ] + }, + + // { + // "id": "openstreetmap-nodes", + // "needs": [ + // "openstreetmap-data" + // ], + // "tasks": [ + // { + // "type": "ExecuteSql", + // "file": "queries/osm_nodes.sql", + // "database": config.database, + // "parallel": true, + // }, + // ] + // }, + // { + // "id": "openstreetmap-ways", + // "needs": [ + // "openstreetmap-data" + // ], + // "tasks": [ + // { + // "type": "ExecuteSql", + // "file": "queries/osm_ways.sql", + // "database": config.database, + // "parallel": true, + // }, + // ] + // }, + // { + // "id": "openstreetmap-relations", + // "needs": [ + // "openstreetmap-data" + // ], + // "tasks": [ + // { + // "type": "ExecuteSql", + // "file": "queries/osm_relations.sql", + // "database": config.database, + // "parallel": true, + // }, + // ] + // }, + // { + // "id": "openstreetmap-member", + // "needs": [ + // "openstreetmap-data" + // ], + // "tasks": [ + // { + // "type": "ExecuteSql", + // "file": "layers/member/prepare.sql", + // "database": config.database, + // }, + // ] + // }, + // { + // "id": "openstreetmap-point", + // "needs": [ + // "openstreetmap-nodes" + // ], + // "tasks": [ + // { + // "type": "ExecuteSql", + // "file": "layers/point/clean.sql", + // "database": config.database, + // }, + // { + // "type": "ExecuteSql", + // "file": "layers/point/simplify.sql", + // "database": config.database, + // "parallel": true, + // }, + // { + // "type": "ExecuteSql", + // "file": "layers/point/index.sql", + // "database": config.database, + // "parallel": true, + // }, + // ] + // }, + // { + // "id": "openstreetmap-linestring", + // "needs": [ + // "openstreetmap-member" + // ], + // "tasks": [ + // { + // "type": "ExecuteSql", + // "file": "layers/linestring/clean.sql", + // "database": config.database, + // }, + // { + // "type": "ExecuteSql", + // "file": "layers/linestring/prepare.sql", + // "database": config.database, + // }, + // { + // "type": "ExecuteSql", + // "file": "layers/linestring/index.sql", + // "database": config.database, + // }, + // ] + // }, + // { + // "id": "openstreetmap-polygon", + // "needs": [ + // "openstreetmap-member", + // ], + // "tasks": [ + // { + // "type": "ExecuteSql", + // "file": "layers/polygon/clean.sql", + // "database": config.database, + // }, + // { + // "type": "ExecuteSql", + // "file": "layers/polygon/prepare.sql", + // "database": config.database, + // }, + // { + // "type": "ExecuteSql", + // "file": "layers/polygon/index.sql", + // "database": config.database, + // }, + // ] + // }, + // { + // "id": "openstreetmap-highway", + // "needs": [ + // "openstreetmap-linestring" + // ], + // "tasks": [ + // { + // "type": "ExecuteSql", + // "file": "layers/highway/clean.sql", + // "database": config.database, + // }, + // { + // "type": "ExecuteSql", + // "file": "layers/highway/prepare.sql", + // "database": config.database, + // }, + // { + // "type": "ExecuteSql", + // "file": "layers/highway/simplify.sql", + // "database": config.database, + // "parallel": true, + // }, + // { + // "type": "ExecuteSql", + // "file": "layers/highway/index.sql", + // "database": config.database, + // "parallel": true, + // }, + // ] + // }, + // { + // "id": "openstreetmap-railway", + // "needs": ["openstreetmap-linestring"], + // "tasks": [ + // { + // "type": "ExecuteSql", + // "file": "layers/railway/clean.sql", + // "database": config.database, + // }, + // { + // "type": "ExecuteSql", + // "file": "layers/railway/prepare.sql", + // "database": config.database, + // }, + // { + // "type": "ExecuteSql", + // "file": "layers/railway/simplify.sql", + // "database": config.database, + // "parallel": true, + // }, + // { + // "type": "ExecuteSql", + // "file": "layers/railway/index.sql", + // "database": config.database, + // "parallel": true, + // }, + // ] + // }, + // { + // "id": "openstreetmap-route", + // "needs": ["openstreetmap-linestring"], + // "tasks": [ + // { + // "type": "ExecuteSql", + // "file": "layers/route/clean.sql", + // "database": config.database, + // }, + // { + // "type": "ExecuteSql", + // "file": "layers/route/prepare.sql", + // "database": config.database, + // }, + // { + // "type": "ExecuteSql", + // "file": "layers/route/simplify.sql", + // "database": config.database, + // "parallel": true, + // }, + // { + // "type": "ExecuteSql", + // "file": "layers/route/index.sql", + // "database": config.database, + // "parallel": true, + // }, + // ] + // }, + // { + // "id": "openstreetmap-natural", + // "needs": ["openstreetmap-polygon"], + // "tasks": [ + // { + // "type": "ExecuteSql", + // "file": "layers/natural/clean.sql", + // "database": config.database, + // }, + // { + // "type": "ExecuteSql", + // "file": "layers/natural/prepare.sql", + // "database": config.database, + // }, + // { + // "type": "ExecuteSql", + // "file": "layers/natural/simplify.sql", + // "database": config.database, + // "parallel": true, + // }, + // { + // "type": "ExecuteSql", + // "file": "layers/natural/index.sql", + // "database": config.database, + // "parallel": true + // }, + // ] + // }, + // { + // "id": "openstreetmap-landuse", + // "needs": [ + // "openstreetmap-polygon" + // ], + // "tasks": [ + // { + // "type": "ExecuteSql", + // "file": "layers/landuse/clean.sql", + // "database": config.database, + // }, + // { + // "type": "ExecuteSql", + // "file": "layers/landuse/prepare.sql", + // "database": config.database, + // }, + // { + // "type": "ExecuteSql", + // "file": "layers/landuse/simplify.sql", + // "database": config.database, + // "parallel": true, + // }, + // { + // "type": "ExecuteSql", + // "file": "layers/landuse/index.sql", + // "database": config.database, + // "parallel": true + // }, + // ] + // }, + // { + // "id": "openstreetmap-waterway", + // "needs": [ + // "openstreetmap-linestring" + // ], + // "tasks": [ + // { + // "type": "ExecuteSql", + // "file": "layers/waterway/clean.sql", + // "database": config.database, + // }, + // { + // "type": "ExecuteSql", + // "file": "layers/waterway/prepare.sql", + // "database": config.database, + // }, + // { + // "type": "ExecuteSql", + // "file": "layers/waterway/simplify.sql", + // "database": config.database, + // "parallel": true, + // }, + // { + // "type": "ExecuteSql", + // "file": "layers/waterway/index.sql", + // "database": config.database, + // "parallel": true + // }, + // ] + // }, + ] +}
