This is an automated email from the ASF dual-hosted git repository. bchapuis pushed a commit to branch stream-exception in repository https://gitbox.apache.org/repos/asf/incubator-baremaps.git
commit fe8eec97d73f33fc52239865410072fadf899c5f Author: Bertil Chapuis <[email protected]> AuthorDate: Sun Nov 26 13:51:28 2023 +0100 Add support for PMTiles (#794) Enable the export of vector tiles in the PMTiles format. Most of the necessary functions have been ported from [go-pmtiles](https://github.com/protomaps/go-pmtiles) to Java. --- .run/basemap-dev.run.xml | 7 +- ...-dev.run.xml => basemap-export-mbtiles.run.xml} | 4 +- ...-dev.run.xml => basemap-export-pmtiles.run.xml} | 4 +- .run/basemap-workflow.run.xml | 2 +- .../baremaps/stream/BufferedSpliterator.java | 2 +- .../org/apache/baremaps/stream/StreamUtils.java | 1 - .../org/apache/baremaps/tilestore/TileCache.java | 6 + .../org/apache/baremaps/tilestore/TileStore.java | 2 +- .../baremaps/tilestore/TileStoreException.java | 10 + .../baremaps/tilestore/file/FileTileStore.java | 5 + .../baremaps/tilestore/mbtiles/MBTilesStore.java | 5 + .../baremaps/tilestore/pmtiles/Compression.java | 74 ++++ .../baremaps/tilestore/pmtiles/Directories.java} | 31 +- .../apache/baremaps/tilestore/pmtiles/Entry.java | 66 ++++ .../apache/baremaps/tilestore/pmtiles/Header.java | 341 ++++++++++++++++++ .../apache/baremaps/tilestore/pmtiles/PMTiles.java | 391 +++++++++++++++++++++ .../baremaps/tilestore/pmtiles/PMTilesReader.java | 95 +++++ .../baremaps/tilestore/pmtiles/PMTilesStore.java | 95 +++++ .../baremaps/tilestore/pmtiles/PMTilesWriter.java | 208 +++++++++++ .../baremaps/tilestore/pmtiles/TileType.java} | 23 +- .../tilestore/postgres/PostgresTileStore.java | 20 +- .../baremaps/workflow/tasks/ExportVectorTiles.java | 79 +++-- .../apache/baremaps/stream/StreamUtilsTest.java | 42 +++ .../baremaps/tilestore/pmtiles/PMTilesTest.java | 288 +++++++++++++++ .../tilestore/postgres/PostgresTileStoreTest.java | 2 +- .../baremaps/vectortile/VectorTileViewer.java | 2 +- .../src/test/resources/pmtiles/empty.pmtiles | 0 .../src/test/resources/pmtiles/invalid.pmtiles | 1 + .../src/test/resources/pmtiles/invalid_v4.pmtiles | Bin 0 -> 468 bytes .../test/resources/pmtiles/test_fixture_1.pmtiles | Bin 0 -> 468 bytes .../test/resources/pmtiles/test_fixture_2.pmtiles | Bin 0 -> 466 bytes basemap/import.js | 16 + basemap/layers/building/extrusion.js | 9 - basemap/layers/building/tileset.js | 68 ++-- basemap/queries/assertions.sql | 48 +++ basemap/queries/functions.sql | 49 +++ 36 files changed, 1879 insertions(+), 117 deletions(-) diff --git a/.run/basemap-dev.run.xml b/.run/basemap-dev.run.xml index 4b61bfcd..af9438a4 100644 --- a/.run/basemap-dev.run.xml +++ b/.run/basemap-dev.run.xml @@ -2,8 +2,13 @@ <configuration default="false" name="basemap-dev" type="Application" factoryName="Application"> <option name="MAIN_CLASS_NAME" value="org.apache.baremaps.cli.Baremaps" /> <module name="baremaps-cli" /> - <option name="PROGRAM_PARAMETERS" value="map dev --tileset tileset.js --style style.js" /> + <option name="PROGRAM_PARAMETERS" value="map dev --tileset tileset.js --style style.js --log-level DEBUG" /> <option name="WORKING_DIRECTORY" value="$PROJECT_DIR$/basemap" /> + <extension name="software.aws.toolkits.jetbrains.core.execution.JavaAwsConnectionExtension"> + <option name="credential" /> + <option name="region" /> + <option name="useCurrentConnection" value="false" /> + </extension> <method v="2"> <option name="Make" enabled="true" /> </method> diff --git a/.run/basemap-dev.run.xml b/.run/basemap-export-mbtiles.run.xml similarity index 57% copy from .run/basemap-dev.run.xml copy to .run/basemap-export-mbtiles.run.xml index 4b61bfcd..d04d8a2a 100644 --- a/.run/basemap-dev.run.xml +++ b/.run/basemap-export-mbtiles.run.xml @@ -1,8 +1,8 @@ <component name="ProjectRunConfigurationManager"> - <configuration default="false" name="basemap-dev" type="Application" factoryName="Application"> + <configuration default="false" name="basemap-export-mbtiles" type="Application" factoryName="Application"> <option name="MAIN_CLASS_NAME" value="org.apache.baremaps.cli.Baremaps" /> <module name="baremaps-cli" /> - <option name="PROGRAM_PARAMETERS" value="map dev --tileset tileset.js --style style.js" /> + <option name="PROGRAM_PARAMETERS" value="map export --tileset tileset.js --repository tiles.mbtiles --format mbtiles" /> <option name="WORKING_DIRECTORY" value="$PROJECT_DIR$/basemap" /> <method v="2"> <option name="Make" enabled="true" /> diff --git a/.run/basemap-dev.run.xml b/.run/basemap-export-pmtiles.run.xml similarity index 57% copy from .run/basemap-dev.run.xml copy to .run/basemap-export-pmtiles.run.xml index 4b61bfcd..840a4774 100644 --- a/.run/basemap-dev.run.xml +++ b/.run/basemap-export-pmtiles.run.xml @@ -1,8 +1,8 @@ <component name="ProjectRunConfigurationManager"> - <configuration default="false" name="basemap-dev" type="Application" factoryName="Application"> + <configuration default="false" name="basemap-export-pmtiles" type="Application" factoryName="Application"> <option name="MAIN_CLASS_NAME" value="org.apache.baremaps.cli.Baremaps" /> <module name="baremaps-cli" /> - <option name="PROGRAM_PARAMETERS" value="map dev --tileset tileset.js --style style.js" /> + <option name="PROGRAM_PARAMETERS" value="map export --tileset tileset.js --repository tiles.mbtiles --format pmtiles" /> <option name="WORKING_DIRECTORY" value="$PROJECT_DIR$/basemap" /> <method v="2"> <option name="Make" enabled="true" /> diff --git a/.run/basemap-workflow.run.xml b/.run/basemap-workflow.run.xml index f0fde5c2..663c874a 100644 --- a/.run/basemap-workflow.run.xml +++ b/.run/basemap-workflow.run.xml @@ -2,7 +2,7 @@ <configuration default="false" name="basemap-workflow" type="Application" factoryName="Application"> <option name="MAIN_CLASS_NAME" value="org.apache.baremaps.cli.Baremaps" /> <module name="baremaps-cli" /> - <option name="PROGRAM_PARAMETERS" value="workflow execute --file workflow.js" /> + <option name="PROGRAM_PARAMETERS" value="workflow execute --file import.js" /> <option name="WORKING_DIRECTORY" value="$PROJECT_DIR$/basemap" /> <method v="2"> <option name="Make" enabled="true" /> diff --git a/baremaps-core/src/main/java/org/apache/baremaps/stream/BufferedSpliterator.java b/baremaps-core/src/main/java/org/apache/baremaps/stream/BufferedSpliterator.java index 3a520933..6cc33854 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/stream/BufferedSpliterator.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/stream/BufferedSpliterator.java @@ -127,7 +127,7 @@ class BufferedSpliterator<T> implements Spliterator<CompletableFuture<T>> { @Override public <T> void registerCompletion(CompletableFuture<T> future, Consumer<CompletableFuture<T>> resultConsumer) { - future.thenAccept(result -> resultConsumer.accept(future)); + future.whenComplete((result, error) -> resultConsumer.accept(future)); } } diff --git a/baremaps-core/src/main/java/org/apache/baremaps/stream/StreamUtils.java b/baremaps-core/src/main/java/org/apache/baremaps/stream/StreamUtils.java index 349a2c06..707fd198 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/stream/StreamUtils.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/stream/StreamUtils.java @@ -152,7 +152,6 @@ public class StreamUtils { try { return f.get(); } catch (InterruptedException | ExecutionException e) { - Thread.currentThread().interrupt(); throw new StreamException(e); } }); diff --git a/baremaps-core/src/main/java/org/apache/baremaps/tilestore/TileCache.java b/baremaps-core/src/main/java/org/apache/baremaps/tilestore/TileCache.java index 7381b6e9..199734a7 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/tilestore/TileCache.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/tilestore/TileCache.java @@ -84,4 +84,10 @@ public class TileCache implements TileStore { tileStore.delete(tileCoord); cache.invalidate(tileCoord); } + + @Override + public void close() throws Exception { + tileStore.close(); + cache.cleanUp(); + } } diff --git a/baremaps-core/src/main/java/org/apache/baremaps/tilestore/TileStore.java b/baremaps-core/src/main/java/org/apache/baremaps/tilestore/TileStore.java index c0332150..e7f34b11 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/tilestore/TileStore.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/tilestore/TileStore.java @@ -24,7 +24,7 @@ import java.util.ArrayList; import java.util.List; /** Represents a store for tiles. */ -public interface TileStore { +public interface TileStore extends AutoCloseable { /** * Reads the content of a tile. diff --git a/baremaps-core/src/main/java/org/apache/baremaps/tilestore/TileStoreException.java b/baremaps-core/src/main/java/org/apache/baremaps/tilestore/TileStoreException.java index d67bc50c..1ad72e1e 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/tilestore/TileStoreException.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/tilestore/TileStoreException.java @@ -37,4 +37,14 @@ public class TileStoreException extends Exception { public TileStoreException(Throwable cause) { super(cause); } + + /** + * Constructs a {@code BlobStoreException} with the specified detail message and cause. + * + * @param message the message + * @param cause the cause + */ + public TileStoreException(String message, Throwable cause) { + super(message, cause); + } } diff --git a/baremaps-core/src/main/java/org/apache/baremaps/tilestore/file/FileTileStore.java b/baremaps-core/src/main/java/org/apache/baremaps/tilestore/file/FileTileStore.java index 41dd412b..5afcdafa 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/tilestore/file/FileTileStore.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/tilestore/file/FileTileStore.java @@ -77,4 +77,9 @@ public class FileTileStore implements TileStore { public Path resolve(TileCoord tileCoord) { return path.resolve(String.format("%s/%s/%s.mvt", tileCoord.z(), tileCoord.x(), tileCoord.y())); } + + @Override + public void close() { + // Do nothing + } } diff --git a/baremaps-core/src/main/java/org/apache/baremaps/tilestore/mbtiles/MBTilesStore.java b/baremaps-core/src/main/java/org/apache/baremaps/tilestore/mbtiles/MBTilesStore.java index b6560778..9787cbeb 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/tilestore/mbtiles/MBTilesStore.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/tilestore/mbtiles/MBTilesStore.java @@ -219,4 +219,9 @@ public class MBTilesStore implements TileStore { private static int reverseY(int y, int z) { return (int) (Math.pow(2.0, z) - 1 - y); } + + @Override + public void close() throws Exception { + // Do nothing + } } diff --git a/baremaps-core/src/main/java/org/apache/baremaps/tilestore/pmtiles/Compression.java b/baremaps-core/src/main/java/org/apache/baremaps/tilestore/pmtiles/Compression.java new file mode 100644 index 00000000..30da86fa --- /dev/null +++ b/baremaps-core/src/main/java/org/apache/baremaps/tilestore/pmtiles/Compression.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.baremaps.tilestore.pmtiles; + +import java.io.*; +import java.util.zip.GZIPInputStream; +import java.util.zip.GZIPOutputStream; + +enum Compression { + Unknown, + None, + Gzip, + Brotli, + Zstd; + + InputStream decompress(InputStream inputStream) throws IOException { + return switch (this) { + case None -> inputStream; + case Gzip -> decompressGzip(inputStream); + case Brotli -> decompressBrotli(inputStream); + case Zstd -> decompressZstd(inputStream); + default -> throw new RuntimeException("Unknown compression"); + }; + } + + static InputStream decompressGzip(InputStream inputStream) throws IOException { + return new GZIPInputStream(inputStream); + } + + static InputStream decompressBrotli(InputStream buffer) { + throw new RuntimeException("Brotli compression not implemented"); + } + + static InputStream decompressZstd(InputStream buffer) { + throw new RuntimeException("Zstd compression not implemented"); + } + + OutputStream compress(OutputStream outputStream) throws IOException { + return switch (this) { + case None -> outputStream; + case Gzip -> compressGzip(outputStream); + case Brotli -> compressBrotli(outputStream); + case Zstd -> compressZstd(outputStream); + default -> throw new RuntimeException("Unknown compression"); + }; + } + + static OutputStream compressGzip(OutputStream outputStream) throws IOException { + return new GZIPOutputStream(outputStream); + } + + static OutputStream compressBrotli(OutputStream outputStream) { + throw new RuntimeException("Brotli compression not implemented"); + } + + static OutputStream compressZstd(OutputStream outputStream) { + throw new RuntimeException("Zstd compression not implemented"); + } +} diff --git a/baremaps-core/src/test/java/org/apache/baremaps/stream/StreamUtilsTest.java b/baremaps-core/src/main/java/org/apache/baremaps/tilestore/pmtiles/Directories.java similarity index 63% copy from baremaps-core/src/test/java/org/apache/baremaps/stream/StreamUtilsTest.java copy to baremaps-core/src/main/java/org/apache/baremaps/tilestore/pmtiles/Directories.java index ee1cc354..ea91b1fa 100644 --- a/baremaps-core/src/test/java/org/apache/baremaps/stream/StreamUtilsTest.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/tilestore/pmtiles/Directories.java @@ -15,20 +15,29 @@ * limitations under the License. */ -package org.apache.baremaps.stream; +package org.apache.baremaps.tilestore.pmtiles; -import static org.junit.jupiter.api.Assertions.assertEquals; +class Directories { -import java.util.List; -import java.util.stream.IntStream; -import org.junit.jupiter.api.Test; + private final byte[] root; + private final byte[] leaves; + private final int numLeaves; -class StreamUtilsTest { + public Directories(byte[] root, byte[] leaves, int numLeaves) { + this.root = root; + this.leaves = leaves; + this.numLeaves = numLeaves; + } + + public byte[] getRoot() { + return root; + } + + public byte[] getLeaves() { + return leaves; + } - @Test - void partition() { - List<Integer> list = IntStream.range(0, 100).boxed().toList(); - List<List<Integer>> partitions = StreamUtils.partition(list.stream(), 10).toList(); - assertEquals(partitions.size(), 10); + public int getNumLeaves() { + return numLeaves; } } diff --git a/baremaps-core/src/main/java/org/apache/baremaps/tilestore/pmtiles/Entry.java b/baremaps-core/src/main/java/org/apache/baremaps/tilestore/pmtiles/Entry.java new file mode 100644 index 00000000..a8b0947e --- /dev/null +++ b/baremaps-core/src/main/java/org/apache/baremaps/tilestore/pmtiles/Entry.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.baremaps.tilestore.pmtiles; + +class Entry { + private long tileId; + private long offset; + private long length; + private long runLength; + + public Entry() {} + + public Entry(long tileId, long offset, long length, long runLength) { + this.tileId = tileId; + this.offset = offset; + this.length = length; + this.runLength = runLength; + } + + public long getTileId() { + return tileId; + } + + public void setTileId(long tileId) { + this.tileId = tileId; + } + + public long getOffset() { + return offset; + } + + public void setOffset(long offset) { + this.offset = offset; + } + + public long getLength() { + return length; + } + + public void setLength(long length) { + this.length = length; + } + + public long getRunLength() { + return runLength; + } + + public void setRunLength(long runLength) { + this.runLength = runLength; + } +} diff --git a/baremaps-core/src/main/java/org/apache/baremaps/tilestore/pmtiles/Header.java b/baremaps-core/src/main/java/org/apache/baremaps/tilestore/pmtiles/Header.java new file mode 100644 index 00000000..b1e3bddf --- /dev/null +++ b/baremaps-core/src/main/java/org/apache/baremaps/tilestore/pmtiles/Header.java @@ -0,0 +1,341 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.baremaps.tilestore.pmtiles; + +import java.util.Objects; + +class Header { + + private int specVersion; + private long rootDirectoryOffset; + private long rootDirectoryLength; + private long jsonMetadataOffset; + private long jsonMetadataLength; + private long leafDirectoryOffset; + private long leafDirectoryLength; + private long tileDataOffset; + private long tileDataLength; + private long numAddressedTiles; + private long numTileEntries; + private long numTileContents; + private boolean clustered; + private Compression internalCompression; + private Compression tileCompression; + private TileType tileType; + private int minZoom; + private int maxZoom; + private double minLon; + private double minLat; + private double maxLon; + private double maxLat; + private int centerZoom; + private double centerLon; + private double centerLat; + + public Header() { + this.specVersion = 3; + } + + public Header( + int specVersion, + long rootDirectoryOffset, + long rootDirectoryLength, + long jsonMetadataOffset, + long jsonMetadataLength, + long leafDirectoryOffset, + long leafDirectoryLength, + long tileDataOffset, + long tileDataLength, + long numAddressedTiles, + long numTileEntries, + long numTileContents, + boolean clustered, + Compression internalCompression, + Compression tileCompression, + TileType tileType, + int minZoom, + int maxZoom, + double minLon, + double minLat, double maxLon, + double maxLat, + int centerZoom, + double centerLon, + double centerLat) { + this.specVersion = specVersion; + this.rootDirectoryOffset = rootDirectoryOffset; + this.rootDirectoryLength = rootDirectoryLength; + this.jsonMetadataOffset = jsonMetadataOffset; + this.jsonMetadataLength = jsonMetadataLength; + this.leafDirectoryOffset = leafDirectoryOffset; + this.leafDirectoryLength = leafDirectoryLength; + this.tileDataOffset = tileDataOffset; + this.tileDataLength = tileDataLength; + this.numAddressedTiles = numAddressedTiles; + this.numTileEntries = numTileEntries; + this.numTileContents = numTileContents; + this.clustered = clustered; + this.internalCompression = internalCompression; + this.tileCompression = tileCompression; + this.tileType = tileType; + this.minZoom = minZoom; + this.maxZoom = maxZoom; + this.minLon = minLon; + this.minLat = minLat; + this.maxLon = maxLon; + this.maxLat = maxLat; + this.centerZoom = centerZoom; + this.centerLon = centerLon; + this.centerLat = centerLat; + } + + public int getSpecVersion() { + return specVersion; + } + + public void setSpecVersion(int specVersion) { + this.specVersion = specVersion; + } + + public long getRootDirectoryOffset() { + return rootDirectoryOffset; + } + + public void setRootOffset(long rootDirectoryOffset) { + this.rootDirectoryOffset = rootDirectoryOffset; + } + + public long getRootDirectoryLength() { + return rootDirectoryLength; + } + + public void setRootLength(long rootDirectoryLength) { + this.rootDirectoryLength = rootDirectoryLength; + } + + public long getJsonMetadataOffset() { + return jsonMetadataOffset; + } + + public void setMetadataOffset(long jsonMetadataOffset) { + this.jsonMetadataOffset = jsonMetadataOffset; + } + + public long getJsonMetadataLength() { + return jsonMetadataLength; + } + + public void setMetadataLength(long jsonMetadataLength) { + this.jsonMetadataLength = jsonMetadataLength; + } + + public long getLeafDirectoryOffset() { + return leafDirectoryOffset; + } + + public void setLeavesOffset(long leafDirectoryOffset) { + this.leafDirectoryOffset = leafDirectoryOffset; + } + + public long getLeafDirectoryLength() { + return leafDirectoryLength; + } + + public void setLeavesLength(long leafDirectoryLength) { + this.leafDirectoryLength = leafDirectoryLength; + } + + public long getTileDataOffset() { + return tileDataOffset; + } + + public void setTilesOffset(long tileDataOffset) { + this.tileDataOffset = tileDataOffset; + } + + public long getTileDataLength() { + return tileDataLength; + } + + public void setTilesLength(long tileDataLength) { + this.tileDataLength = tileDataLength; + } + + public long getNumAddressedTiles() { + return numAddressedTiles; + } + + public void setNumAddressedTiles(long numAddressedTiles) { + this.numAddressedTiles = numAddressedTiles; + } + + public long getNumTileEntries() { + return numTileEntries; + } + + public void setNumTileEntries(long numTileEntries) { + this.numTileEntries = numTileEntries; + } + + public long getNumTileContents() { + return numTileContents; + } + + public void setNumTileContents(long numTileContents) { + this.numTileContents = numTileContents; + } + + public boolean isClustered() { + return clustered; + } + + public void setClustered(boolean clustered) { + this.clustered = clustered; + } + + public Compression getInternalCompression() { + return internalCompression; + } + + public void setInternalCompression(Compression internalCompression) { + this.internalCompression = internalCompression; + } + + public Compression getTileCompression() { + return tileCompression; + } + + public void setTileCompression(Compression tileCompression) { + this.tileCompression = tileCompression; + } + + public TileType getTileType() { + return tileType; + } + + public void setTileType(TileType tileType) { + this.tileType = tileType; + } + + public int getMinZoom() { + return minZoom; + } + + public void setMinZoom(int minZoom) { + this.minZoom = minZoom; + } + + public int getMaxZoom() { + return maxZoom; + } + + public void setMaxZoom(int maxZoom) { + this.maxZoom = maxZoom; + } + + public double getMinLon() { + return minLon; + } + + public void setMinLon(double minLon) { + this.minLon = minLon; + } + + public double getMinLat() { + return minLat; + } + + public void setMinLat(double minLat) { + this.minLat = minLat; + } + + public double getMaxLon() { + return maxLon; + } + + public void setMaxLon(double maxLon) { + this.maxLon = maxLon; + } + + public double getMaxLat() { + return maxLat; + } + + public void setMaxLat(double maxLat) { + this.maxLat = maxLat; + } + + public int getCenterZoom() { + return centerZoom; + } + + public void setCenterZoom(int centerZoom) { + this.centerZoom = centerZoom; + } + + public double getCenterLon() { + return centerLon; + } + + public void setCenterLon(double centerLon) { + this.centerLon = centerLon; + } + + public double getCenterLat() { + return centerLat; + } + + public void setCenterLat(double centerLat) { + this.centerLat = centerLat; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + Header header = (Header) o; + return specVersion == header.specVersion && rootDirectoryOffset == header.rootDirectoryOffset + && rootDirectoryLength == header.rootDirectoryLength + && jsonMetadataOffset == header.jsonMetadataOffset + && jsonMetadataLength == header.jsonMetadataLength + && leafDirectoryOffset == header.leafDirectoryOffset + && leafDirectoryLength == header.leafDirectoryLength + && tileDataOffset == header.tileDataOffset && tileDataLength == header.tileDataLength + && numAddressedTiles == header.numAddressedTiles && numTileEntries == header.numTileEntries + && numTileContents == header.numTileContents && clustered == header.clustered + && minZoom == header.minZoom && maxZoom == header.maxZoom + && Double.compare(header.minLon, minLon) == 0 && Double.compare(header.minLat, minLat) == 0 + && Double.compare(header.maxLon, maxLon) == 0 && Double.compare(header.maxLat, maxLat) == 0 + && centerZoom == header.centerZoom && Double.compare(header.centerLon, centerLon) == 0 + && Double.compare(header.centerLat, centerLat) == 0 + && internalCompression == header.internalCompression + && tileCompression == header.tileCompression && tileType == header.tileType; + } + + @Override + public int hashCode() { + return Objects.hash(specVersion, rootDirectoryOffset, rootDirectoryLength, jsonMetadataOffset, + jsonMetadataLength, leafDirectoryOffset, leafDirectoryLength, tileDataOffset, + tileDataLength, numAddressedTiles, numTileEntries, numTileContents, clustered, + internalCompression, tileCompression, tileType, minZoom, maxZoom, minLon, minLat, maxLon, + maxLat, centerZoom, centerLon, centerLat); + } +} diff --git a/baremaps-core/src/main/java/org/apache/baremaps/tilestore/pmtiles/PMTiles.java b/baremaps-core/src/main/java/org/apache/baremaps/tilestore/pmtiles/PMTiles.java new file mode 100644 index 00000000..e8bf2f30 --- /dev/null +++ b/baremaps-core/src/main/java/org/apache/baremaps/tilestore/pmtiles/PMTiles.java @@ -0,0 +1,391 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.baremaps.tilestore.pmtiles; + +import com.google.common.io.LittleEndianDataInputStream; +import com.google.common.io.LittleEndianDataOutputStream; +import com.google.common.math.LongMath; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +public class PMTiles { + + public static long toNum(long low, long high) { + return high * 0x100000000L + low; + } + + public static long readVarIntRemainder(LittleEndianDataInputStream input, long l) + throws IOException { + long h, b; + b = input.readByte() & 0xff; + h = (b & 0x70) >> 4; + if (b < 0x80) { + return toNum(l, h); + } + b = input.readByte() & 0xff; + h |= (b & 0x7f) << 3; + if (b < 0x80) { + return toNum(l, h); + } + b = input.readByte() & 0xff; + h |= (b & 0x7f) << 10; + if (b < 0x80) { + return toNum(l, h); + } + b = input.readByte() & 0xff; + h |= (b & 0x7f) << 17; + if (b < 0x80) { + return toNum(l, h); + } + b = input.readByte() & 0xff; + h |= (b & 0x7f) << 24; + if (b < 0x80) { + return toNum(l, h); + } + b = input.readByte() & 0xff; + h |= (b & 0x01) << 31; + if (b < 0x80) { + return toNum(l, h); + } + throw new RuntimeException("Expected varint not more than 10 bytes"); + } + + public static int writeVarInt(LittleEndianDataOutputStream output, long value) + throws IOException { + int n = 1; + while (value >= 0x80) { + output.writeByte((byte) (value | 0x80)); + value >>>= 7; + n++; + } + output.writeByte((byte) value); + return n; + } + + public static long readVarInt(LittleEndianDataInputStream input) throws IOException { + long val, b; + b = input.readByte() & 0xff; + val = b & 0x7f; + if (b < 0x80) { + return val; + } + b = input.readByte() & 0xff; + val |= (b & 0x7f) << 7; + if (b < 0x80) { + return val; + } + b = input.readByte() & 0xff; + val |= (b & 0x7f) << 14; + if (b < 0x80) { + return val; + } + b = input.readByte() & 0xff; + val |= (b & 0x7f) << 21; + if (b < 0x80) { + return val; + } + val |= (b & 0x0f) << 28; + return readVarIntRemainder(input, val); + } + + public static void rotate(long n, long[] xy, long rx, long ry) { + if (ry == 0) { + if (rx == 1) { + xy[0] = n - 1 - xy[0]; + xy[1] = n - 1 - xy[1]; + } + long t = xy[0]; + xy[0] = xy[1]; + xy[1] = t; + } + } + + public static long[] idOnLevel(int z, long pos) { + long n = LongMath.pow(2, z); + long rx, ry, t = pos; + long[] xy = new long[] {0, 0}; + long s = 1; + while (s < n) { + rx = 1 & (t / 2); + ry = 1 & (t ^ rx); + rotate(s, xy, rx, ry); + xy[0] += s * rx; + xy[1] += s * ry; + t = t / 4; + s *= 2; + } + return new long[] {z, xy[0], xy[1]}; + } + + private static long[] tzValues = new long[] { + 0, 1, 5, 21, 85, 341, 1365, 5461, 21845, 87381, 349525, 1398101, 5592405, + 22369621, 89478485, 357913941, 1431655765, 5726623061L, 22906492245L, + 91625968981L, 366503875925L, 1466015503701L, 5864062014805L, 23456248059221L, + 93824992236885L, 375299968947541L, 1501199875790165L, + }; + + public static long zxyToTileId(int z, long x, long y) { + if (z > 26) { + throw new RuntimeException("Tile zoom level exceeds max safe number limit (26)"); + } + if (x > Math.pow(2, z) - 1 || y > Math.pow(2, z) - 1) { + throw new RuntimeException("tile x/y outside zoom level bounds"); + } + long acc = tzValues[z]; + long n = LongMath.pow(2, z); + long rx = 0; + long ry = 0; + long d = 0; + long[] xy = new long[] {x, y}; + long s = n / 2; + while (s > 0) { + rx = (xy[0] & s) > 0 ? 1 : 0; + ry = (xy[1] & s) > 0 ? 1 : 0; + d += s * s * ((3 * rx) ^ ry); + rotate(s, xy, rx, ry); + s = s / 2; + } + return acc + d; + } + + public static long[] tileIdToZxy(long i) { + long acc = 0; + for (int z = 0; z < 27; z++) { + long numTiles = (0x1L << z) * (0x1L << z); + if (acc + numTiles > i) { + return idOnLevel(z, i - acc); + } + acc += numTiles; + } + throw new RuntimeException("Tile zoom level exceeds max safe number limit (26)"); + } + + private static final int HEADER_SIZE_BYTES = 127; + + public static Header deserializeHeader(LittleEndianDataInputStream input) throws IOException { + input.skipBytes(7); + return new Header( + input.readByte(), + input.readLong(), + input.readLong(), + input.readLong(), + input.readLong(), + input.readLong(), + input.readLong(), + input.readLong(), + input.readLong(), + input.readLong(), + input.readLong(), + input.readLong(), + input.readByte() == 1, + Compression.values()[input.readByte()], + Compression.values()[input.readByte()], + TileType.values()[input.readByte()], + input.readByte(), + input.readByte(), + (double) input.readInt() / 10000000, + (double) input.readInt() / 10000000, + (double) input.readInt() / 10000000, + (double) input.readInt() / 10000000, + input.readByte(), + (double) input.readInt() / 10000000, + (double) input.readInt() / 10000000); + } + + public static void serializeHeader(LittleEndianDataOutputStream output, Header header) + throws IOException { + output.writeByte((byte) 0x50); + output.writeByte((byte) 0x4D); + output.writeByte((byte) 0x54); + output.writeByte((byte) 0x69); + output.writeByte((byte) 0x6C); + output.writeByte((byte) 0x65); + output.writeByte((byte) 0x73); + output.writeByte((byte) header.getSpecVersion()); + output.writeLong(header.getRootDirectoryOffset()); + output.writeLong(header.getRootDirectoryLength()); + output.writeLong(header.getJsonMetadataOffset()); + output.writeLong(header.getJsonMetadataLength()); + output.writeLong(header.getLeafDirectoryOffset()); + output.writeLong(header.getLeafDirectoryLength()); + output.writeLong(header.getTileDataOffset()); + output.writeLong(header.getTileDataLength()); + output.writeLong(header.getNumAddressedTiles()); + output.writeLong(header.getNumTileEntries()); + output.writeLong(header.getNumTileContents()); + output.writeByte((byte) (header.isClustered() ? 1 : 0)); + output.writeByte((byte) header.getInternalCompression().ordinal()); + output.writeByte((byte) header.getTileCompression().ordinal()); + output.writeByte((byte) header.getTileType().ordinal()); + output.writeByte((byte) header.getMinZoom()); + output.writeByte((byte) header.getMaxZoom()); + output.writeInt((int) (header.getMinLon() * 10000000)); + output.writeInt((int) (header.getMinLat() * 10000000)); + output.writeInt((int) (header.getMaxLon() * 10000000)); + output.writeInt((int) (header.getMaxLat() * 10000000)); + output.writeByte((byte) header.getCenterZoom()); + output.writeInt((int) (header.getCenterLon() * 10000000)); + output.writeInt((int) (header.getCenterLat() * 10000000)); + } + + public static void serializeEntries(LittleEndianDataOutputStream output, List<Entry> entries) + throws IOException { + writeVarInt(output, entries.size()); + long lastId = 0; + for (Entry entry : entries) { + writeVarInt(output, entry.getTileId() - lastId); + lastId = entry.getTileId(); + } + for (Entry entry : entries) { + writeVarInt(output, entry.getRunLength()); + } + for (Entry entry : entries) { + writeVarInt(output, entry.getLength()); + } + for (int i = 0; i < entries.size(); i++) { + Entry entry = entries.get(i); + if (i > 0 + && entry.getOffset() == entries.get(i - 1).getOffset() + entries.get(i - 1).getLength()) { + writeVarInt(output, 0); + } else { + writeVarInt(output, entry.getOffset() + 1); + } + } + } + + public static List<Entry> deserializeEntries(LittleEndianDataInputStream buffer) + throws IOException { + long numEntries = readVarInt(buffer); + List<Entry> entries = new ArrayList<>((int) numEntries); + long lastId = 0; + for (int i = 0; i < numEntries; i++) { + long value = readVarInt(buffer); + lastId = lastId + value; + Entry entry = new Entry(); + entry.setTileId(lastId); + entries.add(entry); + } + for (int i = 0; i < numEntries; i++) { + long value = readVarInt(buffer); + entries.get(i).setRunLength(value); + } + for (int i = 0; i < numEntries; i++) { + long value = readVarInt(buffer); + entries.get(i).setLength(value); + } + for (int i = 0; i < numEntries; i++) { + long value = readVarInt(buffer); + if (value == 0 && i > 0) { + Entry prevEntry = entries.get(i - 1); + entries.get(i).setOffset(prevEntry.getOffset() + prevEntry.getLength());; + } else { + entries.get(i).setOffset(value - 1); + } + } + return entries; + } + + public static Entry findTile(List<Entry> entries, long tileId) { + int m = 0; + int n = entries.size() - 1; + while (m <= n) { + int k = (n + m) >> 1; + long cmp = tileId - entries.get(k).getTileId(); + if (cmp > 0) { + m = k + 1; + } else if (cmp < 0) { + n = k - 1; + } else { + return entries.get(k); + } + } + + // at this point, m > n + if (n >= 0) { + if (entries.get(n).getRunLength() == 0) { + return entries.get(n); + } + if (tileId - entries.get(n).getTileId() < entries.get(n).getRunLength()) { + return entries.get(n); + } + } + return null; + } + + public static Directories buildRootLeaves(List<Entry> entries, int leafSize) throws IOException { + var rootEntries = new ArrayList<Entry>(); + var numLeaves = 0; + byte[] leavesBytes; + byte[] rootBytes; + + try (var leavesOutput = new ByteArrayOutputStream(); + var leavesDataOutput = new LittleEndianDataOutputStream(leavesOutput)) { + for (var i = 0; i < entries.size(); i += leafSize) { + numLeaves++; + var end = i + leafSize; + if (i + leafSize > entries.size()) { + end = entries.size(); + } + + var offset = leavesOutput.size(); + serializeEntries(leavesDataOutput, entries.subList(i, end)); + var length = leavesOutput.size(); + rootEntries.add(new Entry(entries.get(i).getTileId(), offset, length, 0)); + } + + leavesBytes = leavesOutput.toByteArray(); + } + + try (var rootOutput = new ByteArrayOutputStream(); + var rootDataOutput = new LittleEndianDataOutputStream(rootOutput)) { + serializeEntries(rootDataOutput, rootEntries); + rootBytes = rootOutput.toByteArray(); + } + + return new Directories(rootBytes, leavesBytes, numLeaves); + } + + public static Directories optimizeDirectories(List<Entry> entries, int targetRootLenght) + throws IOException { + if (entries.size() < 16384) { + byte[] rootBytes; + try (var rootOutput = new ByteArrayOutputStream(); + var rootDataOutput = new LittleEndianDataOutputStream(rootOutput)) { + serializeEntries(rootDataOutput, entries); + rootBytes = rootOutput.toByteArray(); + } + if (rootBytes.length <= targetRootLenght) { + return new Directories(rootBytes, new byte[] {}, 0); + } + } + + double leafSize = (double) entries.size() / 3500; + if (leafSize < 4096) { + leafSize = 4096; + } + for (;;) { + var directories = buildRootLeaves(entries, (int) leafSize); + if (directories.getRoot().length <= targetRootLenght) { + return directories; + } + leafSize = leafSize * 1.2; + } + } +} diff --git a/baremaps-core/src/main/java/org/apache/baremaps/tilestore/pmtiles/PMTilesReader.java b/baremaps-core/src/main/java/org/apache/baremaps/tilestore/pmtiles/PMTilesReader.java new file mode 100644 index 00000000..42a43af4 --- /dev/null +++ b/baremaps-core/src/main/java/org/apache/baremaps/tilestore/pmtiles/PMTilesReader.java @@ -0,0 +1,95 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.baremaps.tilestore.pmtiles; + +import com.google.common.io.LittleEndianDataInputStream; +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.channels.FileChannel; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; + +public class PMTilesReader { + + private final Path path; + + private Header header; + + private List<Entry> rootEntries; + + public PMTilesReader(Path path) { + this.path = path; + } + + public Header getHeader() { + if (header == null) { + try (var inputStream = new LittleEndianDataInputStream(Files.newInputStream(path))) { + header = PMTiles.deserializeHeader(inputStream); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + return header; + } + + public List<Entry> getRootDirectory() { + if (rootEntries == null) { + var header = getHeader(); + rootEntries = getDirectory(header.getRootDirectoryOffset()); + } + return rootEntries; + } + + public List<Entry> getDirectory(long offset) { + var header = getHeader(); + try (var input = Files.newInputStream(path)) { + input.skip(offset); + try (var decompressed = + new LittleEndianDataInputStream(header.getInternalCompression().decompress(input))) { + return PMTiles.deserializeEntries(decompressed); + } + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + public ByteBuffer getTile(int z, long x, long y) { + var tileId = PMTiles.zxyToTileId(z, x, y); + var header = getHeader(); + var entries = getRootDirectory(); + var entry = PMTiles.findTile(entries, tileId); + + if (entry == null) { + return null; + } + + try (var channel = FileChannel.open(path)) { + var compressed = ByteBuffer.allocate((int) entry.getLength()); + channel.position(header.getTileDataOffset() + entry.getOffset()); + channel.read(compressed); + compressed.flip(); + try (var tile = new ByteArrayInputStream(compressed.array())) { + return ByteBuffer.wrap(tile.readAllBytes()); + } + } catch (IOException e) { + throw new RuntimeException(e); + } + } +} diff --git a/baremaps-core/src/main/java/org/apache/baremaps/tilestore/pmtiles/PMTilesStore.java b/baremaps-core/src/main/java/org/apache/baremaps/tilestore/pmtiles/PMTilesStore.java new file mode 100644 index 00000000..c6319e6c --- /dev/null +++ b/baremaps-core/src/main/java/org/apache/baremaps/tilestore/pmtiles/PMTilesStore.java @@ -0,0 +1,95 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.baremaps.tilestore.pmtiles; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.file.Path; +import java.util.HashMap; +import java.util.List; +import java.util.Optional; +import org.apache.baremaps.tilestore.TileCoord; +import org.apache.baremaps.tilestore.TileStore; +import org.apache.baremaps.tilestore.TileStoreException; +import org.apache.baremaps.vectortile.tileset.Tileset; + +public class PMTilesStore implements TileStore { + + private final PMTilesWriter writer; + + public PMTilesStore(Path path, Tileset tileset) { + try { + var metadata = new HashMap<String, Object>(); + metadata.put("name", tileset.getName()); + metadata.put("type", "baselayer"); + metadata.put("version", tileset.getVersion()); + metadata.put("description", tileset.getDescription()); + metadata.put("attribution", tileset.getAttribution()); + metadata.put("vector_layers", tileset.getVectorLayers()); + + var minZoom = Optional.ofNullable(tileset.getMinzoom()).orElse(0); + var maxZoom = Optional.ofNullable(tileset.getMaxzoom()).orElse(14); + var bounds = Optional.ofNullable(tileset.getBounds()).orElse(List.of(-180d, -90d, 180d, 90d)); + var center = Optional.ofNullable(tileset.getCenter()).orElse(List.of(0d, 0d, 3d)); + + writer = new PMTilesWriter(path); + writer.setMetadata(metadata); + writer.setMinZoom(minZoom); + writer.setMaxZoom(maxZoom); + writer.setMinLon(bounds.get(0)); + writer.setMinLat(bounds.get(1)); + writer.setMaxLon(bounds.get(2)); + writer.setMaxLat(bounds.get(3)); + writer.setCenterLon(center.get(0)); + writer.setCenterLat(center.get(1)); + writer.setMinZoom(tileset.getMinzoom()); + writer.setMaxZoom(tileset.getMaxzoom()); + + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + @Override + public ByteBuffer read(TileCoord tileCoord) throws TileStoreException { + throw new UnsupportedOperationException(); + } + + @Override + public void write(TileCoord tileCoord, ByteBuffer blob) throws TileStoreException { + try { + writer.setTile(tileCoord.z(), tileCoord.x(), tileCoord.y(), blob.array()); + } catch (IOException e) { + throw new TileStoreException(e); + } + } + + @Override + public void delete(TileCoord tileCoord) throws TileStoreException { + throw new UnsupportedOperationException(); + } + + @Override + public void close() throws TileStoreException { + try { + writer.write(); + } catch (IOException e) { + throw new TileStoreException(e); + } + } +} diff --git a/baremaps-core/src/main/java/org/apache/baremaps/tilestore/pmtiles/PMTilesWriter.java b/baremaps-core/src/main/java/org/apache/baremaps/tilestore/pmtiles/PMTilesWriter.java new file mode 100644 index 00000000..f01aaf3e --- /dev/null +++ b/baremaps-core/src/main/java/org/apache/baremaps/tilestore/pmtiles/PMTilesWriter.java @@ -0,0 +1,208 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.baremaps.tilestore.pmtiles; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.hash.Hashing; +import com.google.common.io.LittleEndianDataOutputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.*; + +public class PMTilesWriter { + + private Path path; + + private Map<String, Object> metadata = new HashMap<>(); + + private List<Entry> entries; + + private Map<Long, Long> tileHashToOffset; + + private Long lastTileHash = null; + + private Path tilePath; + + private boolean clustered = true; + + private int minZoom = 0; + + private int maxZoom = 14; + + private double minLon = -180; + + private double minLat = -90; + + private double maxLon = 180; + + private double maxLat = 90; + + private int centerZoom = 3; + + private double centerLat = 0; + + private double centerLon = 0; + + public PMTilesWriter(Path path) throws IOException { + this(path, new ArrayList<>(), new HashMap<>()); + } + + public PMTilesWriter(Path path, List<Entry> entries, Map<Long, Long> tileHashToOffset) + throws IOException { + this.path = path; + this.entries = entries; + this.tileHashToOffset = tileHashToOffset; + this.tilePath = Files.createTempFile(path.getParent(), "tiles_", ".tmp"); + } + + public void setMetadata(Map<String, Object> metadata) { + this.metadata = metadata; + } + + public void setTile(int z, int x, int y, byte[] bytes) throws IOException { + // Write the tile + var tileId = PMTiles.zxyToTileId(z, x, y); + var tileLength = bytes.length; + Long tileHash = Hashing.farmHashFingerprint64().hashBytes(bytes).asLong(); + + // If the tile is not greater than the last one, the index is not clustered + if (entries.size() > 0 && tileId < entries.get(entries.size() - 1).getTileId()) { + clustered = false; + } + + // If the tile is the same as the last one, increment the run length + if (clustered && tileHash.equals(lastTileHash)) { + var lastEntry = entries.get(entries.size() - 1); + lastEntry.setRunLength(lastEntry.getRunLength() + 1); + } + + // Else, if the tile is the same as the last one, increment the run length + else if (tileHashToOffset.containsKey(tileHash)) { + var tileOffset = tileHashToOffset.get(tileHash); + entries.add(new Entry(tileId, tileOffset, tileLength, 1)); + } + + // Else, write the tile and add it to the index + else { + var tileOffset = Files.size(tilePath); + tileHashToOffset.put(tileHash, tileOffset); + lastTileHash = tileHash; + try (var output = new FileOutputStream(tilePath.toFile(), true)) { + output.write(bytes); + entries.add(new Entry(tileId, tileOffset, tileLength, 1)); + } + } + } + + public void setMinZoom(int minZoom) { + this.minZoom = minZoom; + } + + public void setMaxZoom(int maxZoom) { + this.maxZoom = maxZoom; + } + + public void setMinLon(double minLon) { + this.minLon = minLon; + } + + public void setMinLat(double minLat) { + this.minLat = minLat; + } + + public void setMaxLon(double maxLon) { + this.maxLon = maxLon; + } + + public void setMaxLat(double maxLat) { + this.maxLat = maxLat; + } + + public void setCenterZoom(int centerZoom) { + this.centerZoom = centerZoom; + } + + public void setCenterLat(double centerLat) { + this.centerLat = centerLat; + } + + public void setCenterLon(double centerLon) { + this.centerLon = centerLon; + } + + public void write() throws IOException { + // Sort the entries by tile id + if (!clustered) { + entries.sort(Comparator.comparingLong(Entry::getTileId)); + } + + var metadataBytes = new ObjectMapper().writeValueAsBytes(metadata); + + var directories = PMTiles.optimizeDirectories(entries, 16247); + var rootOffset = 127; + var rootLength = directories.getRoot().length; + var metadataOffset = rootOffset + rootLength; + var metadataLength = metadataBytes.length; + var leavesOffset = metadataOffset + metadataLength; + var leavesLength = directories.getLeaves().length; + var tilesOffset = leavesOffset + leavesLength; + var tilesLength = Files.size(tilePath); + var numTiles = entries.size(); + + var header = new Header(); + header.setNumAddressedTiles(numTiles); + header.setNumTileEntries(numTiles); + header.setNumTileContents(numTiles); + header.setClustered(true); + + header.setInternalCompression(Compression.None); + header.setTileCompression(Compression.Gzip); + header.setTileType(TileType.mvt); + header.setRootOffset(rootOffset); + header.setRootLength(rootLength); + header.setMetadataOffset(metadataOffset); + header.setMetadataLength(metadataLength); + header.setLeavesOffset(leavesOffset); + header.setLeavesLength(leavesLength); + header.setTilesOffset(tilesOffset); + header.setTilesLength(tilesLength); + + header.setMinZoom(minZoom); + header.setMaxZoom(maxZoom); + header.setMinLon(minLon); + header.setMinLat(minLat); + header.setMaxLon(maxLon); + header.setMaxLat(maxLat); + header.setCenterZoom(centerZoom); + header.setCenterLat(centerLat); + header.setCenterLon(centerLon); + + try (var output = new LittleEndianDataOutputStream(new FileOutputStream(path.toFile()))) { + PMTiles.serializeHeader(output, header); + output.write(directories.getRoot()); + output.write(metadataBytes); + output.write(directories.getLeaves()); + Files.copy(tilePath, output); + } finally { + Files.delete(tilePath); + } + } + +} diff --git a/baremaps-core/src/test/java/org/apache/baremaps/stream/StreamUtilsTest.java b/baremaps-core/src/main/java/org/apache/baremaps/tilestore/pmtiles/TileType.java similarity index 64% copy from baremaps-core/src/test/java/org/apache/baremaps/stream/StreamUtilsTest.java copy to baremaps-core/src/main/java/org/apache/baremaps/tilestore/pmtiles/TileType.java index ee1cc354..335bc488 100644 --- a/baremaps-core/src/test/java/org/apache/baremaps/stream/StreamUtilsTest.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/tilestore/pmtiles/TileType.java @@ -15,20 +15,13 @@ * limitations under the License. */ -package org.apache.baremaps.stream; +package org.apache.baremaps.tilestore.pmtiles; -import static org.junit.jupiter.api.Assertions.assertEquals; - -import java.util.List; -import java.util.stream.IntStream; -import org.junit.jupiter.api.Test; - -class StreamUtilsTest { - - @Test - void partition() { - List<Integer> list = IntStream.range(0, 100).boxed().toList(); - List<List<Integer>> partitions = StreamUtils.partition(list.stream(), 10).toList(); - assertEquals(partitions.size(), 10); - } +enum TileType { + unknown, + mvt, + png, + jpeg, + webp, + avif, } diff --git a/baremaps-core/src/main/java/org/apache/baremaps/tilestore/postgres/PostgresTileStore.java b/baremaps-core/src/main/java/org/apache/baremaps/tilestore/postgres/PostgresTileStore.java index add70a06..3244765d 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/tilestore/postgres/PostgresTileStore.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/tilestore/postgres/PostgresTileStore.java @@ -99,6 +99,9 @@ public class PostgresTileStore implements TileStore { byte[] bytes = resultSet.getBytes(1); gzip.write(bytes); } + } catch (Exception e) { + throw new TileStoreException(String.format("Failed to execute statement: %s", statement), + e); } // Log slow queries (> 10s) @@ -125,7 +128,7 @@ public class PostgresTileStore implements TileStore { protected static Query prepareQuery(Tileset tileset, int zoom) { // Initialize a builder for the tile sql var tileSql = new StringBuilder(); - tileSql.append("SELECT ("); + tileSql.append("SELECT "); // Iterate over the layers and keep track of the number of layers and parameters included in the // final sql @@ -136,7 +139,7 @@ public class PostgresTileStore implements TileStore { // Initialize a builder for the layer sql var layerSql = new StringBuilder(); - var layerHead = "(WITH mvtGeom AS ("; + var layerHead = String.format("(SELECT ST_AsMVT(mvtGeom.*, '%s') FROM (", layer.getId()); layerSql.append(layerHead); // Iterate over the queries and keep track of the number of queries included in the final @@ -154,7 +157,8 @@ public class PostgresTileStore implements TileStore { } // Add the sql to the layer sql - var querySql = query.getSql() + var querySql = query.getSql().trim() + .replaceAll("\\s+", " ") .replace(";", "") .replace("?", "??") .replace("$zoom", String.valueOf(zoom)); @@ -171,8 +175,7 @@ public class PostgresTileStore implements TileStore { } // Add the tail of the layer sql - var layerQueryTail = - String.format(") SELECT ST_AsMVT(mvtGeom.*, '%s') FROM mvtGeom)", layer.getId()); + var layerQueryTail = ") AS mvtgeom)"; layerSql.append(layerQueryTail); // Only include the layer sql if queries were included for this layer @@ -192,7 +195,7 @@ public class PostgresTileStore implements TileStore { } // Add the tail of the tile sql - var tileQueryTail = ") mvtTile"; + var tileQueryTail = " mvttile"; tileSql.append(tileQueryTail); // Format the sql query @@ -216,4 +219,9 @@ public class PostgresTileStore implements TileStore { public void delete(TileCoord tileCoord) { throw new UnsupportedOperationException("The postgis tile store is read only"); } + + @Override + public void close() throws Exception { + // do nothing + } } diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExportVectorTiles.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExportVectorTiles.java index 3f5e1854..93a27a2c 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExportVectorTiles.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExportVectorTiles.java @@ -33,6 +33,7 @@ import org.apache.baremaps.stream.StreamUtils; import org.apache.baremaps.tilestore.*; import org.apache.baremaps.tilestore.file.FileTileStore; import org.apache.baremaps.tilestore.mbtiles.MBTilesStore; +import org.apache.baremaps.tilestore.pmtiles.PMTilesStore; import org.apache.baremaps.tilestore.postgres.PostgresTileStore; import org.apache.baremaps.utils.SqliteUtils; import org.apache.baremaps.vectortile.tileset.Tileset; @@ -52,7 +53,8 @@ public record ExportVectorTiles( public enum Format { file, - mbtiles + mbtiles, + pmtiles } private static final Logger logger = LoggerFactory.getLogger(ExportVectorTiles.class); @@ -64,40 +66,43 @@ public record ExportVectorTiles( var tileset = objectMapper.readValue(configReader.read(this.tileset), Tileset.class); var datasource = context.getDataSource(tileset.getDatabase()); - var sourceTileStore = sourceTileStore(tileset, datasource); - var targetTileStore = targetTileStore(tileset); - - var envelope = tileset.getBounds().size() == 4 - ? new Envelope( - tileset.getBounds().get(0), tileset.getBounds().get(2), - tileset.getBounds().get(1), tileset.getBounds().get(3)) - : new Envelope(-180, 180, -85.0511, 85.0511); - - var count = TileCoord.count(envelope, tileset.getMinzoom(), tileset.getMaxzoom()); - var start = System.currentTimeMillis(); - - var tileCoordIterator = - TileCoord.iterator(envelope, tileset.getMinzoom(), tileset.getMaxzoom()); - var tileCoordStream = - StreamUtils.stream(tileCoordIterator).peek(new ProgressLogger<>(count, 5000)); - var bufferedTileEntryStream = StreamUtils.bufferInCompletionOrder(tileCoordStream, tile -> { - try { - return new TileEntry(tile, sourceTileStore.read(tile)); - } catch (TileStoreException e) { - throw new RuntimeException(e); - } - }, 1000); - var partitionedTileEntryStream = StreamUtils.partition(bufferedTileEntryStream, 1000); - partitionedTileEntryStream.forEach(batch -> { - try { - targetTileStore.write(batch); - } catch (TileStoreException e) { - throw new RuntimeException(e); - } - }); - - var stop = System.currentTimeMillis(); - logger.info("Exported {} tiles in {}s", count, (stop - start) / 1000); + try (var sourceTileStore = sourceTileStore(tileset, datasource); + var targetTileStore = targetTileStore(tileset)) { + + var envelope = tileset.getBounds().size() == 4 + ? new Envelope( + tileset.getBounds().get(0), tileset.getBounds().get(2), + tileset.getBounds().get(1), tileset.getBounds().get(3)) + : new Envelope(-180, 180, -85.0511, 85.0511); + + var count = TileCoord.count(envelope, tileset.getMinzoom(), tileset.getMaxzoom()); + var start = System.currentTimeMillis(); + + var tileCoordIterator = + TileCoord.iterator(envelope, tileset.getMinzoom(), tileset.getMaxzoom()); + var tileCoordStream = + StreamUtils.stream(tileCoordIterator).peek(new ProgressLogger<>(count, 5000)); + + var bufferedTileEntryStream = StreamUtils.bufferInCompletionOrder(tileCoordStream, tile -> { + try { + return new TileEntry(tile, sourceTileStore.read(tile)); + } catch (TileStoreException e) { + throw new RuntimeException(e); + } + }, 1000); + + var partitionedTileEntryStream = StreamUtils.partition(bufferedTileEntryStream, 1000); + partitionedTileEntryStream.forEach(batch -> { + try { + targetTileStore.write(batch); + } catch (TileStoreException e) { + throw new RuntimeException(e); + } + }); + + var stop = System.currentTimeMillis(); + logger.info("Exported {} tiles in {}s", count, (stop - start) / 1000); + } } private TileStore sourceTileStore(Tileset tileset, DataSource datasource) { @@ -115,6 +120,10 @@ public record ExportVectorTiles( tilesStore.initializeDatabase(); tilesStore.writeMetadata(metadata(source)); return tilesStore; + case pmtiles: + Files.deleteIfExists(repository); + var tileStore = new PMTilesStore(repository, source); + return tileStore; default: throw new IllegalArgumentException("Unsupported format"); } diff --git a/baremaps-core/src/test/java/org/apache/baremaps/stream/StreamUtilsTest.java b/baremaps-core/src/test/java/org/apache/baremaps/stream/StreamUtilsTest.java index ee1cc354..23957274 100644 --- a/baremaps-core/src/test/java/org/apache/baremaps/stream/StreamUtilsTest.java +++ b/baremaps-core/src/test/java/org/apache/baremaps/stream/StreamUtilsTest.java @@ -18,8 +18,10 @@ package org.apache.baremaps.stream; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; import java.util.List; +import java.util.function.Function; import java.util.stream.IntStream; import org.junit.jupiter.api.Test; @@ -31,4 +33,44 @@ class StreamUtilsTest { List<List<Integer>> partitions = StreamUtils.partition(list.stream(), 10).toList(); assertEquals(partitions.size(), 10); } + + @Test + void bufferInSourceOrder() { + List<Integer> l1 = IntStream.range(0, 100).boxed().toList(); + List<Integer> l2 = StreamUtils.bufferInSourceOrder(l1.stream(), i -> i, 10).toList(); + assertEquals(l2.size(), l1.size()); + assertEquals(l2, l1); + } + + @Test + void bufferInSourceOrderWithException() { + assertThrows(StreamException.class, () -> { + List<Integer> l1 = IntStream.range(0, 100).boxed().toList(); + Function<Integer, Integer> throwException = i -> { + throw new RuntimeException(); + }; + StreamUtils.bufferInSourceOrder(l1.stream(), throwException, 10).sorted().toList(); + }); + } + + @Test + void bufferInCompletionOrder() { + List<Integer> l1 = IntStream.range(0, 100).boxed().toList(); + List<Integer> l2 = + StreamUtils.bufferInCompletionOrder(l1.stream(), i -> i, 10).sorted().toList(); + assertEquals(l2.size(), l1.size()); + assertEquals(l2, l1); + } + + @Test + void bufferInCompletionOrderWithException() { + assertThrows(StreamException.class, () -> { + List<Integer> l1 = IntStream.range(0, 100).boxed().toList(); + Function<Integer, Integer> throwException = i -> { + throw new RuntimeException(); + }; + StreamUtils.bufferInCompletionOrder(l1.stream(), throwException, 10).sorted().toList(); + }); + } + } diff --git a/baremaps-core/src/test/java/org/apache/baremaps/tilestore/pmtiles/PMTilesTest.java b/baremaps-core/src/test/java/org/apache/baremaps/tilestore/pmtiles/PMTilesTest.java new file mode 100644 index 00000000..e078d1d5 --- /dev/null +++ b/baremaps-core/src/test/java/org/apache/baremaps/tilestore/pmtiles/PMTilesTest.java @@ -0,0 +1,288 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.baremaps.tilestore.pmtiles; + +import static org.junit.jupiter.api.Assertions.*; + +import com.google.common.io.LittleEndianDataInputStream; +import com.google.common.io.LittleEndianDataOutputStream; +import com.google.common.math.LongMath; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.nio.channels.Channels; +import java.nio.channels.FileChannel; +import java.util.ArrayList; +import java.util.List; +import java.util.Random; +import org.apache.baremaps.testing.TestFiles; +import org.junit.jupiter.api.Test; + +class PMTilesTest { + + @Test + void decodeVarInt() throws IOException { + var b = new LittleEndianDataInputStream(new ByteArrayInputStream(new byte[] { + (byte) 0, (byte) 1, + (byte) 127, (byte) 0xe5, + (byte) 0x8e, (byte) 0x26 + })); + assertEquals(PMTiles.readVarInt(b), 0); + assertEquals(PMTiles.readVarInt(b), 1); + assertEquals(PMTiles.readVarInt(b), 127); + assertEquals(PMTiles.readVarInt(b), 624485); + b = new LittleEndianDataInputStream(new ByteArrayInputStream(new byte[] { + (byte) 0xff, (byte) 0xff, + (byte) 0xff, (byte) 0xff, + (byte) 0xff, (byte) 0xff, + (byte) 0xff, (byte) 0x0f, + })); + assertEquals(PMTiles.readVarInt(b), 9007199254740991L); + } + + @Test + void encodeVarInt() throws IOException { + for (long i = 0; i < 1000; i++) { + var array = new ByteArrayOutputStream(); + var output = new LittleEndianDataOutputStream(array); + PMTiles.writeVarInt(output, i); + var input = new LittleEndianDataInputStream(new ByteArrayInputStream(array.toByteArray())); + assertEquals(i, PMTiles.readVarInt(input)); + } + for (long i = Long.MAX_VALUE - 1000; i < Long.MAX_VALUE; i++) { + var array = new ByteArrayOutputStream(); + var output = new LittleEndianDataOutputStream(array); + PMTiles.writeVarInt(output, i); + var input = new LittleEndianDataInputStream(new ByteArrayInputStream(array.toByteArray())); + assertEquals(i, PMTiles.readVarInt(input)); + } + } + + @Test + void zxyToTileId() { + assertEquals(PMTiles.zxyToTileId(0, 0, 0), 0); + assertEquals(PMTiles.zxyToTileId(1, 0, 0), 1); + assertEquals(PMTiles.zxyToTileId(1, 0, 1), 2); + assertEquals(PMTiles.zxyToTileId(1, 1, 1), 3); + assertEquals(PMTiles.zxyToTileId(1, 1, 0), 4); + assertEquals(PMTiles.zxyToTileId(2, 0, 0), 5); + } + + @Test + void tileIdToZxy() { + assertArrayEquals(PMTiles.tileIdToZxy(0), new long[] {0, 0, 0}); + assertArrayEquals(PMTiles.tileIdToZxy(1), new long[] {1, 0, 0}); + assertArrayEquals(PMTiles.tileIdToZxy(2), new long[] {1, 0, 1}); + assertArrayEquals(PMTiles.tileIdToZxy(3), new long[] {1, 1, 1}); + assertArrayEquals(PMTiles.tileIdToZxy(4), new long[] {1, 1, 0}); + assertArrayEquals(PMTiles.tileIdToZxy(5), new long[] {2, 0, 0}); + } + + @Test + void aLotOfTiles() { + for (int z = 0; z < 9; z++) { + for (long x = 0; x < 1 << z; x++) { + for (long y = 0; y < 1 << z; y++) { + var result = PMTiles.tileIdToZxy(PMTiles.zxyToTileId(z, x, y)); + if (result[0] != z || result[1] != x || result[2] != y) { + fail("roundtrip failed"); + } + } + } + } + } + + @Test + void tileExtremes() { + for (var z = 0; z < 27; z++) { + var dim = LongMath.pow(2, z) - 1; + var tl = PMTiles.tileIdToZxy(PMTiles.zxyToTileId(z, 0, 0)); + assertArrayEquals(new long[] {z, 0, 0}, tl); + var tr = PMTiles.tileIdToZxy(PMTiles.zxyToTileId(z, dim, 0)); + assertArrayEquals(new long[] {z, dim, 0}, tr); + var bl = PMTiles.tileIdToZxy(PMTiles.zxyToTileId(z, 0, dim)); + assertArrayEquals(new long[] {z, 0, dim}, bl); + var br = PMTiles.tileIdToZxy(PMTiles.zxyToTileId(z, dim, dim)); + assertArrayEquals(new long[] {z, dim, dim}, br); + } + } + + @Test + void invalidTiles() { + assertThrows(RuntimeException.class, () -> PMTiles.tileIdToZxy(9007199254740991L)); + assertThrows(RuntimeException.class, () -> PMTiles.zxyToTileId(27, 0, 0)); + assertThrows(RuntimeException.class, () -> PMTiles.zxyToTileId(0, 1, 1)); + } + + @Test + void decodeHeader() throws IOException { + var file = TestFiles.resolve("pmtiles/test_fixture_1.pmtiles"); + try (var channel = FileChannel.open(file)) { + var input = new LittleEndianDataInputStream(Channels.newInputStream(channel)); + var header = PMTiles.deserializeHeader(input); + assertEquals(header.getRootDirectoryOffset(), 127); + assertEquals(header.getRootDirectoryLength(), 25); + assertEquals(header.getJsonMetadataOffset(), 152); + assertEquals(header.getJsonMetadataLength(), 247); + assertEquals(header.getLeafDirectoryOffset(), 0); + assertEquals(header.getLeafDirectoryLength(), 0); + assertEquals(header.getTileDataOffset(), 399); + assertEquals(header.getTileDataLength(), 69); + assertEquals(header.getNumAddressedTiles(), 1); + assertEquals(header.getNumTileEntries(), 1); + assertEquals(header.getNumTileContents(), 1); + assertFalse(header.isClustered()); + assertEquals(header.getInternalCompression(), Compression.Gzip); + assertEquals(header.getTileCompression(), Compression.Gzip); + assertEquals(header.getTileType(), TileType.mvt); + assertEquals(header.getMinZoom(), 0); + assertEquals(header.getMaxZoom(), 0); + assertEquals(header.getMinLon(), 0); + assertEquals(header.getMinLat(), 0); + assertEquals(Math.round(header.getMaxLon()), 1); + assertEquals(Math.round(header.getMaxLat()), 1); + } + } + + @Test + void encodeHeader() throws IOException { + var etag = "1"; + var header = new Header( + 127, + 25, + 152, + 247, + 0, + 0, + 399, + 69, + 1, + 1, + 1, + 10, + false, + Compression.Gzip, + Compression.Gzip, + TileType.mvt, + 0, + 0, + 0, + 1, + 1, + 0, + 0, + 0, + 0); + + var array = new ByteArrayOutputStream(); + + var output = new LittleEndianDataOutputStream(array); + PMTiles.serializeHeader(output, header); + + var input = new LittleEndianDataInputStream(new ByteArrayInputStream(array.toByteArray())); + var header2 = PMTiles.deserializeHeader(input); + + assertEquals(header, header2); + } + + @Test + void searchForMissingEntry() { + var entries = new ArrayList<Entry>(); + assertEquals(PMTiles.findTile(entries, 101), null); + } + + @Test + void searchForFirstEntry() { + var entry = new Entry(100, 1, 1, 1); + var entries = new ArrayList<Entry>(); + entries.add(entry); + assertEquals(PMTiles.findTile(entries, 100), entry); + } + + @Test + void searchWithRunLength() { + var entry = new Entry(3, 3, 1, 2); + var entries = new ArrayList<Entry>(); + entries.add(entry); + entries.add(new Entry(5, 5, 1, 2)); + assertEquals(PMTiles.findTile(entries, 4), entry); + } + + @Test + void searchWithMultipleTileEntries() { + var entries = new ArrayList<Entry>(); + entries.add(new Entry(100, 1, 1, 2)); + var entry = PMTiles.findTile(entries, 101); + assertEquals(entry.getOffset(), 1); + assertEquals(entry.getLength(), 1); + + entries = new ArrayList<Entry>(); + entries.add(new Entry(100, 1, 1, 1)); + entries.add(new Entry(150, 2, 2, 2)); + entry = PMTiles.findTile(entries, 151); + assertEquals(entry.getOffset(), 2); + assertEquals(entry.getLength(), 2); + + entries = new ArrayList<>(); + entries.add(new Entry(50, 1, 1, 2)); + entries.add(new Entry(100, 2, 2, 1)); + entries.add(new Entry(150, 3, 3, 1)); + entry = PMTiles.findTile(entries, 51); + assertEquals(entry.getOffset(), 1); + assertEquals(entry.getLength(), 1); + } + + @Test + void leafSearch() { + var entries = new ArrayList<Entry>(); + entries.add(new Entry(100, 1, 1, 0)); + var entry = PMTiles.findTile(entries, 150); + assertEquals(entry.getOffset(), 1); + assertEquals(entry.getLength(), 1); + } + + @Test + void buildRootLeaves() throws IOException { + var entries = List.of(new Entry(100, 1, 1, 0)); + var directories = PMTiles.buildRootLeaves(entries, 1); + assertEquals(directories.getNumLeaves(), 1); + + } + + @Test + void optimizeDirectories() throws IOException { + var random = new Random(3857); + var entries = new ArrayList<Entry>(); + entries.add(new Entry(0, 0, 100, 1)); + var directories = PMTiles.optimizeDirectories(entries, 100); + assertFalse(directories.getLeaves().length > 0); + assertEquals(0, directories.getNumLeaves()); + + entries = new ArrayList<>(); + int offset = 0; + for (var i = 0; i < 1000; i++) { + var randTileSize = random.nextInt(1000000); + entries.add(new Entry(i, offset, randTileSize, 1)); + offset += randTileSize; + } + directories = PMTiles.optimizeDirectories(entries, 1024); + assertFalse(directories.getRoot().length > 1024); + assertFalse(directories.getNumLeaves() == 0); + assertFalse(directories.getLeaves().length == 0); + } +} diff --git a/baremaps-core/src/test/java/org/apache/baremaps/tilestore/postgres/PostgresTileStoreTest.java b/baremaps-core/src/test/java/org/apache/baremaps/tilestore/postgres/PostgresTileStoreTest.java index f146a2cf..24892ac1 100644 --- a/baremaps-core/src/test/java/org/apache/baremaps/tilestore/postgres/PostgresTileStoreTest.java +++ b/baremaps-core/src/test/java/org/apache/baremaps/tilestore/postgres/PostgresTileStoreTest.java @@ -40,7 +40,7 @@ class PostgresTileStoreTest { List.of(new TilesetQuery(0, 20, "SELECT id, tags, geom FROM table"))))); var query = PostgresTileStore.prepareQuery(tileset, 10); assertEquals( - "SELECT ((WITH mvtGeom AS (SELECT ST_AsMVTGeom(t.geom, ST_TileEnvelope(?, ?, ?)) AS geom, t.tags, t.id FROM (SELECT id, tags, geom FROM table) AS t WHERE t.geom && ST_TileEnvelope(?, ?, ?, margin => (64.0/4096))) SELECT ST_AsMVT(mvtGeom.*, 'a') FROM mvtGeom) || (WITH mvtGeom AS (SELECT ST_AsMVTGeom(t.geom, ST_TileEnvelope(?, ?, ?)) AS geom, t.tags, t.id FROM (SELECT id, tags, geom FROM table) AS t WHERE t.geom && ST_TileEnvelope(?, ?, ?, margin => (64.0/4096))) SELECT ST_AsMVT(mv [...] + "SELECT (SELECT ST_AsMVT(mvtGeom.*, 'a') FROM (SELECT ST_AsMVTGeom(t.geom, ST_TileEnvelope(?, ?, ?)) AS geom, t.tags, t.id FROM (SELECT id, tags, geom FROM table) AS t WHERE t.geom && ST_TileEnvelope(?, ?, ?, margin => (64.0/4096))) AS mvtGeom) || (SELECT ST_AsMVT(mvtGeom.*, 'b') FROM (SELECT ST_AsMVTGeom(t.geom, ST_TileEnvelope(?, ?, ?)) AS geom, t.tags, t.id FROM (SELECT id, tags, geom FROM table) AS t WHERE t.geom && ST_TileEnvelope(?, ?, ?, margin => (64.0/4096))) AS mvtGeom) [...] query.sql()); } } diff --git a/baremaps-core/src/test/java/org/apache/baremaps/vectortile/VectorTileViewer.java b/baremaps-core/src/test/java/org/apache/baremaps/vectortile/VectorTileViewer.java index faada968..36e925f4 100644 --- a/baremaps-core/src/test/java/org/apache/baremaps/vectortile/VectorTileViewer.java +++ b/baremaps-core/src/test/java/org/apache/baremaps/vectortile/VectorTileViewer.java @@ -49,7 +49,7 @@ public class VectorTileViewer { } } - static class TilePanel extends JPanel { + public static class TilePanel extends JPanel { private final Tile tile; diff --git a/baremaps-core/src/test/resources/pmtiles/empty.pmtiles b/baremaps-core/src/test/resources/pmtiles/empty.pmtiles new file mode 100644 index 00000000..e69de29b diff --git a/baremaps-core/src/test/resources/pmtiles/invalid.pmtiles b/baremaps-core/src/test/resources/pmtiles/invalid.pmtiles new file mode 100644 index 00000000..f2b720ba --- /dev/null +++ b/baremaps-core/src/test/resources/pmtiles/invalid.pmtiles @@ -0,0 +1 @@ +This is an invalid tile archive, a test case to make sure that the code throws an error, but it needs to be the minimum size to pass the first test diff --git a/baremaps-core/src/test/resources/pmtiles/invalid_v4.pmtiles b/baremaps-core/src/test/resources/pmtiles/invalid_v4.pmtiles new file mode 100644 index 00000000..1871cb27 Binary files /dev/null and b/baremaps-core/src/test/resources/pmtiles/invalid_v4.pmtiles differ diff --git a/baremaps-core/src/test/resources/pmtiles/test_fixture_1.pmtiles b/baremaps-core/src/test/resources/pmtiles/test_fixture_1.pmtiles new file mode 100644 index 00000000..c86db1f2 Binary files /dev/null and b/baremaps-core/src/test/resources/pmtiles/test_fixture_1.pmtiles differ diff --git a/baremaps-core/src/test/resources/pmtiles/test_fixture_2.pmtiles b/baremaps-core/src/test/resources/pmtiles/test_fixture_2.pmtiles new file mode 100644 index 00000000..cb19dd5f Binary files /dev/null and b/baremaps-core/src/test/resources/pmtiles/test_fixture_2.pmtiles differ diff --git a/basemap/import.js b/basemap/import.js index fec1e00a..75715991 100644 --- a/basemap/import.js +++ b/basemap/import.js @@ -18,6 +18,22 @@ import config from "./config.js"; export default { "steps": [ + { + "id": "initialize-database", + "needs": [], + "tasks": [ + { + "type": "ExecuteSqlScript", + "file": "queries/assertions.sql", + "database": config.database, + }, + { + "type": "ExecuteSqlScript", + "file": "queries/functions.sql", + "database": config.database, + }, + ] + }, { "id": "natural-earth", "needs": [], diff --git a/basemap/layers/building/extrusion.js b/basemap/layers/building/extrusion.js index 51081931..f5ebe305 100644 --- a/basemap/layers/building/extrusion.js +++ b/basemap/layers/building/extrusion.js @@ -45,15 +45,6 @@ export default { 16, ['get', "extrusion:height"] ], - "fill-extrusion-opacity": [ - 'interpolate', - ['linear'], - ['zoom'], - 15, - 0, - 16, - 0.8 - ], "fill-extrusion-color": theme.buildingShapeFillColor, // Having muliple colors for building parts results in z-fighting // https://github.com/maplibre/maplibre-gl-js/issues/3157 diff --git a/basemap/layers/building/tileset.js b/basemap/layers/building/tileset.js index e7fabd2f..12c9a8b7 100644 --- a/basemap/layers/building/tileset.js +++ b/basemap/layers/building/tileset.js @@ -25,23 +25,27 @@ export default { SELECT id, tags - || jsonb_build_object('extrusion:base', (CASE - WHEN tags ? 'building:min_height' - THEN (tags ->> 'building:min_height')::real - WHEN tags ->> 'building:min_level' ~ '^[0-9.]+$' - THEN (tags ->> 'building:min_level')::real * 3 - ELSE 0 END)) - || jsonb_build_object('extrusion:height', (CASE - WHEN tags ? 'height' - THEN (SUBSTRING(tags ->> 'height' FROM '^[0-9]+'))::real - WHEN tags ? 'building:height' - THEN (tags ->> 'building:height')::real - WHEN tags ->> 'building:levels' ~ '^[0-9.]+$' - THEN (tags ->> 'building:levels')::real * 3 - ELSE 6 END)) as tags, + || jsonb_build_object('extrusion:base', + CASE + WHEN tags ? 'building:min_height' + THEN convert_to_number(tags ->> 'building:min_height', 0) + WHEN tags ? 'building:min_level' + THEN convert_to_number(tags ->> 'building:min_level', 0) * 3 + ELSE 0 + END) + || jsonb_build_object('extrusion:height', + CASE + WHEN tags ? 'height' + THEN convert_to_number(tags ->> 'height', 6) + WHEN tags ? 'building:height' + THEN convert_to_number(tags ->> 'building:height', 6) + WHEN tags ? 'building:levels' + THEN convert_to_number(tags ->> 'building:levels', 2) * 3 + ELSE 6 + END) as tags, geom FROM osm_ways - WHERE (tags ? 'building' OR tags ? 'building:part') AND ((NOT tags ? 'layer') OR (tags ->> 'layer')::real >= 0)`, + WHERE (tags ? 'building' OR tags ? 'building:part') AND ((NOT tags ? 'layer') OR convert_to_number(tags ->> 'layer', 0) >= 0)`, }, { minzoom: 13, @@ -50,23 +54,27 @@ export default { SELECT id, tags - || jsonb_build_object('extrusion:base', (CASE - WHEN tags ? 'building:min_height' - THEN (tags ->> 'building:min_height')::real - WHEN tags ->> 'building:min_level' ~ '^[0-9.]+$' - THEN (tags ->> 'building:min_level')::real * 3 - ELSE 0 END)) - || jsonb_build_object('extrusion:height', (CASE - WHEN tags ? 'height' - THEN (SUBSTRING(tags ->> 'height' FROM '^[0-9]+'))::real - WHEN tags ? 'building:height' - THEN (tags ->> 'building:height')::real - WHEN tags ->> 'building:levels' ~ '^[0-9.]+$' - THEN (tags ->> 'building:levels')::real * 3 - ELSE 6 END)) as tags, + || jsonb_build_object('extrusion:base', + CASE + WHEN tags ? 'building:min_height' + THEN convert_to_number(tags ->> 'building:min_height', 0) + WHEN tags ? 'building:min_level' + THEN convert_to_number(tags ->> 'building:min_level', 0) * 3 + ELSE 0 + END) + || jsonb_build_object('extrusion:height', + CASE + WHEN tags ? 'height' + THEN convert_to_number(tags ->> 'height', 6) + WHEN tags ? 'building:height' + THEN convert_to_number(tags ->> 'building:height', 6) + WHEN tags ? 'building:levels' + THEN convert_to_number(tags ->> 'building:levels', 2) * 3 + ELSE 6 + END) as tags, geom FROM osm_relations - WHERE (tags ? 'building' OR tags ? 'building:part') AND ((NOT tags ? 'layer') OR (tags ->> 'layer')::real >= 0)`, + WHERE (tags ? 'building' OR tags ? 'building:part') AND ((NOT tags ? 'layer') OR convert_to_number(tags ->> 'layer', 0) >= 0)`, }, ], } diff --git a/basemap/queries/assertions.sql b/basemap/queries/assertions.sql new file mode 100644 index 00000000..7a2019c1 --- /dev/null +++ b/basemap/queries/assertions.sql @@ -0,0 +1,48 @@ +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to you under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. + +-- Asserts that the test function raises the expected exception. +CREATE OR REPLACE FUNCTION assert_exception(test_query text, expected_exception text) + RETURNS void AS $$ +BEGIN + BEGIN + -- Execute the test query using dynamic SQL + EXECUTE test_query; + + -- If no exception is raised, fail the test + RAISE EXCEPTION 'Assertion Failed: Expected exception %, but no exception was raised', expected_exception; + EXCEPTION + WHEN OTHERS THEN + -- Check if the raised exception matches the expected exception + IF SQLERRM NOT LIKE expected_exception THEN + RAISE EXCEPTION 'Assertion Failed: Expected exception %, but was %', expected_exception, SQLERRM; + END IF; + END; +END; +$$ LANGUAGE plpgsql; + +-- Asserts that the actual value is equal to the expected value. +CREATE OR REPLACE FUNCTION assert_equals(actual numeric, expected numeric) + RETURNS void AS $$ +BEGIN + IF actual != expected THEN + RAISE EXCEPTION 'Assertion Failed: Expected %, but was %', expected, actual; + END IF; +END; +$$ LANGUAGE plpgsql; + +-- Test cases for the assert_equals function +SELECT assert_equals(1, 1); +SELECT assert_exception('SELECT assert_equals(1, 2)', 'Assertion Failed: Expected 2, but was 1'); \ No newline at end of file diff --git a/basemap/queries/functions.sql b/basemap/queries/functions.sql new file mode 100644 index 00000000..f07d3362 --- /dev/null +++ b/basemap/queries/functions.sql @@ -0,0 +1,49 @@ +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to you under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. + +-- Converts a string to a number +CREATE OR REPLACE FUNCTION convert_to_number(input_string text, default_value numeric) + RETURNS numeric AS $$ +DECLARE + result numeric; +BEGIN + -- Replace comma with dot + input_string := REPLACE(input_string, ',', '.'); + + -- Use a regular expression to extract the first number from the string + input_string := SUBSTRING(input_string FROM '^[0-9]+\.?[0-9]*'); + + -- Convert the extracted string to a numeric type + result := input_string::numeric; + + IF result IS NULL THEN + RETURN default_value; + END IF; + + RETURN result; +EXCEPTION + WHEN others THEN + -- Return the default value in case of any error + RETURN default_value; +END; +$$ LANGUAGE plpgsql; + +-- Test cases for the convert_to_number function +SELECT assert_equals(convert_to_number('1', 0), 1); +SELECT assert_equals(convert_to_number('2.3', 0), 2.3); +SELECT assert_equals(convert_to_number('3,4', 0), 3.4); +SELECT assert_equals(convert_to_number('1.5m', 0), 1.5); +SELECT assert_equals(convert_to_number('6.6 m', 0), 6.6); +SELECT assert_equals(convert_to_number('abc', 0), 0); \ No newline at end of file
