This is an automated email from the ASF dual-hosted git repository. bchapuis pushed a commit to branch pmtiles in repository https://gitbox.apache.org/repos/asf/incubator-baremaps.git
commit 56c8ef8f113a83bf9dc5752cfaa7fba01e3405b5 Author: Bertil Chapuis <[email protected]> AuthorDate: Mon Sep 18 18:46:10 2023 +0200 Add a minimalistic tile writer --- .../org/apache/baremaps/tilestore/TileCache.java | 6 + .../org/apache/baremaps/tilestore/TileStore.java | 2 +- .../baremaps/tilestore/file/FileTileStore.java | 5 + .../baremaps/tilestore/mbtiles/MBTilesStore.java | 5 + .../baremaps/tilestore/pmtiles/Compression.java | 109 ++-- .../apache/baremaps/tilestore/pmtiles/Entry.java | 104 ++-- .../apache/baremaps/tilestore/pmtiles/Header.java | 556 ++++++++++++--------- .../apache/baremaps/tilestore/pmtiles/PMTiles.java | 263 ++++++---- .../baremaps/tilestore/pmtiles/PMTilesReader.java | 91 ++++ .../baremaps/tilestore/pmtiles/PMTilesStore.java | 61 +++ .../baremaps/tilestore/pmtiles/PMTilesWriter.java | 119 +++++ .../baremaps/tilestore/pmtiles/TileType.java | 24 +- .../tilestore/postgres/PostgresTileStore.java | 4 + .../baremaps/workflow/tasks/ExportVectorTiles.java | 34 +- .../baremaps/tilestore/pmtiles/PMTilesReader.java | 77 --- .../baremaps/tilestore/pmtiles/PMTilesTest.java | 125 ++--- 16 files changed, 983 insertions(+), 602 deletions(-) diff --git a/baremaps-core/src/main/java/org/apache/baremaps/tilestore/TileCache.java b/baremaps-core/src/main/java/org/apache/baremaps/tilestore/TileCache.java index 7381b6e9..199734a7 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/tilestore/TileCache.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/tilestore/TileCache.java @@ -84,4 +84,10 @@ public class TileCache implements TileStore { tileStore.delete(tileCoord); cache.invalidate(tileCoord); } + + @Override + public void close() throws Exception { + tileStore.close(); + cache.cleanUp(); + } } diff --git a/baremaps-core/src/main/java/org/apache/baremaps/tilestore/TileStore.java b/baremaps-core/src/main/java/org/apache/baremaps/tilestore/TileStore.java index 5d2b1a2f..1cf71a15 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/tilestore/TileStore.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/tilestore/TileStore.java @@ -22,7 +22,7 @@ package org.apache.baremaps.tilestore; import java.nio.ByteBuffer; /** Represents a store for tiles. */ -public interface TileStore { +public interface TileStore extends AutoCloseable { /** * Reads the content of a tile. diff --git a/baremaps-core/src/main/java/org/apache/baremaps/tilestore/file/FileTileStore.java b/baremaps-core/src/main/java/org/apache/baremaps/tilestore/file/FileTileStore.java index 41dd412b..5afcdafa 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/tilestore/file/FileTileStore.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/tilestore/file/FileTileStore.java @@ -77,4 +77,9 @@ public class FileTileStore implements TileStore { public Path resolve(TileCoord tileCoord) { return path.resolve(String.format("%s/%s/%s.mvt", tileCoord.z(), tileCoord.x(), tileCoord.y())); } + + @Override + public void close() { + // Do nothing + } } diff --git a/baremaps-core/src/main/java/org/apache/baremaps/tilestore/mbtiles/MBTilesStore.java b/baremaps-core/src/main/java/org/apache/baremaps/tilestore/mbtiles/MBTilesStore.java index 22baf690..22d04ecd 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/tilestore/mbtiles/MBTilesStore.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/tilestore/mbtiles/MBTilesStore.java @@ -188,4 +188,9 @@ public class MBTilesStore implements TileStore { private static int reverseY(int y, int z) { return (int) (Math.pow(2.0, z) - 1 - y); } + + @Override + public void close() throws Exception { + // Do nothing + } } diff --git a/baremaps-core/src/main/java/org/apache/baremaps/tilestore/pmtiles/Compression.java b/baremaps-core/src/main/java/org/apache/baremaps/tilestore/pmtiles/Compression.java index 4360d9c7..90613fef 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/tilestore/pmtiles/Compression.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/tilestore/pmtiles/Compression.java @@ -1,70 +1,69 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + */ + package org.apache.baremaps.tilestore.pmtiles; -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.nio.ByteBuffer; +import java.io.*; import java.util.zip.GZIPInputStream; import java.util.zip.GZIPOutputStream; enum Compression { - Unknown, - None, - Gzip, - Brotli, - Zstd; + Unknown, + None, + Gzip, + Brotli, + Zstd; - ByteBuffer decompress(ByteBuffer buffer) { - return switch (this) { - case None -> buffer; - case Gzip -> decompressGzip(buffer); - case Brotli -> decompressBrotli(buffer); - case Zstd -> decompressZstd(buffer); - default -> throw new RuntimeException("Unknown compression"); - }; - } + InputStream decompress(InputStream inputStream) throws IOException { + return switch (this) { + case None -> inputStream; + case Gzip -> decompressGzip(inputStream); + case Brotli -> decompressBrotli(inputStream); + case Zstd -> decompressZstd(inputStream); + default -> throw new RuntimeException("Unknown compression"); + }; + } - static ByteBuffer decompressGzip(ByteBuffer buffer) { - try(var inputStream = new GZIPInputStream(new ByteArrayInputStream(buffer.array()))) { - return ByteBuffer.wrap(inputStream.readAllBytes()); - } catch (IOException e) { - throw new RuntimeException(e); - } - } + static InputStream decompressGzip(InputStream inputStream) throws IOException { + return new GZIPInputStream(inputStream); + } - static ByteBuffer decompressBrotli(ByteBuffer buffer) { - throw new RuntimeException("Brotli compression not implemented"); - } + static InputStream decompressBrotli(InputStream buffer) { + throw new RuntimeException("Brotli compression not implemented"); + } - static ByteBuffer decompressZstd(ByteBuffer buffer) { - throw new RuntimeException("Zstd compression not implemented"); - } + static InputStream decompressZstd(InputStream buffer) { + throw new RuntimeException("Zstd compression not implemented"); + } - ByteBuffer compress(ByteBuffer buffer) { - return switch (this) { - case None -> buffer; - case Gzip -> compressGzip(buffer); - case Brotli -> compressBrotli(buffer); - case Zstd -> compressZstd(buffer); - default -> throw new RuntimeException("Unknown compression"); - }; - } + OutputStream compress(OutputStream outputStream) throws IOException { + return switch (this) { + case None -> outputStream; + case Gzip -> compressGzip(outputStream); + case Brotli -> compressBrotli(outputStream); + case Zstd -> compressZstd(outputStream); + default -> throw new RuntimeException("Unknown compression"); + }; + } - static ByteBuffer compressGzip(ByteBuffer buffer) { - try(var outputStream = new ByteArrayOutputStream(); - var gzipOutputStream = new GZIPOutputStream(outputStream)) { - gzipOutputStream.write(buffer.array()); - return ByteBuffer.wrap(outputStream.toByteArray()); - } catch (IOException e) { - throw new RuntimeException(e); - } - } + static OutputStream compressGzip(OutputStream outputStream) throws IOException { + return new GZIPOutputStream(outputStream); + } - static ByteBuffer compressBrotli(ByteBuffer buffer) { - throw new RuntimeException("Brotli compression not implemented"); - } + static OutputStream compressBrotli(OutputStream outputStream) { + throw new RuntimeException("Brotli compression not implemented"); + } - static ByteBuffer compressZstd(ByteBuffer buffer) { - throw new RuntimeException("Zstd compression not implemented"); - } + static OutputStream compressZstd(OutputStream outputStream) { + throw new RuntimeException("Zstd compression not implemented"); + } } diff --git a/baremaps-core/src/main/java/org/apache/baremaps/tilestore/pmtiles/Entry.java b/baremaps-core/src/main/java/org/apache/baremaps/tilestore/pmtiles/Entry.java index 97fe00e2..7d194df4 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/tilestore/pmtiles/Entry.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/tilestore/pmtiles/Entry.java @@ -1,51 +1,61 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + */ + package org.apache.baremaps.tilestore.pmtiles; public class Entry { - private long tileId; - private long offset; - private long length; - private long runLength; - - public Entry() { - - } - - public Entry(long tileId, long offset, long length, long runLength) { - this.tileId = tileId; - this.offset = offset; - this.length = length; - this.runLength = runLength; - } - - public long getTileId() { - return tileId; - } - - public void setTileId(long tileId) { - this.tileId = tileId; - } - - public long getOffset() { - return offset; - } - - public void setOffset(long offset) { - this.offset = offset; - } - - public long getLength() { - return length; - } - - public void setLength(long length) { - this.length = length; - } - - public long getRunLength() { - return runLength; - } - - public void setRunLength(long runLength) { - this.runLength = runLength; - } + private long tileId; + private long offset; + private long length; + private long runLength; + + public Entry() {} + + public Entry(long tileId, long offset, long length, long runLength) { + this.tileId = tileId; + this.offset = offset; + this.length = length; + this.runLength = runLength; + } + + public long getTileId() { + return tileId; + } + + public void setTileId(long tileId) { + this.tileId = tileId; + } + + public long getOffset() { + return offset; + } + + public void setOffset(long offset) { + this.offset = offset; + } + + public long getLength() { + return length; + } + + public void setLength(long length) { + this.length = length; + } + + public long getRunLength() { + return runLength; + } + + public void setRunLength(long runLength) { + this.runLength = runLength; + } } diff --git a/baremaps-core/src/main/java/org/apache/baremaps/tilestore/pmtiles/Header.java b/baremaps-core/src/main/java/org/apache/baremaps/tilestore/pmtiles/Header.java index d3cb42cb..8497d3a9 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/tilestore/pmtiles/Header.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/tilestore/pmtiles/Header.java @@ -1,258 +1,318 @@ -package org.apache.baremaps.tilestore.pmtiles; - -public class Header { - - private int specVersion; - private long rootDirectoryOffset; - private long rootDirectoryLength; - private long jsonMetadataOffset; - private long jsonMetadataLength; - private long leafDirectoryOffset; - private long leafDirectoryLength; - private long tileDataOffset; - private long tileDataLength; - private long numAddressedTiles; - private long numTileEntries; - private long numTileContents; - private boolean clustered; - private Compression internalCompression; - private Compression tileCompression; - private TileType tileType; - private int minZoom; - private int maxZoom; - private double minLon; - private double minLat; - private double maxLon; - private double maxLat; - private int centerZoom; - private double centerLon; - private double centerLat; - - public Header(int specVersion, long rootDirectoryOffset, long rootDirectoryLength, long jsonMetadataOffset, long jsonMetadataLength, long leafDirectoryOffset, long leafDirectoryLength, long tileDataOffset, long tileDataLength, long numAddressedTiles, long numTileEntries, long numTileContents, boolean clustered, Compression internalCompression, Compression tileCompression, TileType tileType, int minZoom, int maxZoom, double minLon, double minLat, double maxLon, double maxLat, int cent [...] - this.specVersion = specVersion; - this.rootDirectoryOffset = rootDirectoryOffset; - this.rootDirectoryLength = rootDirectoryLength; - this.jsonMetadataOffset = jsonMetadataOffset; - this.jsonMetadataLength = jsonMetadataLength; - this.leafDirectoryOffset = leafDirectoryOffset; - this.leafDirectoryLength = leafDirectoryLength; - this.tileDataOffset = tileDataOffset; - this.tileDataLength = tileDataLength; - this.numAddressedTiles = numAddressedTiles; - this.numTileEntries = numTileEntries; - this.numTileContents = numTileContents; - this.clustered = clustered; - this.internalCompression = internalCompression; - this.tileCompression = tileCompression; - this.tileType = tileType; - this.minZoom = minZoom; - this.maxZoom = maxZoom; - this.minLon = minLon; - this.minLat = minLat; - this.maxLon = maxLon; - this.maxLat = maxLat; - this.centerZoom = centerZoom; - this.centerLon = centerLon; - this.centerLat = centerLat; - } - - public int getSpecVersion() { - return specVersion; - } - - public void setSpecVersion(int specVersion) { - this.specVersion = specVersion; - } - - public long getRootDirectoryOffset() { - return rootDirectoryOffset; - } - - public void setRootDirectoryOffset(long rootDirectoryOffset) { - this.rootDirectoryOffset = rootDirectoryOffset; - } - - public long getRootDirectoryLength() { - return rootDirectoryLength; - } - - public void setRootDirectoryLength(long rootDirectoryLength) { - this.rootDirectoryLength = rootDirectoryLength; - } - - public long getJsonMetadataOffset() { - return jsonMetadataOffset; - } - - public void setJsonMetadataOffset(long jsonMetadataOffset) { - this.jsonMetadataOffset = jsonMetadataOffset; - } - - public long getJsonMetadataLength() { - return jsonMetadataLength; - } - - public void setJsonMetadataLength(long jsonMetadataLength) { - this.jsonMetadataLength = jsonMetadataLength; - } - - public long getLeafDirectoryOffset() { - return leafDirectoryOffset; - } - - public void setLeafDirectoryOffset(long leafDirectoryOffset) { - this.leafDirectoryOffset = leafDirectoryOffset; - } - - public long getLeafDirectoryLength() { - return leafDirectoryLength; - } - - public void setLeafDirectoryLength(long leafDirectoryLength) { - this.leafDirectoryLength = leafDirectoryLength; - } - - public long getTileDataOffset() { - return tileDataOffset; - } - - public void setTileDataOffset(long tileDataOffset) { - this.tileDataOffset = tileDataOffset; - } - - public long getTileDataLength() { - return tileDataLength; - } - - public void setTileDataLength(long tileDataLength) { - this.tileDataLength = tileDataLength; - } - - public long getNumAddressedTiles() { - return numAddressedTiles; - } - - public void setNumAddressedTiles(long numAddressedTiles) { - this.numAddressedTiles = numAddressedTiles; - } - - public long getNumTileEntries() { - return numTileEntries; - } - - public void setNumTileEntries(long numTileEntries) { - this.numTileEntries = numTileEntries; - } +/* + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + */ - public long getNumTileContents() { - return numTileContents; - } - - public void setNumTileContents(long numTileContents) { - this.numTileContents = numTileContents; - } - - public boolean isClustered() { - return clustered; - } - - public void setClustered(boolean clustered) { - this.clustered = clustered; - } - - public Compression getInternalCompression() { - return internalCompression; - } - - public void setInternalCompression(Compression internalCompression) { - this.internalCompression = internalCompression; - } - - public Compression getTileCompression() { - return tileCompression; - } - - public void setTileCompression(Compression tileCompression) { - this.tileCompression = tileCompression; - } - - public TileType getTileType() { - return tileType; - } - - public void setTileType(TileType tileType) { - this.tileType = tileType; - } - - public int getMinZoom() { - return minZoom; - } - - public void setMinZoom(int minZoom) { - this.minZoom = minZoom; - } - - public int getMaxZoom() { - return maxZoom; - } - - public void setMaxZoom(int maxZoom) { - this.maxZoom = maxZoom; - } - - public double getMinLon() { - return minLon; - } - - public void setMinLon(double minLon) { - this.minLon = minLon; - } - - public double getMinLat() { - return minLat; - } - - public void setMinLat(double minLat) { - this.minLat = minLat; - } - - public double getMaxLon() { - return maxLon; - } - - public void setMaxLon(double maxLon) { - this.maxLon = maxLon; - } +package org.apache.baremaps.tilestore.pmtiles; - public double getMaxLat() { - return maxLat; - } +import java.util.Objects; - public void setMaxLat(double maxLat) { - this.maxLat = maxLat; - } +public class Header { - public int getCenterZoom() { - return centerZoom; - } + private int specVersion; + private long rootDirectoryOffset; + private long rootDirectoryLength; + private long jsonMetadataOffset; + private long jsonMetadataLength; + private long leafDirectoryOffset; + private long leafDirectoryLength; + private long tileDataOffset; + private long tileDataLength; + private long numAddressedTiles; + private long numTileEntries; + private long numTileContents; + private boolean clustered; + private Compression internalCompression; + private Compression tileCompression; + private TileType tileType; + private int minZoom; + private int maxZoom; + private double minLon; + private double minLat; + private double maxLon; + private double maxLat; + private int centerZoom; + private double centerLon; + private double centerLat; + + public Header() { + this.specVersion = 3; + } + + public Header(int specVersion, long rootDirectoryOffset, long rootDirectoryLength, + long jsonMetadataOffset, long jsonMetadataLength, long leafDirectoryOffset, + long leafDirectoryLength, long tileDataOffset, long tileDataLength, long numAddressedTiles, + long numTileEntries, long numTileContents, boolean clustered, Compression internalCompression, + Compression tileCompression, TileType tileType, int minZoom, int maxZoom, double minLon, + double minLat, double maxLon, double maxLat, int centerZoom, double centerLon, + double centerLat) { + this.specVersion = specVersion; + this.rootDirectoryOffset = rootDirectoryOffset; + this.rootDirectoryLength = rootDirectoryLength; + this.jsonMetadataOffset = jsonMetadataOffset; + this.jsonMetadataLength = jsonMetadataLength; + this.leafDirectoryOffset = leafDirectoryOffset; + this.leafDirectoryLength = leafDirectoryLength; + this.tileDataOffset = tileDataOffset; + this.tileDataLength = tileDataLength; + this.numAddressedTiles = numAddressedTiles; + this.numTileEntries = numTileEntries; + this.numTileContents = numTileContents; + this.clustered = clustered; + this.internalCompression = internalCompression; + this.tileCompression = tileCompression; + this.tileType = tileType; + this.minZoom = minZoom; + this.maxZoom = maxZoom; + this.minLon = minLon; + this.minLat = minLat; + this.maxLon = maxLon; + this.maxLat = maxLat; + this.centerZoom = centerZoom; + this.centerLon = centerLon; + this.centerLat = centerLat; + } + + public int getSpecVersion() { + return specVersion; + } + + public void setSpecVersion(int specVersion) { + this.specVersion = specVersion; + } + + public long getRootDirectoryOffset() { + return rootDirectoryOffset; + } + + public void setRootOffset(long rootDirectoryOffset) { + this.rootDirectoryOffset = rootDirectoryOffset; + } + + public long getRootDirectoryLength() { + return rootDirectoryLength; + } + + public void setRootLength(long rootDirectoryLength) { + this.rootDirectoryLength = rootDirectoryLength; + } + + public long getJsonMetadataOffset() { + return jsonMetadataOffset; + } + + public void setMetadataOffset(long jsonMetadataOffset) { + this.jsonMetadataOffset = jsonMetadataOffset; + } + + public long getJsonMetadataLength() { + return jsonMetadataLength; + } + + public void setMetadataLength(long jsonMetadataLength) { + this.jsonMetadataLength = jsonMetadataLength; + } + + public long getLeafDirectoryOffset() { + return leafDirectoryOffset; + } + + public void setLeavesOffset(long leafDirectoryOffset) { + this.leafDirectoryOffset = leafDirectoryOffset; + } + + public long getLeafDirectoryLength() { + return leafDirectoryLength; + } + + public void setLeavesLength(long leafDirectoryLength) { + this.leafDirectoryLength = leafDirectoryLength; + } + + public long getTileDataOffset() { + return tileDataOffset; + } + + public void setTilesOffset(long tileDataOffset) { + this.tileDataOffset = tileDataOffset; + } + + public long getTileDataLength() { + return tileDataLength; + } + + public void setTilesLength(long tileDataLength) { + this.tileDataLength = tileDataLength; + } + + public long getNumAddressedTiles() { + return numAddressedTiles; + } + + public void setNumAddressedTiles(long numAddressedTiles) { + this.numAddressedTiles = numAddressedTiles; + } + + public long getNumTileEntries() { + return numTileEntries; + } + + public void setNumTileEntries(long numTileEntries) { + this.numTileEntries = numTileEntries; + } + + public long getNumTileContents() { + return numTileContents; + } + + public void setNumTileContents(long numTileContents) { + this.numTileContents = numTileContents; + } - public void setCenterZoom(int centerZoom) { - this.centerZoom = centerZoom; - } + public boolean isClustered() { + return clustered; + } - public double getCenterLon() { - return centerLon; - } + public void setClustered(boolean clustered) { + this.clustered = clustered; + } - public void setCenterLon(double centerLon) { - this.centerLon = centerLon; - } + public Compression getInternalCompression() { + return internalCompression; + } - public double getCenterLat() { - return centerLat; - } + public void setInternalCompression(Compression internalCompression) { + this.internalCompression = internalCompression; + } - public void setCenterLat(double centerLat) { - this.centerLat = centerLat; - } + public Compression getTileCompression() { + return tileCompression; + } + + public void setTileCompression(Compression tileCompression) { + this.tileCompression = tileCompression; + } + + public TileType getTileType() { + return tileType; + } + + public void setTileType(TileType tileType) { + this.tileType = tileType; + } + + public int getMinZoom() { + return minZoom; + } + + public void setMinZoom(int minZoom) { + this.minZoom = minZoom; + } + + public int getMaxZoom() { + return maxZoom; + } + + public void setMaxZoom(int maxZoom) { + this.maxZoom = maxZoom; + } + + public double getMinLon() { + return minLon; + } + + public void setMinLon(double minLon) { + this.minLon = minLon; + } + + public double getMinLat() { + return minLat; + } + + public void setMinLat(double minLat) { + this.minLat = minLat; + } + + public double getMaxLon() { + return maxLon; + } + + public void setMaxLon(double maxLon) { + this.maxLon = maxLon; + } + + public double getMaxLat() { + return maxLat; + } + + public void setMaxLat(double maxLat) { + this.maxLat = maxLat; + } + + public int getCenterZoom() { + return centerZoom; + } + + public void setCenterZoom(int centerZoom) { + this.centerZoom = centerZoom; + } + + public double getCenterLon() { + return centerLon; + } + + public void setCenterLon(double centerLon) { + this.centerLon = centerLon; + } + + public double getCenterLat() { + return centerLat; + } + + public void setCenterLat(double centerLat) { + this.centerLat = centerLat; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + Header header = (Header) o; + return specVersion == header.specVersion && rootDirectoryOffset == header.rootDirectoryOffset + && rootDirectoryLength == header.rootDirectoryLength + && jsonMetadataOffset == header.jsonMetadataOffset + && jsonMetadataLength == header.jsonMetadataLength + && leafDirectoryOffset == header.leafDirectoryOffset + && leafDirectoryLength == header.leafDirectoryLength + && tileDataOffset == header.tileDataOffset && tileDataLength == header.tileDataLength + && numAddressedTiles == header.numAddressedTiles && numTileEntries == header.numTileEntries + && numTileContents == header.numTileContents && clustered == header.clustered + && minZoom == header.minZoom && maxZoom == header.maxZoom + && Double.compare(header.minLon, minLon) == 0 && Double.compare(header.minLat, minLat) == 0 + && Double.compare(header.maxLon, maxLon) == 0 && Double.compare(header.maxLat, maxLat) == 0 + && centerZoom == header.centerZoom && Double.compare(header.centerLon, centerLon) == 0 + && Double.compare(header.centerLat, centerLat) == 0 + && internalCompression == header.internalCompression + && tileCompression == header.tileCompression && tileType == header.tileType; + } + + @Override + public int hashCode() { + return Objects.hash(specVersion, rootDirectoryOffset, rootDirectoryLength, jsonMetadataOffset, + jsonMetadataLength, leafDirectoryOffset, leafDirectoryLength, tileDataOffset, + tileDataLength, numAddressedTiles, numTileEntries, numTileContents, clustered, + internalCompression, tileCompression, tileType, minZoom, maxZoom, minLon, minLat, maxLon, + maxLat, centerZoom, centerLon, centerLat); + } } diff --git a/baremaps-core/src/main/java/org/apache/baremaps/tilestore/pmtiles/PMTiles.java b/baremaps-core/src/main/java/org/apache/baremaps/tilestore/pmtiles/PMTiles.java index 7d260a5b..b81bf7cb 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/tilestore/pmtiles/PMTiles.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/tilestore/pmtiles/PMTiles.java @@ -12,9 +12,11 @@ package org.apache.baremaps.tilestore.pmtiles; +import com.google.common.io.LittleEndianDataInputStream; +import com.google.common.io.LittleEndianDataOutputStream; import com.google.common.math.LongMath; -import java.nio.ByteBuffer; -import java.nio.ByteOrder; +import java.io.ByteArrayOutputStream; +import java.io.IOException; import java.util.ArrayList; import java.util.List; @@ -24,34 +26,35 @@ public class PMTiles { return high * 0x100000000L + low; } - public static long readVarIntRemainder(long l, ByteBuffer buf) { + public static long readVarIntRemainder(LittleEndianDataInputStream input, long l) + throws IOException { long h, b; - b = buf.get() & 0xff; + b = input.readByte() & 0xff; h = (b & 0x70) >> 4; if (b < 0x80) { return toNum(l, h); } - b = buf.get() & 0xff; + b = input.readByte() & 0xff; h |= (b & 0x7f) << 3; if (b < 0x80) { return toNum(l, h); } - b = buf.get() & 0xff; + b = input.readByte() & 0xff; h |= (b & 0x7f) << 10; if (b < 0x80) { return toNum(l, h); } - b = buf.get() & 0xff; + b = input.readByte() & 0xff; h |= (b & 0x7f) << 17; if (b < 0x80) { return toNum(l, h); } - b = buf.get() & 0xff; + b = input.readByte() & 0xff; h |= (b & 0x7f) << 24; if (b < 0x80) { return toNum(l, h); } - b = buf.get() & 0xff; + b = input.readByte() & 0xff; h |= (b & 0x01) << 31; if (b < 0x80) { return toNum(l, h); @@ -59,41 +62,42 @@ public class PMTiles { throw new RuntimeException("Expected varint not more than 10 bytes"); } - public static int encodeVarInt(ByteBuffer buf, long value) { + public static int writeVarInt(LittleEndianDataOutputStream output, long value) + throws IOException { int n = 1; while (value >= 0x80) { - buf.put((byte) (value | 0x80)); + output.writeByte((byte) (value | 0x80)); value >>>= 7; n++; } - buf.put((byte) value); + output.writeByte((byte) value); return n; } - public static long decodeVarInt(ByteBuffer buf) { + public static long readVarInt(LittleEndianDataInputStream input) throws IOException { long val, b; - b = buf.get() & 0xff; + b = input.readByte() & 0xff; val = b & 0x7f; if (b < 0x80) { return val; } - b = buf.get() & 0xff; + b = input.readByte() & 0xff; val |= (b & 0x7f) << 7; if (b < 0x80) { return val; } - b = buf.get() & 0xff; + b = input.readByte() & 0xff; val |= (b & 0x7f) << 14; if (b < 0x80) { return val; } - b = buf.get() & 0xff; + b = input.readByte() & 0xff; val |= (b & 0x7f) << 21; if (b < 0x80) { return val; } val |= (b & 0x0f) << 28; - return readVarIntRemainder(val, buf); + return readVarIntRemainder(input, val); } public static void rotate(long n, long[] xy, long rx, long ry) { @@ -170,119 +174,119 @@ public class PMTiles { private static final int HEADER_SIZE_BYTES = 127; - public static Header decodeHeader(ByteBuffer buf) { - buf.order(ByteOrder.LITTLE_ENDIAN); + public static Header deserializeHeader(LittleEndianDataInputStream input) throws IOException { + input.skipBytes(7); return new Header( - buf.get(7), - buf.getLong(8), - buf.getLong(16), - buf.getLong(24), - buf.getLong(32), - buf.getLong(40), - buf.getLong(48), - buf.getLong(56), - buf.getLong(64), - buf.getLong(72), - buf.getLong(80), - buf.getLong(88), - buf.get(96) == 1, - Compression.values()[buf.get(97)], - Compression.values()[buf.get(98)], - TileType.values()[buf.get(99)], - buf.get(100), - buf.get(101), - (double) buf.getInt(102) / 10000000, - (double) buf.getInt(106) / 10000000, - (double) buf.getInt(110) / 10000000, - (double) buf.getInt(114) / 10000000, - buf.get(118), - (double) buf.getInt(119) / 10000000, - (double) buf.getInt(123) / 10000000 - ); + input.readByte(), + input.readLong(), + input.readLong(), + input.readLong(), + input.readLong(), + input.readLong(), + input.readLong(), + input.readLong(), + input.readLong(), + input.readLong(), + input.readLong(), + input.readLong(), + input.readByte() == 1, + Compression.values()[input.readByte()], + Compression.values()[input.readByte()], + TileType.values()[input.readByte()], + input.readByte(), + input.readByte(), + (double) input.readInt() / 10000000, + (double) input.readInt() / 10000000, + (double) input.readInt() / 10000000, + (double) input.readInt() / 10000000, + input.readByte(), + (double) input.readInt() / 10000000, + (double) input.readInt() / 10000000); } - public static void encodeHeader(Header header, ByteBuffer buf) { - buf.order(ByteOrder.LITTLE_ENDIAN); - buf.put(0, (byte) 0x50); - buf.put(1, (byte) 0x4D); - buf.put(2, (byte) 0x54); - buf.put(3, (byte) 0x69); - buf.put(4, (byte) 0x6C); - buf.put(5, (byte) 0x65); - buf.put(6, (byte) 0x73); - buf.put(7, (byte) header.getSpecVersion()); - buf.putLong(8, header.getRootDirectoryOffset()); - buf.putLong(16, header.getRootDirectoryLength()); - buf.putLong(24, header.getJsonMetadataOffset()); - buf.putLong(32, header.getJsonMetadataLength()); - buf.putLong(40, header.getLeafDirectoryOffset()); - buf.putLong(48, header.getLeafDirectoryLength()); - buf.putLong(56, header.getTileDataOffset()); - buf.putLong(64, header.getTileDataLength()); - buf.putLong(72, header.getNumAddressedTiles()); - buf.putLong(80, header.getNumTileEntries()); - buf.putLong(88, header.getNumTileContents()); - buf.put(96, (byte) (header.isClustered() ? 1 : 0)); - buf.put(97, (byte) header.getInternalCompression().ordinal()); - buf.put(98, (byte) header.getTileCompression().ordinal()); - buf.put(99, (byte) header.getTileType().ordinal()); - buf.put(100, (byte) header.getMinZoom()); - buf.put(101, (byte) header.getMaxZoom()); - buf.putInt(102, (int) (header.getMinLon() * 10000000)); - buf.putInt(106, (int) (header.getMinLat() * 10000000)); - buf.putInt(110, (int) (header.getMaxLon() * 10000000)); - buf.putInt(114, (int) (header.getMaxLat() * 10000000)); - buf.put(118, (byte) header.getCenterZoom()); - buf.putInt(119, (int) (header.getCenterLon() * 10000000)); - buf.putInt(123, (int) (header.getCenterLat() * 10000000)); + public static void serializeHeader(LittleEndianDataOutputStream output, Header header) + throws IOException { + output.writeByte((byte) 0x50); + output.writeByte((byte) 0x4D); + output.writeByte((byte) 0x54); + output.writeByte((byte) 0x69); + output.writeByte((byte) 0x6C); + output.writeByte((byte) 0x65); + output.writeByte((byte) 0x73); + output.writeByte((byte) header.getSpecVersion()); + output.writeLong(header.getRootDirectoryOffset()); + output.writeLong(header.getRootDirectoryLength()); + output.writeLong(header.getJsonMetadataOffset()); + output.writeLong(header.getJsonMetadataLength()); + output.writeLong(header.getLeafDirectoryOffset()); + output.writeLong(header.getLeafDirectoryLength()); + output.writeLong(header.getTileDataOffset()); + output.writeLong(header.getTileDataLength()); + output.writeLong(header.getNumAddressedTiles()); + output.writeLong(header.getNumTileEntries()); + output.writeLong(header.getNumTileContents()); + output.writeByte((byte) (header.isClustered() ? 1 : 0)); + output.writeByte((byte) header.getInternalCompression().ordinal()); + output.writeByte((byte) header.getTileCompression().ordinal()); + output.writeByte((byte) header.getTileType().ordinal()); + output.writeByte((byte) header.getMinZoom()); + output.writeByte((byte) header.getMaxZoom()); + output.writeInt((int) (header.getMinLon() * 10000000)); + output.writeInt((int) (header.getMinLat() * 10000000)); + output.writeInt((int) (header.getMaxLon() * 10000000)); + output.writeInt((int) (header.getMaxLat() * 10000000)); + output.writeByte((byte) header.getCenterZoom()); + output.writeInt((int) (header.getCenterLon() * 10000000)); + output.writeInt((int) (header.getCenterLat() * 10000000)); } - public static void encodeDirectory(ByteBuffer buffer, List<Entry> entries) { - buffer.order(ByteOrder.LITTLE_ENDIAN); - encodeVarInt(buffer, entries.size()); + public static void serializeEntries(LittleEndianDataOutputStream output, List<Entry> entries) + throws IOException { + writeVarInt(output, entries.size()); long lastId = 0; for (Entry entry : entries) { - encodeVarInt(buffer, entry.getTileId() - lastId); + writeVarInt(output, entry.getTileId() - lastId); lastId = entry.getTileId(); } for (Entry entry : entries) { - encodeVarInt(buffer, entry.getRunLength()); + writeVarInt(output, entry.getRunLength()); } for (Entry entry : entries) { - encodeVarInt(buffer, entry.getLength()); + writeVarInt(output, entry.getLength()); } - for (Entry entry : entries) { - if (entry.getOffset() == 0 && entry.getLength() > 0) { - Entry prevEntry = entries.get(entries.indexOf(entry) - 1); - encodeVarInt(buffer, prevEntry.getOffset() + prevEntry.getLength() + 1); + for (int i = 0; i < entries.size(); i++) { + Entry entry = entries.get(i); + if (i > 0 + && entry.getOffset() == entries.get(i - 1).getOffset() + entries.get(i - 1).getLength()) { + writeVarInt(output, 0); } else { - encodeVarInt(buffer, entry.getOffset() + 1); + writeVarInt(output, entry.getOffset() + 1); } } } - public static List<Entry> decodeDirectory(ByteBuffer buffer) { - buffer.order(ByteOrder.LITTLE_ENDIAN); - long numEntries = decodeVarInt(buffer); + public static List<Entry> deserializeEntries(LittleEndianDataInputStream buffer) + throws IOException { + long numEntries = readVarInt(buffer); List<Entry> entries = new ArrayList<>((int) numEntries); long lastId = 0; for (int i = 0; i < numEntries; i++) { - long value = decodeVarInt(buffer); + long value = readVarInt(buffer); lastId = lastId + value; Entry entry = new Entry(); entry.setTileId(lastId); entries.add(entry); } for (int i = 0; i < numEntries; i++) { - long value = decodeVarInt(buffer); + long value = readVarInt(buffer); entries.get(i).setRunLength(value); } for (int i = 0; i < numEntries; i++) { - long value = decodeVarInt(buffer); + long value = readVarInt(buffer); entries.get(i).setLength(value); } for (int i = 0; i < numEntries; i++) { - long value = decodeVarInt(buffer); + long value = readVarInt(buffer); if (value == 0 && i > 0) { Entry prevEntry = entries.get(i - 1); entries.get(i).setOffset(prevEntry.getOffset() + prevEntry.getLength());; @@ -320,4 +324,67 @@ public class PMTiles { return null; } + record Directories(byte[] root, byte[] leaves, int numLeaves) { + } + + public static Directories buildRootLeaves(List<Entry> entries, int leafSize) throws IOException { + var rootEntries = new ArrayList<Entry>(); + var numLeaves = 0; + byte[] leavesBytes = null; + byte[] rootBytes = null; + + try (var leavesOutput = new ByteArrayOutputStream(); + var leavesDataOutput = new LittleEndianDataOutputStream(leavesOutput)) { + for (var i = 0; i < entries.size(); i += leafSize) { + numLeaves++; + var end = i + leafSize; + if (i + leafSize > entries.size()) { + end = entries.size(); + } + + var offset = leavesOutput.size(); + serializeEntries(leavesDataOutput, entries.subList(i, end)); + var length = leavesOutput.size(); + rootEntries.add(new Entry(entries.get(i).getTileId(), offset, length, 0)); + } + + leavesBytes = leavesOutput.toByteArray(); + } + + try (var rootOutput = new ByteArrayOutputStream(); + var rootDataOutput = new LittleEndianDataOutputStream(rootOutput)) { + serializeEntries(rootDataOutput, rootEntries); + rootBytes = rootOutput.toByteArray(); + } + + return new Directories(rootBytes, leavesBytes, numLeaves); + } + + public static Directories optimizeDirectories(List<Entry> entries, int targetRootLenght) + throws IOException { + if (entries.size() < 16384) { + byte[] rootBytes = null; + try (var rootOutput = new ByteArrayOutputStream(); + var rootDataOutput = new LittleEndianDataOutputStream(rootOutput)) { + serializeEntries(rootDataOutput, entries); + rootBytes = rootOutput.toByteArray(); + } + if (rootBytes.length <= targetRootLenght) { + return new Directories(rootBytes, new byte[] {}, 0); + } + } + + double leafSize = (double) entries.size() / 3500d; + if (leafSize < 4096d) { + leafSize = 4096d; + } + for (;;) { + var directories = buildRootLeaves(entries, (int) leafSize); + if (directories.root.length <= targetRootLenght) { + return directories; + } + leafSize = leafSize * 1.2; + } + + } } diff --git a/baremaps-core/src/main/java/org/apache/baremaps/tilestore/pmtiles/PMTilesReader.java b/baremaps-core/src/main/java/org/apache/baremaps/tilestore/pmtiles/PMTilesReader.java new file mode 100644 index 00000000..ca0d45cf --- /dev/null +++ b/baremaps-core/src/main/java/org/apache/baremaps/tilestore/pmtiles/PMTilesReader.java @@ -0,0 +1,91 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + */ + +package org.apache.baremaps.tilestore.pmtiles; + +import com.google.common.io.LittleEndianDataInputStream; +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.channels.FileChannel; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; + +public class PMTilesReader { + + private final Path path; + + private Header header; + + private List<Entry> rootEntries; + + public PMTilesReader(Path path) { + this.path = path; + } + + public Header getHeader() { + if (header == null) { + try (var inputStream = new LittleEndianDataInputStream(Files.newInputStream(path))) { + header = PMTiles.deserializeHeader(inputStream); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + return header; + } + + public List<Entry> getRootDirectory() { + if (rootEntries == null) { + var header = getHeader(); + rootEntries = + getDirectory(header.getRootDirectoryOffset(), (int) header.getRootDirectoryLength()); + } + return rootEntries; + } + + public List<Entry> getDirectory(long offset, int length) { + var header = getHeader(); + try (var input = Files.newInputStream(path)) { + input.skip(offset); + try (var decompressed = + new LittleEndianDataInputStream(header.getInternalCompression().decompress(input))) { + return PMTiles.deserializeEntries(decompressed); + } + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + public ByteBuffer getTile(int z, long x, long y) { + var tileId = PMTiles.zxyToTileId(z, x, y); + var header = getHeader(); + var entries = getRootDirectory(); + var entry = PMTiles.findTile(entries, tileId); + + if (entry == null) { + return null; + } + + try (var channel = FileChannel.open(path)) { + var compressed = ByteBuffer.allocate((int) entry.getLength()); + channel.position(header.getTileDataOffset() + entry.getOffset()); + channel.read(compressed); + compressed.flip(); + try (var tile = new ByteArrayInputStream(compressed.array())) { + return ByteBuffer.wrap(tile.readAllBytes()); + } + } catch (IOException e) { + throw new RuntimeException(e); + } + } +} diff --git a/baremaps-core/src/main/java/org/apache/baremaps/tilestore/pmtiles/PMTilesStore.java b/baremaps-core/src/main/java/org/apache/baremaps/tilestore/pmtiles/PMTilesStore.java new file mode 100644 index 00000000..5047e333 --- /dev/null +++ b/baremaps-core/src/main/java/org/apache/baremaps/tilestore/pmtiles/PMTilesStore.java @@ -0,0 +1,61 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + */ + +package org.apache.baremaps.tilestore.pmtiles; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.file.Path; +import org.apache.baremaps.tilestore.TileCoord; +import org.apache.baremaps.tilestore.TileStore; +import org.apache.baremaps.tilestore.TileStoreException; + +public class PMTilesStore implements TileStore { + + private final PMTilesWriter writer; + + public PMTilesStore(Path path) { + try { + this.writer = new PMTilesWriter(path); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + @Override + public ByteBuffer read(TileCoord tileCoord) throws TileStoreException { + throw new UnsupportedOperationException(); + } + + @Override + public void write(TileCoord tileCoord, ByteBuffer blob) throws TileStoreException { + try { + writer.writeTile(tileCoord.z(), tileCoord.x(), tileCoord.y(), blob.array()); + } catch (IOException e) { + throw new TileStoreException(e); + } + } + + @Override + public void delete(TileCoord tileCoord) throws TileStoreException { + throw new UnsupportedOperationException(); + } + + @Override + public void close() throws TileStoreException { + try { + writer.finalize(); + } catch (IOException e) { + throw new TileStoreException(e); + } + } +} diff --git a/baremaps-core/src/main/java/org/apache/baremaps/tilestore/pmtiles/PMTilesWriter.java b/baremaps-core/src/main/java/org/apache/baremaps/tilestore/pmtiles/PMTilesWriter.java new file mode 100644 index 00000000..267f6047 --- /dev/null +++ b/baremaps-core/src/main/java/org/apache/baremaps/tilestore/pmtiles/PMTilesWriter.java @@ -0,0 +1,119 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + */ + +package org.apache.baremaps.tilestore.pmtiles; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.io.LittleEndianDataOutputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.*; + +public class PMTilesWriter { + + private final Path path; + + private List<Entry> entries; + + private Path tilePath; + + private boolean clustered = true; + + public PMTilesWriter(Path path) throws IOException { + this.path = path; + this.entries = new ArrayList<>(); + this.tilePath = Files.createTempFile(path.getParent(), "tiles", ".tmp"); + } + + public void writeTile(int z, int x, int y, byte[] bytes) throws IOException { + var tileId = PMTiles.zxyToTileId(z, x, y); + var offset = Files.size(tilePath); + var length = bytes.length; + + if (entries.size() > 0 && tileId < entries.get(entries.size() - 1).getTileId()) { + clustered = false; + } + + try (var output = new FileOutputStream(tilePath.toFile(), true)) { + output.write(bytes); + entries.add(new Entry(tileId, offset, length, 1)); + } + } + + public void finalize() throws IOException { + // Sort the entries by tile id + if (!clustered) { + entries.sort(Comparator.comparingLong(Entry::getTileId)); + } + + var metadataMap = new HashMap<String, Object>(); + metadataMap.put("name", "baremaps"); + metadataMap.put("type", "baselayer"); + metadataMap.put("version", "0.0.1"); + metadataMap.put("description", "PMTiles generated by Baremaps"); + metadataMap.put("attribution", "OpenStreetMap contributors"); + metadataMap.put("vector_layers", List.of("aerialway", "aeroway", "amenity", "attraction", + "barrier", "boundary", "building", "highway", "landuse", + "leisure", "man_made", "natural", "ocean", "point", + "power", "railway", "route", "waterway") + .stream().map(s -> Map.of("id", s)).toList()); + + var metadata = new ObjectMapper().writeValueAsBytes(metadataMap); + + + var directories = PMTiles.optimizeDirectories(entries, 16247); + long rootOffset = 127; + long rootLength = directories.root().length; + long metadataOffset = rootOffset + rootLength; + long metadataLength = metadata.length; + long leavesOffset = metadataOffset + metadataLength; + long leavesLength = directories.leaves().length; + long tilesOffset = leavesOffset + leavesLength; + long tilesLength = Files.size(tilePath); + int minZoom = (int) PMTiles.tileIdToZxy(entries.get(0).getTileId())[0]; + int maxZoom = (int) PMTiles.tileIdToZxy(entries.get(entries.size() - 1).getTileId())[0]; + var numTiles = entries.size(); + + var header = new Header(); + header.setNumAddressedTiles(numTiles); + header.setNumTileEntries(numTiles); + header.setNumTileContents(numTiles); + header.setClustered(true); + header.setInternalCompression(Compression.None); + header.setTileCompression(Compression.Gzip); + header.setTileType(TileType.Mvt); + header.setMinZoom(minZoom); + header.setMaxZoom(maxZoom); + header.setRootOffset(rootOffset); + header.setRootLength(rootLength); + header.setMetadataOffset(metadataOffset); + header.setMetadataLength(metadataLength); + header.setLeavesOffset(leavesOffset); + header.setLeavesLength(leavesLength); + header.setTilesOffset(tilesOffset); + header.setTilesLength(tilesLength); + + + try (var output = new LittleEndianDataOutputStream(new FileOutputStream(path.toFile()))) { + PMTiles.serializeHeader(output, header); + output.write(directories.root()); + output.write(metadata); + output.write(directories.leaves()); + Files.copy(tilePath, output); + } finally { + Files.delete(tilePath); + } + } + +} diff --git a/baremaps-core/src/main/java/org/apache/baremaps/tilestore/pmtiles/TileType.java b/baremaps-core/src/main/java/org/apache/baremaps/tilestore/pmtiles/TileType.java index 10bf41c8..f52673c1 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/tilestore/pmtiles/TileType.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/tilestore/pmtiles/TileType.java @@ -1,10 +1,22 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + */ + package org.apache.baremaps.tilestore.pmtiles; enum TileType { - Unknown, - Mvt, - Png, - Jpeg, - Webp, - Avif, + Unknown, + Mvt, + Png, + Jpeg, + Webp, + Avif, } diff --git a/baremaps-core/src/main/java/org/apache/baremaps/tilestore/postgres/PostgresTileStore.java b/baremaps-core/src/main/java/org/apache/baremaps/tilestore/postgres/PostgresTileStore.java index b5edbd83..9bd71cc0 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/tilestore/postgres/PostgresTileStore.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/tilestore/postgres/PostgresTileStore.java @@ -263,4 +263,8 @@ public class PostgresTileStore implements TileStore { throw new UnsupportedOperationException("The postgis tile store is read only"); } + @Override + public void close() throws Exception { + // do nothing + } } diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExportVectorTiles.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExportVectorTiles.java index 06d1455c..7a829bce 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExportVectorTiles.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExportVectorTiles.java @@ -34,6 +34,7 @@ import org.apache.baremaps.stream.StreamUtils; import org.apache.baremaps.tilestore.*; import org.apache.baremaps.tilestore.file.FileTileStore; import org.apache.baremaps.tilestore.mbtiles.MBTilesStore; +import org.apache.baremaps.tilestore.pmtiles.PMTilesStore; import org.apache.baremaps.tilestore.postgres.PostgresTileStore; import org.apache.baremaps.utils.SqliteUtils; import org.apache.baremaps.vectortile.tileset.Tileset; @@ -53,7 +54,8 @@ public record ExportVectorTiles( public enum Format { file, - mbtiles + mbtiles, + pmtiles } private static final Logger logger = LoggerFactory.getLogger(ExportVectorTiles.class); @@ -65,22 +67,24 @@ public record ExportVectorTiles( var tileset = objectMapper.readValue(configReader.read(this.tileset), Tileset.class); var datasource = context.getDataSource(tileset.getDatabase()); - var sourceTileStore = sourceTileStore(tileset, datasource); - var targetTileStore = targetTileStore(tileset); + try (var sourceTileStore = sourceTileStore(tileset, datasource); + var targetTileStore = targetTileStore(tileset)) { - var envelope = tileset.getBounds().size() == 4 - ? new Envelope( - tileset.getBounds().get(0), tileset.getBounds().get(2), - tileset.getBounds().get(1), tileset.getBounds().get(3)) - : new Envelope(-180, 180, -85.0511, 85.0511); + var envelope = tileset.getBounds().size() == 4 + ? new Envelope( + tileset.getBounds().get(0), tileset.getBounds().get(2), + tileset.getBounds().get(1), tileset.getBounds().get(3)) + : new Envelope(-180, 180, -85.0511, 85.0511); - var count = TileCoord.count(envelope, tileset.getMinzoom(), tileset.getMaxzoom()); + var count = TileCoord.count(envelope, tileset.getMinzoom(), tileset.getMaxzoom()); - var stream = - StreamUtils.stream(TileCoord.iterator(envelope, tileset.getMinzoom(), tileset.getMaxzoom())) - .peek(new ProgressLogger<>(count, 5000)); + var stream = + StreamUtils + .stream(TileCoord.iterator(envelope, tileset.getMinzoom(), tileset.getMaxzoom())) + .peek(new ProgressLogger<>(count, 5000)); - StreamUtils.batch(stream).forEach(new TileChannel(sourceTileStore, targetTileStore)); + StreamUtils.batch(stream).forEach(new TileChannel(sourceTileStore, targetTileStore)); + } } private TileStore sourceTileStore(Tileset tileset, DataSource datasource) { @@ -98,6 +102,10 @@ public record ExportVectorTiles( tilesStore.initializeDatabase(); tilesStore.writeMetadata(metadata(source)); return tilesStore; + case pmtiles: + Files.deleteIfExists(repository); + var tileStore = new PMTilesStore(repository); + return tileStore; default: throw new IllegalArgumentException("Unsupported format"); } diff --git a/baremaps-core/src/test/java/org/apache/baremaps/tilestore/pmtiles/PMTilesReader.java b/baremaps-core/src/test/java/org/apache/baremaps/tilestore/pmtiles/PMTilesReader.java deleted file mode 100644 index 4ca92112..00000000 --- a/baremaps-core/src/test/java/org/apache/baremaps/tilestore/pmtiles/PMTilesReader.java +++ /dev/null @@ -1,77 +0,0 @@ -package org.apache.baremaps.tilestore.pmtiles; - -import java.io.IOException; -import java.nio.ByteBuffer; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.List; - -public class PMTilesReader { - - private final Path path; - - private Header header; - - private List<Entry> rootEntries; - - public PMTilesReader(Path path) { - this.path = path; - } - - public Header getHeader() { - if (header == null) { - try (var channel = Files.newByteChannel(path)) { - var buffer = ByteBuffer.allocate(127); - channel.read(buffer); - buffer.flip(); - header = PMTiles.decodeHeader(buffer); - } catch (IOException e) { - throw new RuntimeException(e); - } - } - return header; - } - - public List<Entry> getRootDirectory() { - if (rootEntries == null) { - var header = getHeader(); - rootEntries = getDirectory(header.getRootDirectoryOffset(), (int) header.getRootDirectoryLength()); - } - return rootEntries; - } - - public List<Entry> getDirectory(long offset, int length) { - var header = getHeader(); - try (var channel = Files.newByteChannel(path)) { - var compressed = ByteBuffer.allocate(length); - channel.position(offset); - channel.read(compressed); - compressed.flip(); - var decompressed = header.getInternalCompression().decompress(compressed); - return PMTiles.decodeDirectory(decompressed); - } catch (IOException e) { - throw new RuntimeException(e); - } - } - - public ByteBuffer getTile(int z, long x, long y) { - var tileId = PMTiles.zxyToTileId(z, x, y); - var header = getHeader(); - var entries = getRootDirectory(); - var entry = PMTiles.findTile(entries, tileId); - - if (entry == null) { - return null; - } - - try (var channel = Files.newByteChannel(path)) { - var compressed = ByteBuffer.allocate((int) entry.getLength()); - channel.position(header.getTileDataOffset() + entry.getOffset()); - channel.read(compressed); - compressed.flip(); - return header.getTileCompression().decompress(compressed); - } catch (IOException e) { - throw new RuntimeException(e); - } - } -} diff --git a/baremaps-core/src/test/java/org/apache/baremaps/tilestore/pmtiles/PMTilesTest.java b/baremaps-core/src/test/java/org/apache/baremaps/tilestore/pmtiles/PMTilesTest.java index 84211805..fbe7fcd7 100644 --- a/baremaps-core/src/test/java/org/apache/baremaps/tilestore/pmtiles/PMTilesTest.java +++ b/baremaps-core/src/test/java/org/apache/baremaps/tilestore/pmtiles/PMTilesTest.java @@ -14,57 +14,57 @@ package org.apache.baremaps.tilestore.pmtiles; import static org.junit.jupiter.api.Assertions.*; +import com.google.common.io.LittleEndianDataInputStream; +import com.google.common.io.LittleEndianDataOutputStream; import com.google.common.math.LongMath; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; import java.io.IOException; -import java.nio.ByteBuffer; -import java.nio.file.Files; +import java.nio.channels.Channels; +import java.nio.channels.FileChannel; import java.util.ArrayList; - -import com.google.protobuf.InvalidProtocolBufferException; +import java.util.List; +import java.util.Random; import org.apache.baremaps.testing.TestFiles; -import org.apache.baremaps.vectortile.VectorTileDecoder; -import org.apache.baremaps.vectortile.VectorTileViewer; -import org.apache.baremaps.vectortile.VectorTileViewer.TilePanel; import org.junit.jupiter.api.Test; -import javax.swing.*; - class PMTilesTest { @Test - void decodeVarInt() { - var b = ByteBuffer.wrap(new byte[] { + void decodeVarInt() throws IOException { + var b = new LittleEndianDataInputStream(new ByteArrayInputStream(new byte[] { (byte) 0, (byte) 1, (byte) 127, (byte) 0xe5, (byte) 0x8e, (byte) 0x26 - }); - assertEquals(PMTiles.decodeVarInt(b), 0); - assertEquals(PMTiles.decodeVarInt(b), 1); - assertEquals(PMTiles.decodeVarInt(b), 127); - assertEquals(PMTiles.decodeVarInt(b), 624485); - b = ByteBuffer.wrap(new byte[] { + })); + assertEquals(PMTiles.readVarInt(b), 0); + assertEquals(PMTiles.readVarInt(b), 1); + assertEquals(PMTiles.readVarInt(b), 127); + assertEquals(PMTiles.readVarInt(b), 624485); + b = new LittleEndianDataInputStream(new ByteArrayInputStream(new byte[] { (byte) 0xff, (byte) 0xff, (byte) 0xff, (byte) 0xff, (byte) 0xff, (byte) 0xff, (byte) 0xff, (byte) 0x0f, - }); - assertEquals(PMTiles.decodeVarInt(b), 9007199254740991L); + })); + assertEquals(PMTiles.readVarInt(b), 9007199254740991L); } @Test - void encodeVarInt() { - var buffer = ByteBuffer.allocate(10); + void encodeVarInt() throws IOException { for (long i = 0; i < 1000; i++) { - PMTiles.encodeVarInt(buffer, i); - buffer.flip(); - assertEquals(i, PMTiles.decodeVarInt(buffer)); - buffer.clear(); + var array = new ByteArrayOutputStream(); + var output = new LittleEndianDataOutputStream(array); + PMTiles.writeVarInt(output, i); + var input = new LittleEndianDataInputStream(new ByteArrayInputStream(array.toByteArray())); + assertEquals(i, PMTiles.readVarInt(input)); } for (long i = Long.MAX_VALUE - 1000; i < Long.MAX_VALUE; i++) { - PMTiles.encodeVarInt(buffer, i); - buffer.flip(); - assertEquals(i, PMTiles.decodeVarInt(buffer)); - buffer.clear(); + var array = new ByteArrayOutputStream(); + var output = new LittleEndianDataOutputStream(array); + PMTiles.writeVarInt(output, i); + var input = new LittleEndianDataInputStream(new ByteArrayInputStream(array.toByteArray())); + assertEquals(i, PMTiles.readVarInt(input)); } } @@ -127,11 +127,9 @@ class PMTilesTest { @Test void decodeHeader() throws IOException { var file = TestFiles.resolve("pmtiles/test_fixture_1.pmtiles"); - try (var channel = Files.newByteChannel(file)) { - var buffer = ByteBuffer.allocate(127); - channel.read(buffer); - buffer.flip(); - var header = PMTiles.decodeHeader(buffer); + try (var channel = FileChannel.open(file)) { + var input = new LittleEndianDataInputStream(Channels.newInputStream(channel)); + var header = PMTiles.deserializeHeader(input); assertEquals(header.getRootDirectoryOffset(), 127); assertEquals(header.getRootDirectoryLength(), 25); assertEquals(header.getJsonMetadataOffset(), 152); @@ -159,7 +157,6 @@ class PMTilesTest { @Test void encodeHeader() throws IOException { var etag = "1"; - var buffer = ByteBuffer.allocate(127); var header = new Header( 127, 25, @@ -185,10 +182,18 @@ class PMTilesTest { 0, 0, 0, - 0 - ); - PMTiles.encodeHeader(header, buffer); - var header2 = PMTiles.decodeHeader(buffer); + 0); + + var array = new ByteArrayOutputStream(); + + var output = new LittleEndianDataOutputStream(array); + PMTiles.serializeHeader(output, header); + + var array2 = array.toByteArray(); + + var input = new LittleEndianDataInputStream(new ByteArrayInputStream(array.toByteArray())); + var header2 = PMTiles.deserializeHeader(input); + assertEquals(header, header2); } @@ -249,26 +254,32 @@ class PMTilesTest { } @Test - void reader() throws InvalidProtocolBufferException, InterruptedException { - var reader = new PMTilesReader(TestFiles.resolve("pmtiles/test_fixture_1.pmtiles")); - var header = reader.getHeader(); - assertEquals(header.getRootDirectoryOffset(), 127); - var rootDirectory = reader.getRootDirectory(); - assertEquals(rootDirectory.size(), 1); - var entry = rootDirectory.get(0); - ByteBuffer buffer = reader.getTile(0,0,0); + void buildRootLeaves() throws IOException { + var entries = List.of(new Entry(100, 1, 1, 0)); + var directories = PMTiles.buildRootLeaves(entries, 1); + assertEquals(directories.numLeaves(), 1); - VectorTileViewer viewer = new VectorTileViewer(); - var parsed = org.apache.baremaps.mvt.binary.VectorTile.Tile.parseFrom(buffer); - var tile = new VectorTileDecoder().decodeTile(parsed); - JFrame f = new JFrame("Vector Tile Viewer"); - f.setDefaultCloseOperation(JFrame.EXIT_ON_CLOSE); - f.add(new TilePanel(tile, 1000)); - f.pack(); - f.setVisible(true); - - Thread.sleep(10000); } + @Test + void optimizeDirectories() throws IOException { + var random = new Random(3857); + var entries = new ArrayList<Entry>(); + entries.add(new Entry(0, 0, 100, 1)); + var directories = PMTiles.optimizeDirectories(entries, 100); + assertFalse(directories.leaves().length > 0); + assertEquals(0, directories.numLeaves()); + entries = new ArrayList<>(); + int offset = 0; + for (var i = 0; i < 1000; i++) { + var randTileSize = random.nextInt(1000000); + entries.add(new Entry(i, offset, randTileSize, 1)); + offset += randTileSize; + } + directories = PMTiles.optimizeDirectories(entries, 1024); + assertFalse(directories.root().length > 1024); + assertFalse(directories.numLeaves() == 0); + assertFalse(directories.leaves().length == 0); + } }
