This is an automated email from the ASF dual-hosted git repository. bchapuis pushed a commit to branch pmtiles-refactoring in repository https://gitbox.apache.org/repos/asf/incubator-baremaps.git
commit 82101d128f0e4ac18b1e4d5804c14a8fa826ed66 Author: Bertil Chapuis <[email protected]> AuthorDate: Fri Apr 4 21:01:24 2025 +0200 Refactor the PMTiles module - Extract serialization logic - Introduce builder patter - Reduce visibility of internal classes - Improve resource management Extract serialization logic Fix object construction and split tests Reduce the visibility of classes Impove resource management Remove files --- .../org/apache/baremaps/pmtiles/Compression.java | 21 ++ .../org/apache/baremaps/pmtiles/Directories.java | 114 +++++- .../baremaps/pmtiles/DirectorySerializer.java | 132 +++++++ .../java/org/apache/baremaps/pmtiles/Entry.java | 130 ++++++- .../apache/baremaps/pmtiles/EntrySerializer.java | 145 ++++++++ .../java/org/apache/baremaps/pmtiles/Header.java | 408 +++++++++++++++++++- .../apache/baremaps/pmtiles/HeaderSerializer.java | 122 ++++++ .../org/apache/baremaps/pmtiles/PMTilesReader.java | 22 +- .../org/apache/baremaps/pmtiles/PMTilesUtils.java | 410 --------------------- .../org/apache/baremaps/pmtiles/PMTilesWriter.java | 126 ++++--- .../pmtiles/{Directories.java => Serializer.java} | 45 ++- .../apache/baremaps/pmtiles/TileIdConverter.java | 127 +++++++ .../apache/baremaps/pmtiles/VarIntSerializer.java | 138 +++++++ .../baremaps/pmtiles/DirectorySerializerTest.java | 64 ++++ .../baremaps/pmtiles/EntrySerializerTest.java | 87 +++++ .../baremaps/pmtiles/HeaderSerializerTest.java | 100 +++++ ...esUtilsTest.java => PMTilesSerializerTest.java} | 171 ++++----- .../baremaps/pmtiles/TileIdConverterTest.java | 85 +++++ .../baremaps/pmtiles/VarIntSerializerTest.java | 70 ++++ 19 files changed, 1940 insertions(+), 577 deletions(-) diff --git a/baremaps-pmtiles/src/main/java/org/apache/baremaps/pmtiles/Compression.java b/baremaps-pmtiles/src/main/java/org/apache/baremaps/pmtiles/Compression.java index a339e6063..ec775f85f 100644 --- a/baremaps-pmtiles/src/main/java/org/apache/baremaps/pmtiles/Compression.java +++ b/baremaps-pmtiles/src/main/java/org/apache/baremaps/pmtiles/Compression.java @@ -28,6 +28,16 @@ public enum Compression { BROTLI, ZSTD; + /** + * Decompresses an input stream using the compression algorithm represented by this enum value. + * <p> + * <strong>Note:</strong> The caller is responsible for closing both the input stream and the + * returned stream. + * + * @param inputStream the input stream to decompress + * @return a new input stream that decompresses the given stream + * @throws IOException if an I/O error occurs + */ InputStream decompress(InputStream inputStream) throws IOException { return switch (this) { case NONE -> inputStream; @@ -50,6 +60,17 @@ public enum Compression { throw new UnsupportedOperationException("Zstd compression not implemented"); } + /** + * Compresses an output stream using the compression algorithm represented by this enum value. + * <p> + * <strong>Note:</strong> The caller is responsible for closing both the output stream and the + * returned stream. The returned compressed stream should be closed before the underlying output + * stream to ensure all compressed data is flushed properly. + * + * @param outputStream the output stream to compress + * @return a new output stream that compresses to the given stream + * @throws IOException if an I/O error occurs + */ OutputStream compress(OutputStream outputStream) throws IOException { return switch (this) { case NONE -> outputStream; diff --git a/baremaps-pmtiles/src/main/java/org/apache/baremaps/pmtiles/Directories.java b/baremaps-pmtiles/src/main/java/org/apache/baremaps/pmtiles/Directories.java index 2306efdfc..36f98451b 100644 --- a/baremaps-pmtiles/src/main/java/org/apache/baremaps/pmtiles/Directories.java +++ b/baremaps-pmtiles/src/main/java/org/apache/baremaps/pmtiles/Directories.java @@ -17,18 +17,50 @@ package org.apache.baremaps.pmtiles; -class Directories { +import java.util.Arrays; +import java.util.Objects; + +public class Directories { private final byte[] root; private final byte[] leaves; private final int numLeaves; - public Directories(byte[] root, byte[] leaves, int numLeaves) { + /** + * Creates a new Directories object with the specified values. + * <p> + * Consider using {@link Builder} for a more fluent creation approach. + * + * @param root the root directory data + * @param leaves the leaf directory data + * @param numLeaves the number of leaves + */ + private Directories(byte[] root, byte[] leaves, int numLeaves) { this.root = root; this.leaves = leaves; this.numLeaves = numLeaves; } + /** + * Creates a new Directories object from a Builder. + * + * @param builder the builder to use + */ + private Directories(Builder builder) { + this.root = builder.root; + this.leaves = builder.leaves; + this.numLeaves = builder.numLeaves; + } + + /** + * Creates a new Builder for Directories objects. + * + * @return a new builder + */ + public static Builder builder() { + return new Builder(); + } + public byte[] getRoot() { return root; } @@ -40,4 +72,82 @@ class Directories { public int getNumLeaves() { return numLeaves; } + + @Override + public boolean equals(Object o) { + if (this == o) + return true; + if (o == null || getClass() != o.getClass()) + return false; + Directories that = (Directories) o; + return numLeaves == that.numLeaves && + Arrays.equals(root, that.root) && + Arrays.equals(leaves, that.leaves); + } + + @Override + public int hashCode() { + int result = Objects.hash(numLeaves); + result = 31 * result + Arrays.hashCode(root); + result = 31 * result + Arrays.hashCode(leaves); + return result; + } + + /** + * Builder for Directories objects. + */ + public static class Builder { + private byte[] root = new byte[0]; + private byte[] leaves = new byte[0]; + private int numLeaves = 0; + + /** + * Creates a new Builder with default values. + */ + private Builder() { + // Use static factory method + } + + /** + * Sets the root directory data. + * + * @param root the root directory data + * @return this builder + */ + public Builder root(byte[] root) { + this.root = root; + return this; + } + + /** + * Sets the leaf directory data. + * + * @param leaves the leaf directory data + * @return this builder + */ + public Builder leaves(byte[] leaves) { + this.leaves = leaves; + return this; + } + + /** + * Sets the number of leaves. + * + * @param numLeaves the number of leaves + * @return this builder + */ + public Builder numLeaves(int numLeaves) { + this.numLeaves = numLeaves; + return this; + } + + /** + * Builds a new Directories object. + * + * @return a new Directories object + */ + public Directories build() { + return new Directories(this); + } + } } diff --git a/baremaps-pmtiles/src/main/java/org/apache/baremaps/pmtiles/DirectorySerializer.java b/baremaps-pmtiles/src/main/java/org/apache/baremaps/pmtiles/DirectorySerializer.java new file mode 100644 index 000000000..e615cc423 --- /dev/null +++ b/baremaps-pmtiles/src/main/java/org/apache/baremaps/pmtiles/DirectorySerializer.java @@ -0,0 +1,132 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.baremaps.pmtiles; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +/** + * Serializer for PMTiles Directory structures. + */ +class DirectorySerializer { + + private final EntrySerializer entrySerializer; + + DirectorySerializer() { + this.entrySerializer = new EntrySerializer(); + } + + /** + * Build root and leaf directories from entries. + * + * @param entries the list of entries + * @param leafSize the number of entries per leaf + * @param compression the compression to use + * @return the built directories + * @throws IOException if an I/O error occurs + */ + public Directories buildRootLeaves(List<Entry> entries, int leafSize, + Compression compression) throws IOException { + var rootEntries = new ArrayList<Entry>(); + var numLeaves = 0; + byte[] leavesBytes; + byte[] rootBytes; + + try (var leavesOutput = new ByteArrayOutputStream()) { + for (var i = 0; i < entries.size(); i += leafSize) { + numLeaves++; + var end = i + leafSize; + if (i + leafSize > entries.size()) { + end = entries.size(); + } + var offset = leavesOutput.size(); + try (var leafOutput = new ByteArrayOutputStream()) { + try (var compressedLeafOutput = compression.compress(leafOutput)) { + entrySerializer.serialize(entries.subList(i, end), compressedLeafOutput); + } + var length = leafOutput.size(); + rootEntries.add(Entry.builder() + .tileId(entries.get(i).getTileId()) + .offset(offset) + .length(length) + .runLength(0) + .build()); + leavesOutput.write(leafOutput.toByteArray()); + } + } + leavesBytes = leavesOutput.toByteArray(); + } + + try (var rootOutput = new ByteArrayOutputStream()) { + try (var compressedRootOutput = compression.compress(rootOutput)) { + entrySerializer.serialize(rootEntries, compressedRootOutput); + } + rootBytes = rootOutput.toByteArray(); + } + + return Directories.builder() + .root(rootBytes) + .leaves(leavesBytes) + .numLeaves(numLeaves) + .build(); + } + + /** + * Optimize directories to fit within targetRootLength. + * + * @param entries the list of entries + * @param targetRootLength the target length of the root directory + * @param compression the compression to use + * @return the optimized directories + * @throws IOException if an I/O error occurs + */ + public Directories optimizeDirectories(List<Entry> entries, int targetRootLength, + Compression compression) throws IOException { + if (entries.size() < 16384) { + try (var rootOutput = new ByteArrayOutputStream()) { + try (var compressedOutput = compression.compress(rootOutput)) { + entrySerializer.serialize(entries, compressedOutput); + } + byte[] rootBytes = rootOutput.toByteArray(); + if (rootBytes.length <= targetRootLength) { + return Directories.builder() + .root(rootBytes) + .leaves(new byte[] {}) + .numLeaves(0) + .build(); + } + } + } + + double leafSize = Math.max((double) entries.size() / 3500, 4096); + while (true) { + Directories directories = buildRootLeaves(entries, (int) leafSize, compression); + if (directories.getRoot().length <= targetRootLength) { + return directories; + } + leafSize = leafSize * 1.2; + // Add a safety check to prevent infinite loops + if (leafSize > entries.size()) { + throw new IOException( + "Could not optimize directories to fit within target length: " + targetRootLength); + } + } + } +} diff --git a/baremaps-pmtiles/src/main/java/org/apache/baremaps/pmtiles/Entry.java b/baremaps-pmtiles/src/main/java/org/apache/baremaps/pmtiles/Entry.java index 882b86973..feea001b6 100644 --- a/baremaps-pmtiles/src/main/java/org/apache/baremaps/pmtiles/Entry.java +++ b/baremaps-pmtiles/src/main/java/org/apache/baremaps/pmtiles/Entry.java @@ -17,21 +17,59 @@ package org.apache.baremaps.pmtiles; +import java.util.Objects; + public class Entry { private long tileId; private long offset; private long length; private long runLength; - public Entry() {} + /** + * Creates a new Entry with default values. + * <p> + * Use {@link Builder} for a more fluent way to create Entry objects. + */ + private Entry() {} - public Entry(long tileId, long offset, long length, long runLength) { + /** + * Creates a new Entry with the specified values. + * <p> + * Consider using {@link Builder} for a more fluent creation approach. + * + * @param tileId the tile ID + * @param offset the offset within the tile data section + * @param length the length of the tile data + * @param runLength the run length for compressed entries + */ + private Entry(long tileId, long offset, long length, long runLength) { this.tileId = tileId; this.offset = offset; this.length = length; this.runLength = runLength; } + /** + * Creates a new Entry from a Builder. + * + * @param builder the builder to use + */ + private Entry(Builder builder) { + this.tileId = builder.tileId; + this.offset = builder.offset; + this.length = builder.length; + this.runLength = builder.runLength; + } + + /** + * Creates a new Builder for Entry objects. + * + * @return a new builder + */ + public static Builder builder() { + return new Builder(); + } + public long getTileId() { return tileId; } @@ -63,4 +101,92 @@ public class Entry { public void setRunLength(long runLength) { this.runLength = runLength; } + + @Override + public boolean equals(Object o) { + if (this == o) + return true; + if (o == null || getClass() != o.getClass()) + return false; + Entry entry = (Entry) o; + return tileId == entry.tileId && + offset == entry.offset && + length == entry.length && + runLength == entry.runLength; + } + + @Override + public int hashCode() { + return Objects.hash(tileId, offset, length, runLength); + } + + /** + * Builder for Entry objects. + */ + public static class Builder { + private long tileId; + private long offset; + private long length; + private long runLength; + + /** + * Creates a new Builder with default values. + */ + private Builder() { + // Use static factory method + } + + /** + * Sets the tile ID. + * + * @param tileId the tile ID + * @return this builder + */ + public Builder tileId(long tileId) { + this.tileId = tileId; + return this; + } + + /** + * Sets the offset within the tile data section. + * + * @param offset the offset + * @return this builder + */ + public Builder offset(long offset) { + this.offset = offset; + return this; + } + + /** + * Sets the length of the tile data. + * + * @param length the length + * @return this builder + */ + public Builder length(long length) { + this.length = length; + return this; + } + + /** + * Sets the run length for compressed entries. + * + * @param runLength the run length + * @return this builder + */ + public Builder runLength(long runLength) { + this.runLength = runLength; + return this; + } + + /** + * Builds a new Entry object. + * + * @return a new Entry + */ + public Entry build() { + return new Entry(this); + } + } } diff --git a/baremaps-pmtiles/src/main/java/org/apache/baremaps/pmtiles/EntrySerializer.java b/baremaps-pmtiles/src/main/java/org/apache/baremaps/pmtiles/EntrySerializer.java new file mode 100644 index 000000000..ecbd53e1d --- /dev/null +++ b/baremaps-pmtiles/src/main/java/org/apache/baremaps/pmtiles/EntrySerializer.java @@ -0,0 +1,145 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.baremaps.pmtiles; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; + +/** + * Serializer for PMTiles Entry objects. + */ +class EntrySerializer implements Serializer<List<Entry>> { + + private final VarIntSerializer varIntSerializer; + + EntrySerializer() { + this.varIntSerializer = new VarIntSerializer(); + } + + @Override + public void serialize(List<Entry> entries, OutputStream output) throws IOException { + var buffer = ByteBuffer.allocate(entries.size() * 48); + varIntSerializer.writeVarInt(output, entries.size()); + + // Write tileIds as deltas + long lastId = 0; + for (Entry entry : entries) { + varIntSerializer.writeVarInt(output, entry.getTileId() - lastId); + lastId = entry.getTileId(); + } + + // Write run lengths + for (Entry entry : entries) { + varIntSerializer.writeVarInt(output, entry.getRunLength()); + } + + // Write lengths + for (Entry entry : entries) { + varIntSerializer.writeVarInt(output, entry.getLength()); + } + + // Write offsets (with RLE compression) + for (int i = 0; i < entries.size(); i++) { + Entry entry = entries.get(i); + if (i > 0 + && entry.getOffset() == entries.get(i - 1).getOffset() + entries.get(i - 1).getLength()) { + varIntSerializer.writeVarInt(output, 0); + } else { + varIntSerializer.writeVarInt(output, entry.getOffset() + 1); + } + } + } + + @Override + public List<Entry> deserialize(InputStream input) throws IOException { + long numEntries = varIntSerializer.readVarInt(input); + List<Entry> entries = new ArrayList<>((int) numEntries); + + // Read tileIds + long lastId = 0; + for (int i = 0; i < numEntries; i++) { + long value = varIntSerializer.readVarInt(input); + lastId = lastId + value; + Entry entry = Entry.builder().tileId(lastId).build(); + entries.add(entry); + } + + // Read run lengths + for (int i = 0; i < numEntries; i++) { + long value = varIntSerializer.readVarInt(input); + entries.get(i).setRunLength(value); + } + + // Read lengths + for (int i = 0; i < numEntries; i++) { + long value = varIntSerializer.readVarInt(input); + entries.get(i).setLength(value); + } + + // Read offsets + for (int i = 0; i < numEntries; i++) { + long value = varIntSerializer.readVarInt(input); + if (value == 0 && i > 0) { + Entry prevEntry = entries.get(i - 1); + entries.get(i).setOffset(prevEntry.getOffset() + prevEntry.getLength()); + } else { + entries.get(i).setOffset(value - 1); + } + } + + return entries; + } + + /** + * Find a tile entry by its tile ID. + * + * @param entries list of entries to search + * @param tileId the tile ID to find + * @return the entry if found, null otherwise + */ + public Entry findTile(List<Entry> entries, long tileId) { + int m = 0; + int n = entries.size() - 1; + while (m <= n) { + int k = (n + m) >> 1; + long cmp = tileId - entries.get(k).getTileId(); + if (cmp > 0) { + m = k + 1; + } else if (cmp < 0) { + n = k - 1; + } else { + return entries.get(k); + } + } + + // at this point, m > n + if (n >= 0) { + if (entries.get(n).getRunLength() == 0) { + return entries.get(n); + } + if (tileId - entries.get(n).getTileId() < entries.get(n).getRunLength()) { + return entries.get(n); + } + } + return null; + } +} diff --git a/baremaps-pmtiles/src/main/java/org/apache/baremaps/pmtiles/Header.java b/baremaps-pmtiles/src/main/java/org/apache/baremaps/pmtiles/Header.java index e79b7c1a5..527965ef5 100644 --- a/baremaps-pmtiles/src/main/java/org/apache/baremaps/pmtiles/Header.java +++ b/baremaps-pmtiles/src/main/java/org/apache/baremaps/pmtiles/Header.java @@ -82,10 +82,20 @@ public class Header { private double centerLon; private double centerLat; - public Header() { + /** + * Creates a new Header with default values. + * <p> + * Use {@link Builder} for a more fluent way to create Header objects. + */ + private Header() { this.specVersion = 3; } + /** + * Creates a new Header with the specified values. + * <p> + * This constructor has many parameters. Consider using {@link Builder} instead. + */ @SuppressWarnings("squid:S107") public Header( int specVersion, @@ -139,6 +149,48 @@ public class Header { this.centerLat = centerLat; } + /** + * Creates a new Header from a Builder. + * + * @param builder the builder to use + */ + private Header(Builder builder) { + this.specVersion = builder.specVersion; + this.rootDirectoryOffset = builder.rootDirectoryOffset; + this.rootDirectoryLength = builder.rootDirectoryLength; + this.jsonMetadataOffset = builder.jsonMetadataOffset; + this.jsonMetadataLength = builder.jsonMetadataLength; + this.leafDirectoryOffset = builder.leafDirectoryOffset; + this.leafDirectoryLength = builder.leafDirectoryLength; + this.tileDataOffset = builder.tileDataOffset; + this.tileDataLength = builder.tileDataLength; + this.numAddressedTiles = builder.numAddressedTiles; + this.numTileEntries = builder.numTileEntries; + this.numTileContents = builder.numTileContents; + this.clustered = builder.clustered; + this.internalCompression = builder.internalCompression; + this.tileCompression = builder.tileCompression; + this.tileType = builder.tileType; + this.minZoom = builder.minZoom; + this.maxZoom = builder.maxZoom; + this.minLon = builder.minLon; + this.minLat = builder.minLat; + this.maxLon = builder.maxLon; + this.maxLat = builder.maxLat; + this.centerZoom = builder.centerZoom; + this.centerLon = builder.centerLon; + this.centerLat = builder.centerLat; + } + + /** + * Creates a new Builder for Header objects. + * + * @return a new builder + */ + public static Builder builder() { + return new Builder(); + } + public int getSpecVersion() { return specVersion; } @@ -374,4 +426,358 @@ public class Header { internalCompression, tileCompression, tileType, minZoom, maxZoom, minLon, minLat, maxLon, maxLat, centerZoom, centerLon, centerLat); } + + /** + * Builder for Header objects. + */ + public static class Builder { + private int specVersion = 3; + private long rootDirectoryOffset; + private long rootDirectoryLength; + private long jsonMetadataOffset; + private long jsonMetadataLength; + private long leafDirectoryOffset; + private long leafDirectoryLength; + private long tileDataOffset; + private long tileDataLength; + private long numAddressedTiles; + private long numTileEntries; + private long numTileContents; + private boolean clustered; + private Compression internalCompression = Compression.GZIP; + private Compression tileCompression = Compression.GZIP; + private TileType tileType = TileType.MVT; + private int minZoom; + private int maxZoom = 14; + private double minLon = -180; + private double minLat = -90; + private double maxLon = 180; + private double maxLat = 90; + private int centerZoom = 3; + private double centerLon; + private double centerLat; + + /** + * Creates a new Builder with default values. + */ + private Builder() { + // Use static factory method + } + + /** + * Sets the spec version. + * + * @param specVersion the spec version + * @return this builder + */ + public Builder specVersion(int specVersion) { + this.specVersion = specVersion; + return this; + } + + /** + * Sets the root directory offset. + * + * @param rootDirectoryOffset the root directory offset + * @return this builder + */ + public Builder rootDirectoryOffset(long rootDirectoryOffset) { + this.rootDirectoryOffset = rootDirectoryOffset; + return this; + } + + /** + * Sets the root directory length. + * + * @param rootDirectoryLength the root directory length + * @return this builder + */ + public Builder rootDirectoryLength(long rootDirectoryLength) { + this.rootDirectoryLength = rootDirectoryLength; + return this; + } + + /** + * Sets the JSON metadata offset. + * + * @param jsonMetadataOffset the JSON metadata offset + * @return this builder + */ + public Builder jsonMetadataOffset(long jsonMetadataOffset) { + this.jsonMetadataOffset = jsonMetadataOffset; + return this; + } + + /** + * Sets the JSON metadata length. + * + * @param jsonMetadataLength the JSON metadata length + * @return this builder + */ + public Builder jsonMetadataLength(long jsonMetadataLength) { + this.jsonMetadataLength = jsonMetadataLength; + return this; + } + + /** + * Sets the leaf directory offset. + * + * @param leafDirectoryOffset the leaf directory offset + * @return this builder + */ + public Builder leafDirectoryOffset(long leafDirectoryOffset) { + this.leafDirectoryOffset = leafDirectoryOffset; + return this; + } + + /** + * Sets the leaf directory length. + * + * @param leafDirectoryLength the leaf directory length + * @return this builder + */ + public Builder leafDirectoryLength(long leafDirectoryLength) { + this.leafDirectoryLength = leafDirectoryLength; + return this; + } + + /** + * Sets the tile data offset. + * + * @param tileDataOffset the tile data offset + * @return this builder + */ + public Builder tileDataOffset(long tileDataOffset) { + this.tileDataOffset = tileDataOffset; + return this; + } + + /** + * Sets the tile data length. + * + * @param tileDataLength the tile data length + * @return this builder + */ + public Builder tileDataLength(long tileDataLength) { + this.tileDataLength = tileDataLength; + return this; + } + + /** + * Sets the number of addressed tiles. + * + * @param numAddressedTiles the number of addressed tiles + * @return this builder + */ + public Builder numAddressedTiles(long numAddressedTiles) { + this.numAddressedTiles = numAddressedTiles; + return this; + } + + /** + * Sets the number of tile entries. + * + * @param numTileEntries the number of tile entries + * @return this builder + */ + public Builder numTileEntries(long numTileEntries) { + this.numTileEntries = numTileEntries; + return this; + } + + /** + * Sets the number of tile contents. + * + * @param numTileContents the number of tile contents + * @return this builder + */ + public Builder numTileContents(long numTileContents) { + this.numTileContents = numTileContents; + return this; + } + + /** + * Sets whether the tiles are clustered. + * + * @param clustered whether the tiles are clustered + * @return this builder + */ + public Builder clustered(boolean clustered) { + this.clustered = clustered; + return this; + } + + /** + * Sets the internal compression. + * + * @param internalCompression the internal compression + * @return this builder + */ + public Builder internalCompression(Compression internalCompression) { + this.internalCompression = internalCompression; + return this; + } + + /** + * Sets the tile compression. + * + * @param tileCompression the tile compression + * @return this builder + */ + public Builder tileCompression(Compression tileCompression) { + this.tileCompression = tileCompression; + return this; + } + + /** + * Sets the tile type. + * + * @param tileType the tile type + * @return this builder + */ + public Builder tileType(TileType tileType) { + this.tileType = tileType; + return this; + } + + /** + * Sets the minimum zoom level. + * + * @param minZoom the minimum zoom level + * @return this builder + */ + public Builder minZoom(int minZoom) { + this.minZoom = minZoom; + return this; + } + + /** + * Sets the maximum zoom level. + * + * @param maxZoom the maximum zoom level + * @return this builder + */ + public Builder maxZoom(int maxZoom) { + this.maxZoom = maxZoom; + return this; + } + + /** + * Sets the minimum longitude. + * + * @param minLon the minimum longitude + * @return this builder + */ + public Builder minLon(double minLon) { + this.minLon = minLon; + return this; + } + + /** + * Sets the minimum latitude. + * + * @param minLat the minimum latitude + * @return this builder + */ + public Builder minLat(double minLat) { + this.minLat = minLat; + return this; + } + + /** + * Sets the maximum longitude. + * + * @param maxLon the maximum longitude + * @return this builder + */ + public Builder maxLon(double maxLon) { + this.maxLon = maxLon; + return this; + } + + /** + * Sets the maximum latitude. + * + * @param maxLat the maximum latitude + * @return this builder + */ + public Builder maxLat(double maxLat) { + this.maxLat = maxLat; + return this; + } + + /** + * Sets the center zoom level. + * + * @param centerZoom the center zoom level + * @return this builder + */ + public Builder centerZoom(int centerZoom) { + this.centerZoom = centerZoom; + return this; + } + + /** + * Sets the center longitude. + * + * @param centerLon the center longitude + * @return this builder + */ + public Builder centerLon(double centerLon) { + this.centerLon = centerLon; + return this; + } + + /** + * Sets the center latitude. + * + * @param centerLat the center latitude + * @return this builder + */ + public Builder centerLat(double centerLat) { + this.centerLat = centerLat; + return this; + } + + /** + * Sets the bounds of the tiles. + * + * @param minLon the minimum longitude + * @param minLat the minimum latitude + * @param maxLon the maximum longitude + * @param maxLat the maximum latitude + * @return this builder + */ + public Builder bounds(double minLon, double minLat, double maxLon, double maxLat) { + this.minLon = minLon; + this.minLat = minLat; + this.maxLon = maxLon; + this.maxLat = maxLat; + return this; + } + + /** + * Sets the center of the tiles. + * + * @param centerZoom the center zoom level + * @param centerLon the center longitude + * @param centerLat the center latitude + * @return this builder + */ + public Builder center(int centerZoom, double centerLon, double centerLat) { + this.centerZoom = centerZoom; + this.centerLon = centerLon; + this.centerLat = centerLat; + return this; + } + + /** + * Builds a new Header object. + * + * @return a new Header + */ + public Header build() { + return new Header(this); + } + } } diff --git a/baremaps-pmtiles/src/main/java/org/apache/baremaps/pmtiles/HeaderSerializer.java b/baremaps-pmtiles/src/main/java/org/apache/baremaps/pmtiles/HeaderSerializer.java new file mode 100644 index 000000000..3adc92a80 --- /dev/null +++ b/baremaps-pmtiles/src/main/java/org/apache/baremaps/pmtiles/HeaderSerializer.java @@ -0,0 +1,122 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.baremaps.pmtiles; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; + +/** + * Serializer for PMTiles Header objects. + */ +class HeaderSerializer implements Serializer<Header> { + + private static final int HEADER_SIZE_BYTES = 127; + private static final byte[] MAGIC_BYTES = {0x50, 0x4D, 0x54, 0x69, 0x6C, 0x65, 0x73}; + + HeaderSerializer() { + + } + + @Override + public void serialize(Header header, OutputStream output) throws IOException { + var buffer = ByteBuffer.allocate(HEADER_SIZE_BYTES).order(ByteOrder.LITTLE_ENDIAN); + // Write PMTiles magic bytes + buffer.put(MAGIC_BYTES); + + buffer.put((byte) header.getSpecVersion()); + buffer.putLong(header.getRootDirectoryOffset()); + buffer.putLong(header.getRootDirectoryLength()); + buffer.putLong(header.getJsonMetadataOffset()); + buffer.putLong(header.getJsonMetadataLength()); + buffer.putLong(header.getLeafDirectoryOffset()); + buffer.putLong(header.getLeafDirectoryLength()); + buffer.putLong(header.getTileDataOffset()); + buffer.putLong(header.getTileDataLength()); + buffer.putLong(header.getNumAddressedTiles()); + buffer.putLong(header.getNumTileEntries()); + buffer.putLong(header.getNumTileContents()); + buffer.put((byte) (header.isClustered() ? 1 : 0)); + buffer.put((byte) header.getInternalCompression().ordinal()); + buffer.put((byte) header.getTileCompression().ordinal()); + buffer.put((byte) header.getTileType().ordinal()); + buffer.put((byte) header.getMinZoom()); + buffer.put((byte) header.getMaxZoom()); + buffer.putInt((int) (header.getMinLon() * 10000000)); + buffer.putInt((int) (header.getMinLat() * 10000000)); + buffer.putInt((int) (header.getMaxLon() * 10000000)); + buffer.putInt((int) (header.getMaxLat() * 10000000)); + buffer.put((byte) header.getCenterZoom()); + buffer.putInt((int) (header.getCenterLon() * 10000000)); + buffer.putInt((int) (header.getCenterLat() * 10000000)); + + buffer.flip(); + output.write(buffer.array()); + } + + @Override + public Header deserialize(InputStream input) throws IOException { + byte[] bytes = new byte[HEADER_SIZE_BYTES]; + var num = input.read(bytes); + if (num != HEADER_SIZE_BYTES) { + throw new IOException("Invalid header size"); + } + + var buffer = ByteBuffer.wrap(bytes).order(ByteOrder.LITTLE_ENDIAN); + + // Validate magic bytes + byte[] magic = new byte[7]; + buffer.get(magic); + for (int i = 0; i < MAGIC_BYTES.length; i++) { + if (magic[i] != MAGIC_BYTES[i]) { + throw new IOException("Invalid PMTiles header magic bytes"); + } + } + + // Use the builder pattern + return Header.builder() + .specVersion(buffer.get()) + .rootDirectoryOffset(buffer.getLong()) + .rootDirectoryLength(buffer.getLong()) + .jsonMetadataOffset(buffer.getLong()) + .jsonMetadataLength(buffer.getLong()) + .leafDirectoryOffset(buffer.getLong()) + .leafDirectoryLength(buffer.getLong()) + .tileDataOffset(buffer.getLong()) + .tileDataLength(buffer.getLong()) + .numAddressedTiles(buffer.getLong()) + .numTileEntries(buffer.getLong()) + .numTileContents(buffer.getLong()) + .clustered(buffer.get() == 1) + .internalCompression(Compression.values()[buffer.get()]) + .tileCompression(Compression.values()[buffer.get()]) + .tileType(TileType.values()[buffer.get()]) + .minZoom(buffer.get()) + .maxZoom(buffer.get()) + .minLon((double) buffer.getInt() / 10000000) + .minLat((double) buffer.getInt() / 10000000) + .maxLon((double) buffer.getInt() / 10000000) + .maxLat((double) buffer.getInt() / 10000000) + .centerZoom(buffer.get()) + .centerLon((double) buffer.getInt() / 10000000) + .centerLat((double) buffer.getInt() / 10000000) + .build(); + } +} diff --git a/baremaps-pmtiles/src/main/java/org/apache/baremaps/pmtiles/PMTilesReader.java b/baremaps-pmtiles/src/main/java/org/apache/baremaps/pmtiles/PMTilesReader.java index 6d9a259cf..1d8b4c2cd 100644 --- a/baremaps-pmtiles/src/main/java/org/apache/baremaps/pmtiles/PMTilesReader.java +++ b/baremaps-pmtiles/src/main/java/org/apache/baremaps/pmtiles/PMTilesReader.java @@ -26,22 +26,25 @@ import java.nio.file.Files; import java.nio.file.Path; import java.util.List; -public class PMTilesReader { +public class PMTilesReader implements AutoCloseable { private final Path path; + private final HeaderSerializer headerSerializer; + private final EntrySerializer entrySerializer; private Header header; - private List<Entry> rootEntries; public PMTilesReader(Path path) { this.path = path; + this.headerSerializer = new HeaderSerializer(); + this.entrySerializer = new EntrySerializer(); } public Header getHeader() throws IOException { if (header == null) { - try (var inputStream = new LittleEndianDataInputStream(Files.newInputStream(path))) { - header = PMTilesUtils.deserializeHeader(inputStream); + try (var inputStream = Files.newInputStream(path)) { + header = headerSerializer.deserialize(inputStream); } } return header; @@ -64,16 +67,16 @@ public class PMTilesReader { } try (var decompressed = new LittleEndianDataInputStream(header.getInternalCompression().decompress(input))) { - return PMTilesUtils.deserializeEntries(decompressed); + return entrySerializer.deserialize(decompressed); } } } public ByteBuffer getTile(int z, long x, long y) throws IOException { - var tileId = PMTilesUtils.zxyToTileId(z, x, y); + var tileId = TileIdConverter.zxyToTileId(z, x, y); var fileHeader = getHeader(); var entries = getRootDirectory(); - var entry = PMTilesUtils.findTile(entries, tileId); + var entry = entrySerializer.findTile(entries, tileId); if (entry == null) { return null; @@ -89,4 +92,9 @@ public class PMTilesReader { } } } + + @Override + public void close() { + // No resources to close at the class level since all I/O operations use try-with-resources + } } diff --git a/baremaps-pmtiles/src/main/java/org/apache/baremaps/pmtiles/PMTilesUtils.java b/baremaps-pmtiles/src/main/java/org/apache/baremaps/pmtiles/PMTilesUtils.java deleted file mode 100644 index 1f8974dd4..000000000 --- a/baremaps-pmtiles/src/main/java/org/apache/baremaps/pmtiles/PMTilesUtils.java +++ /dev/null @@ -1,410 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to you under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.baremaps.pmtiles; - -import com.google.common.math.LongMath; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.nio.ByteBuffer; -import java.nio.ByteOrder; -import java.util.ArrayList; -import java.util.List; - -class PMTilesUtils { - - private static final int HEADER_SIZE_BYTES = 127; - - private PMTilesUtils() { - // Prevent instantiation - } - - static long toNum(long low, long high) { - return high * 0x100000000L + low; - } - - static long readVarIntRemainder(InputStream input, long l) - throws IOException { - long h, b; - b = input.read() & 0xff; - h = (b & 0x70) >> 4; - if (b < 0x80) { - return toNum(l, h); - } - b = input.read() & 0xff; - h |= (b & 0x7f) << 3; - if (b < 0x80) { - return toNum(l, h); - } - b = input.read() & 0xff; - h |= (b & 0x7f) << 10; - if (b < 0x80) { - return toNum(l, h); - } - b = input.read() & 0xff; - h |= (b & 0x7f) << 17; - if (b < 0x80) { - return toNum(l, h); - } - b = input.read() & 0xff; - h |= (b & 0x7f) << 24; - if (b < 0x80) { - return toNum(l, h); - } - b = input.read() & 0xff; - h |= (b & 0x01) << 31; - if (b < 0x80) { - return toNum(l, h); - } - throw new IllegalArgumentException("Expected varint not more than 10 bytes"); - } - - static int writeVarInt(OutputStream output, long value) - throws IOException { - int n = 1; - while (value >= 0x80) { - output.write((byte) (value | 0x80)); - value >>>= 7; - n++; - } - output.write((byte) value); - return n; - } - - static long readVarInt(InputStream input) throws IOException { - long val, b; - b = input.read() & 0xff; - val = b & 0x7f; - if (b < 0x80) { - return val; - } - b = input.read() & 0xff; - val |= (b & 0x7f) << 7; - if (b < 0x80) { - return val; - } - b = input.read() & 0xff; - val |= (b & 0x7f) << 14; - if (b < 0x80) { - return val; - } - b = input.read() & 0xff; - val |= (b & 0x7f) << 21; - if (b < 0x80) { - return val; - } - val |= (b & 0x0f) << 28; - return readVarIntRemainder(input, val); - } - - static void rotate(long n, long[] xy, long rx, long ry) { - if (ry == 0) { - if (rx == 1) { - xy[0] = n - 1 - xy[0]; - xy[1] = n - 1 - xy[1]; - } - long t = xy[0]; - xy[0] = xy[1]; - xy[1] = t; - } - } - - static long[] idOnLevel(int z, long pos) { - long n = LongMath.pow(2, z); - long rx, ry, t = pos; - long[] xy = new long[] {0, 0}; - long s = 1; - while (s < n) { - rx = 1 & (t / 2); - ry = 1 & (t ^ rx); - rotate(s, xy, rx, ry); - xy[0] += s * rx; - xy[1] += s * ry; - t = t / 4; - s *= 2; - } - return new long[] {z, xy[0], xy[1]}; - } - - private static final long[] tzValues = new long[] { - 0, 1, 5, 21, 85, 341, 1365, 5461, 21845, 87381, 349525, 1398101, 5592405, - 22369621, 89478485, 357913941, 1431655765, 5726623061L, 22906492245L, - 91625968981L, 366503875925L, 1466015503701L, 5864062014805L, 23456248059221L, - 93824992236885L, 375299968947541L, 1501199875790165L, - }; - - static long zxyToTileId(int z, long x, long y) { - if (z > 26) { - throw new IllegalArgumentException("Tile zoom level exceeds max safe number limit (26)"); - } - if (x > Math.pow(2, z) - 1 || y > Math.pow(2, z) - 1) { - throw new IllegalArgumentException("tile x/y outside zoom level bounds"); - } - long acc = tzValues[z]; - long n = LongMath.pow(2, z); - long rx = 0; - long ry = 0; - long d = 0; - long[] xy = new long[] {x, y}; - long s = n / 2; - while (s > 0) { - rx = (xy[0] & s) > 0 ? 1 : 0; - ry = (xy[1] & s) > 0 ? 1 : 0; - d += s * s * ((3 * rx) ^ ry); - rotate(s, xy, rx, ry); - s = s / 2; - } - return acc + d; - } - - static long[] tileIdToZxy(long i) { - long acc = 0; - for (int z = 0; z < 27; z++) { - long numTiles = (0x1L << z) * (0x1L << z); - if (acc + numTiles > i) { - return idOnLevel(z, i - acc); - } - acc += numTiles; - } - throw new IllegalArgumentException("Tile zoom level exceeds max safe number limit (26)"); - } - - static Header deserializeHeader(InputStream input) throws IOException { - byte[] bytes = new byte[HEADER_SIZE_BYTES]; - var num = input.read(bytes); - if (num != HEADER_SIZE_BYTES) { - throw new IOException("Invalid header size"); - } - var buffer = ByteBuffer.wrap(bytes).order(ByteOrder.LITTLE_ENDIAN); - buffer.position(7); - return new Header( - buffer.get(), - buffer.getLong(), - buffer.getLong(), - buffer.getLong(), - buffer.getLong(), - buffer.getLong(), - buffer.getLong(), - buffer.getLong(), - buffer.getLong(), - buffer.getLong(), - buffer.getLong(), - buffer.getLong(), - buffer.get() == 1, - Compression.values()[buffer.get()], - Compression.values()[buffer.get()], - TileType.values()[buffer.get()], - buffer.get(), - buffer.get(), - (double) buffer.getInt() / 10000000, - (double) buffer.getInt() / 10000000, - (double) buffer.getInt() / 10000000, - (double) buffer.getInt() / 10000000, - buffer.get(), - (double) buffer.getInt() / 10000000, - (double) buffer.getInt() / 10000000); - } - - static byte[] serializeHeader(Header header) { - var buffer = ByteBuffer.allocate(HEADER_SIZE_BYTES).order(ByteOrder.LITTLE_ENDIAN); - buffer.put((byte) 0x50); - buffer.put((byte) 0x4D); - buffer.put((byte) 0x54); - buffer.put((byte) 0x69); - buffer.put((byte) 0x6C); - buffer.put((byte) 0x65); - buffer.put((byte) 0x73); - buffer.put((byte) header.getSpecVersion()); - buffer.putLong(header.getRootDirectoryOffset()); - buffer.putLong(header.getRootDirectoryLength()); - buffer.putLong(header.getJsonMetadataOffset()); - buffer.putLong(header.getJsonMetadataLength()); - buffer.putLong(header.getLeafDirectoryOffset()); - buffer.putLong(header.getLeafDirectoryLength()); - buffer.putLong(header.getTileDataOffset()); - buffer.putLong(header.getTileDataLength()); - buffer.putLong(header.getNumAddressedTiles()); - buffer.putLong(header.getNumTileEntries()); - buffer.putLong(header.getNumTileContents()); - buffer.put((byte) (header.isClustered() ? 1 : 0)); - buffer.put((byte) header.getInternalCompression().ordinal()); - buffer.put((byte) header.getTileCompression().ordinal()); - buffer.put((byte) header.getTileType().ordinal()); - buffer.put((byte) header.getMinZoom()); - buffer.put((byte) header.getMaxZoom()); - buffer.putInt((int) (header.getMinLon() * 10000000)); - buffer.putInt((int) (header.getMinLat() * 10000000)); - buffer.putInt((int) (header.getMaxLon() * 10000000)); - buffer.putInt((int) (header.getMaxLat() * 10000000)); - buffer.put((byte) header.getCenterZoom()); - buffer.putInt((int) (header.getCenterLon() * 10000000)); - buffer.putInt((int) (header.getCenterLat() * 10000000)); - buffer.flip(); - return buffer.array(); - } - - static void serializeEntries(OutputStream output, List<Entry> entries) - throws IOException { - var buffer = ByteBuffer.allocate(entries.size() * 48); - writeVarInt(output, entries.size()); - long lastId = 0; - for (Entry entry : entries) { - writeVarInt(output, entry.getTileId() - lastId); - lastId = entry.getTileId(); - } - for (Entry entry : entries) { - writeVarInt(output, entry.getRunLength()); - } - for (Entry entry : entries) { - writeVarInt(output, entry.getLength()); - } - for (int i = 0; i < entries.size(); i++) { - Entry entry = entries.get(i); - if (i > 0 - && entry.getOffset() == entries.get(i - 1).getOffset() + entries.get(i - 1).getLength()) { - writeVarInt(output, 0); - } else { - writeVarInt(output, entry.getOffset() + 1); - } - } - buffer.flip(); - output.write(buffer.array(), 0, buffer.limit()); - } - - static List<Entry> deserializeEntries(InputStream buffer) - throws IOException { - long numEntries = readVarInt(buffer); - List<Entry> entries = new ArrayList<>((int) numEntries); - long lastId = 0; - for (int i = 0; i < numEntries; i++) { - long value = readVarInt(buffer); - lastId = lastId + value; - Entry entry = new Entry(); - entry.setTileId(lastId); - entries.add(entry); - } - for (int i = 0; i < numEntries; i++) { - long value = readVarInt(buffer); - entries.get(i).setRunLength(value); - } - for (int i = 0; i < numEntries; i++) { - long value = readVarInt(buffer); - entries.get(i).setLength(value); - } - for (int i = 0; i < numEntries; i++) { - long value = readVarInt(buffer); - if (value == 0 && i > 0) { - Entry prevEntry = entries.get(i - 1); - entries.get(i).setOffset(prevEntry.getOffset() + prevEntry.getLength()); - } else { - entries.get(i).setOffset(value - 1); - } - } - return entries; - } - - static Entry findTile(List<Entry> entries, long tileId) { - int m = 0; - int n = entries.size() - 1; - while (m <= n) { - int k = (n + m) >> 1; - long cmp = tileId - entries.get(k).getTileId(); - if (cmp > 0) { - m = k + 1; - } else if (cmp < 0) { - n = k - 1; - } else { - return entries.get(k); - } - } - - // at this point, m > n - if (n >= 0) { - if (entries.get(n).getRunLength() == 0) { - return entries.get(n); - } - if (tileId - entries.get(n).getTileId() < entries.get(n).getRunLength()) { - return entries.get(n); - } - } - return null; - } - - static Directories buildRootLeaves(List<Entry> entries, int leafSize, - Compression compression) throws IOException { - var rootEntries = new ArrayList<Entry>(); - var numLeaves = 0; - byte[] leavesBytes; - byte[] rootBytes; - - try (var leavesOutput = new ByteArrayOutputStream()) { - for (var i = 0; i < entries.size(); i += leafSize) { - numLeaves++; - var end = i + leafSize; - if (i + leafSize > entries.size()) { - end = entries.size(); - } - var offset = leavesOutput.size(); - try (var leafOutput = new ByteArrayOutputStream()) { - try (var compressedLeafOutput = compression.compress(leafOutput)) { - serializeEntries(compressedLeafOutput, entries.subList(i, end)); - } - var length = leafOutput.size(); - rootEntries.add(new Entry(entries.get(i).getTileId(), offset, length, 0)); - leavesOutput.write(leafOutput.toByteArray()); - } - } - leavesBytes = leavesOutput.toByteArray(); - } - - try (var rootOutput = new ByteArrayOutputStream()) { - try (var compressedRootOutput = compression.compress(rootOutput)) { - serializeEntries(compressedRootOutput, rootEntries); - } - rootBytes = rootOutput.toByteArray(); - } - - return new Directories(rootBytes, leavesBytes, numLeaves); - } - - static Directories optimizeDirectories(List<Entry> entries, int targetRootLength, - Compression compression) - throws IOException { - if (entries.size() < 16384) { - try (var rootOutput = new ByteArrayOutputStream()) { - try (var compressedOutput = compression.compress(rootOutput)) { - serializeEntries(compressedOutput, entries); - } - byte[] rootBytes = rootOutput.toByteArray(); - if (rootBytes.length <= targetRootLength) { - return new Directories(rootBytes, new byte[] {}, 0); - } - } - } - - double leafSize = Math.max((double) entries.size() / 3500, 4096); - for (;;) { - var directories = buildRootLeaves(entries, (int) leafSize, compression); - if (directories.getRoot().length <= targetRootLength) { - return directories; - } - leafSize = leafSize * 1.2; - } - } -} diff --git a/baremaps-pmtiles/src/main/java/org/apache/baremaps/pmtiles/PMTilesWriter.java b/baremaps-pmtiles/src/main/java/org/apache/baremaps/pmtiles/PMTilesWriter.java index 2e90d579d..ff835f5c1 100644 --- a/baremaps-pmtiles/src/main/java/org/apache/baremaps/pmtiles/PMTilesWriter.java +++ b/baremaps-pmtiles/src/main/java/org/apache/baremaps/pmtiles/PMTilesWriter.java @@ -26,40 +26,28 @@ import java.nio.file.Files; import java.nio.file.Path; import java.util.*; -public class PMTilesWriter { +public class PMTilesWriter implements AutoCloseable { private final Compression compression; - private final Path path; - private final List<Entry> entries; - private final Map<Long, Long> tileHashToOffset; - private final Path tilePath; + private final HeaderSerializer headerSerializer; + private final DirectorySerializer directorySerializer; + private boolean isClosed = false; private Map<String, Object> metadata = new HashMap<>(); - private Long lastTileHash = null; - private boolean clustered = true; - private int minZoom = 0; - private int maxZoom = 14; - private double minLon = -180; - private double minLat = -90; - private double maxLon = 180; - private double maxLat = 90; - private int centerZoom = 3; - private double centerLat = 0; - private double centerLon = 0; public PMTilesWriter(Path path) throws IOException { @@ -73,7 +61,23 @@ public class PMTilesWriter { this.path = path; this.entries = entries; this.tileHashToOffset = tileHashToOffset; - this.tilePath = Files.createTempFile(path.getParent(), "tiles_", ".tmp"); + + Path tempPath = null; + try { + tempPath = Files.createTempFile(path.getParent(), "tiles_", ".tmp"); + this.tilePath = tempPath; + this.headerSerializer = new HeaderSerializer(); + this.directorySerializer = new DirectorySerializer(); + } catch (IOException e) { + if (tempPath != null && Files.exists(tempPath)) { + try { + Files.delete(tempPath); + } catch (IOException ex) { + e.addSuppressed(ex); + } + } + throw e; + } } public void setMetadata(Map<String, Object> metadata) { @@ -81,8 +85,12 @@ public class PMTilesWriter { } public void setTile(int z, int x, int y, byte[] bytes) throws IOException { + if (isClosed) { + throw new IOException("PMTilesWriter has been closed"); + } + // Write the tile - var tileId = PMTilesUtils.zxyToTileId(z, x, y); + var tileId = TileIdConverter.zxyToTileId(z, x, y); var tileLength = bytes.length; Long tileHash = Hashing.farmHashFingerprint64().hashBytes(bytes).asLong(); @@ -100,7 +108,12 @@ public class PMTilesWriter { // Else, if the tile is the same as the last one, increment the run length else if (tileHashToOffset.containsKey(tileHash)) { var tileOffset = tileHashToOffset.get(tileHash); - entries.add(new Entry(tileId, tileOffset, tileLength, 1)); + entries.add(Entry.builder() + .tileId(tileId) + .offset(tileOffset) + .length(tileLength) + .runLength(1) + .build()); } // Else, write the tile and add it to the index @@ -110,7 +123,12 @@ public class PMTilesWriter { lastTileHash = tileHash; try (var output = new FileOutputStream(tilePath.toFile(), true)) { output.write(bytes); - entries.add(new Entry(tileId, tileOffset, tileLength, 1)); + entries.add(Entry.builder() + .tileId(tileId) + .offset(tileOffset) + .length(tileLength) + .runLength(1) + .build()); } } } @@ -152,12 +170,16 @@ public class PMTilesWriter { } public void write() throws IOException { + if (isClosed) { + throw new IOException("PMTilesWriter has been closed"); + } + // Sort the entries by tile id if (!clustered) { entries.sort(Comparator.comparingLong(Entry::getTileId)); } - var directories = PMTilesUtils.optimizeDirectories(entries, 16247, compression); + var directories = directorySerializer.optimizeDirectories(entries, 16247, compression); byte[] metadataBytes; try (var metadataOutput = new ByteArrayOutputStream()) { @@ -177,43 +199,47 @@ public class PMTilesWriter { var tilesLength = Files.size(tilePath); var numTiles = entries.size(); - var header = new Header(); - header.setNumAddressedTiles(numTiles); - header.setNumTileEntries(numTiles); - header.setNumTileContents(tileHashToOffset.size()); - header.setClustered(true); - - header.setInternalCompression(compression); - header.setTileCompression(compression); - header.setTileType(TileType.MVT); - header.setRootOffset(rootOffset); - header.setRootLength(rootLength); - header.setMetadataOffset(metadataOffset); - header.setMetadataLength(metadataLength); - header.setLeavesOffset(leavesOffset); - header.setLeavesLength(leavesLength); - header.setTilesOffset(tilesOffset); - header.setTilesLength(tilesLength); - - header.setMinZoom(minZoom); - header.setMaxZoom(maxZoom); - header.setMinLon(minLon); - header.setMinLat(minLat); - header.setMaxLon(maxLon); - header.setMaxLat(maxLat); - header.setCenterZoom(centerZoom); - header.setCenterLat(centerLat); - header.setCenterLon(centerLon); + // Use builder pattern for creating Header + var header = Header.builder() + .numAddressedTiles(numTiles) + .numTileEntries(numTiles) + .numTileContents(tileHashToOffset.size()) + .clustered(true) + .internalCompression(compression) + .tileCompression(compression) + .tileType(TileType.MVT) + .rootDirectoryOffset(rootOffset) + .rootDirectoryLength(rootLength) + .jsonMetadataOffset(metadataOffset) + .jsonMetadataLength(metadataLength) + .leafDirectoryOffset(leavesOffset) + .leafDirectoryLength(leavesLength) + .tileDataOffset(tilesOffset) + .tileDataLength(tilesLength) + .minZoom(minZoom) + .maxZoom(maxZoom) + .bounds(minLon, minLat, maxLon, maxLat) + .center(centerZoom, centerLon, centerLat) + .build(); try (var output = new FileOutputStream(path.toFile())) { - output.write(PMTilesUtils.serializeHeader(header)); + headerSerializer.serialize(header, output); output.write(directories.getRoot()); output.write(metadataBytes); output.write(directories.getLeaves()); Files.copy(tilePath, output); } finally { - Files.delete(tilePath); + close(); } } + @Override + public void close() throws IOException { + if (!isClosed) { + if (Files.exists(tilePath)) { + Files.delete(tilePath); + } + isClosed = true; + } + } } diff --git a/baremaps-pmtiles/src/main/java/org/apache/baremaps/pmtiles/Directories.java b/baremaps-pmtiles/src/main/java/org/apache/baremaps/pmtiles/Serializer.java similarity index 52% copy from baremaps-pmtiles/src/main/java/org/apache/baremaps/pmtiles/Directories.java copy to baremaps-pmtiles/src/main/java/org/apache/baremaps/pmtiles/Serializer.java index 2306efdfc..8e0d00591 100644 --- a/baremaps-pmtiles/src/main/java/org/apache/baremaps/pmtiles/Directories.java +++ b/baremaps-pmtiles/src/main/java/org/apache/baremaps/pmtiles/Serializer.java @@ -17,27 +17,32 @@ package org.apache.baremaps.pmtiles; -class Directories { +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; - private final byte[] root; - private final byte[] leaves; - private final int numLeaves; - - public Directories(byte[] root, byte[] leaves, int numLeaves) { - this.root = root; - this.leaves = leaves; - this.numLeaves = numLeaves; - } - - public byte[] getRoot() { - return root; - } +/** + * Generic serializer interface for PMTiles components. + * + * @param <T> the type of object to serialize/deserialize + */ +public interface Serializer<T> { - public byte[] getLeaves() { - return leaves; - } + /** + * Serialize an object to an output stream. + * + * @param object the object to serialize + * @param output the output stream to write to + * @throws IOException if an I/O error occurs + */ + void serialize(T object, OutputStream output) throws IOException; - public int getNumLeaves() { - return numLeaves; - } + /** + * Deserialize an object from an input stream. + * + * @param input the input stream to read from + * @return the deserialized object + * @throws IOException if an I/O error occurs + */ + T deserialize(InputStream input) throws IOException; } diff --git a/baremaps-pmtiles/src/main/java/org/apache/baremaps/pmtiles/TileIdConverter.java b/baremaps-pmtiles/src/main/java/org/apache/baremaps/pmtiles/TileIdConverter.java new file mode 100644 index 000000000..e701abeca --- /dev/null +++ b/baremaps-pmtiles/src/main/java/org/apache/baremaps/pmtiles/TileIdConverter.java @@ -0,0 +1,127 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.baremaps.pmtiles; + +import com.google.common.math.LongMath; + +/** + * Utility to convert between tile coordinates and tileIds. + */ +class TileIdConverter { + + private static final long[] TZ_VALUES = new long[] { + 0, 1, 5, 21, 85, 341, 1365, 5461, 21845, 87381, 349525, 1398101, 5592405, + 22369621, 89478485, 357913941, 1431655765, 5726623061L, 22906492245L, + 91625968981L, 366503875925L, 1466015503701L, 5864062014805L, 23456248059221L, + 93824992236885L, 375299968947541L, 1501199875790165L, + }; + + /** + * Rotate coordinates. + * + * @param n the size of the quadrant + * @param xy the coordinates to rotate + * @param rx the x transform + * @param ry the y transform + */ + private static void rotate(long n, long[] xy, long rx, long ry) { + if (ry == 0) { + if (rx == 1) { + xy[0] = n - 1 - xy[0]; + xy[1] = n - 1 - xy[1]; + } + long t = xy[0]; + xy[0] = xy[1]; + xy[1] = t; + } + } + + /** + * Convert a position to z, x, y coordinates. + * + * @param z the zoom level + * @param pos the position + * @return the z, x, y coordinates + */ + public static long[] idOnLevel(int z, long pos) { + long n = LongMath.pow(2, z); + long rx, ry, t = pos; + long[] xy = new long[] {0, 0}; + long s = 1; + while (s < n) { + rx = 1 & (t / 2); + ry = 1 & (t ^ rx); + rotate(s, xy, rx, ry); + xy[0] += s * rx; + xy[1] += s * ry; + t = t / 4; + s *= 2; + } + return new long[] {z, xy[0], xy[1]}; + } + + /** + * Convert z, x, y coordinates to a tileId. + * + * @param z the zoom level + * @param x the x coordinate + * @param y the y coordinate + * @return the tileId + */ + public static long zxyToTileId(int z, long x, long y) { + if (z > 26) { + throw new IllegalArgumentException("Tile zoom level exceeds max safe number limit (26)"); + } + if (x > Math.pow(2, z) - 1 || y > Math.pow(2, z) - 1) { + throw new IllegalArgumentException("tile x/y outside zoom level bounds"); + } + long acc = TZ_VALUES[z]; + long n = LongMath.pow(2, z); + long rx = 0; + long ry = 0; + long d = 0; + long[] xy = new long[] {x, y}; + long s = n / 2; + while (s > 0) { + rx = (xy[0] & s) > 0 ? 1 : 0; + ry = (xy[1] & s) > 0 ? 1 : 0; + d += s * s * ((3 * rx) ^ ry); + rotate(s, xy, rx, ry); + s = s / 2; + } + return acc + d; + } + + /** + * Convert a tileId to z, x, y coordinates. + * + * @param i the tileId + * @return the z, x, y coordinates + */ + public static long[] tileIdToZxy(long i) { + long acc = 0; + for (int z = 0; z < 27; z++) { + long numTiles = (0x1L << z) * (0x1L << z); + if (acc + numTiles > i) { + return idOnLevel(z, i - acc); + } + acc += numTiles; + } + throw new IllegalArgumentException("Tile zoom level exceeds max safe number limit (26)"); + } +} diff --git a/baremaps-pmtiles/src/main/java/org/apache/baremaps/pmtiles/VarIntSerializer.java b/baremaps-pmtiles/src/main/java/org/apache/baremaps/pmtiles/VarIntSerializer.java new file mode 100644 index 000000000..a7f8685d9 --- /dev/null +++ b/baremaps-pmtiles/src/main/java/org/apache/baremaps/pmtiles/VarIntSerializer.java @@ -0,0 +1,138 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.baremaps.pmtiles; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; + +/** + * Serializer for variable-length integers used in PMTiles format. + */ +class VarIntSerializer { + + VarIntSerializer() { + + } + + /** + * Combine low and high bits into a single number. + * + * @param low the low 32 bits + * @param high the high 32 bits + * @return the combined 64-bit number + */ + private long toNum(long low, long high) { + return high * 0x100000000L + low; + } + + /** + * Read variable integer remainder from input stream. + * + * @param input the input stream + * @param l the low bits + * @return the read varint + * @throws IOException if an I/O error occurs + */ + private long readVarIntRemainder(InputStream input, long l) throws IOException { + long h, b; + b = input.read() & 0xff; + h = (b & 0x70) >> 4; + if (b < 0x80) { + return toNum(l, h); + } + b = input.read() & 0xff; + h |= (b & 0x7f) << 3; + if (b < 0x80) { + return toNum(l, h); + } + b = input.read() & 0xff; + h |= (b & 0x7f) << 10; + if (b < 0x80) { + return toNum(l, h); + } + b = input.read() & 0xff; + h |= (b & 0x7f) << 17; + if (b < 0x80) { + return toNum(l, h); + } + b = input.read() & 0xff; + h |= (b & 0x7f) << 24; + if (b < 0x80) { + return toNum(l, h); + } + b = input.read() & 0xff; + h |= (b & 0x01) << 31; + if (b < 0x80) { + return toNum(l, h); + } + throw new IllegalArgumentException("Expected varint not more than 10 bytes"); + } + + /** + * Write a variable-length integer to the output stream. + * + * @param output the output stream + * @param value the value to write + * @return the number of bytes written + * @throws IOException if an I/O error occurs + */ + public int writeVarInt(OutputStream output, long value) throws IOException { + int n = 1; + while (value >= 0x80) { + output.write((byte) (value | 0x80)); + value >>>= 7; + n++; + } + output.write((byte) value); + return n; + } + + /** + * Read a variable-length integer from the input stream. + * + * @param input the input stream + * @return the read varint + * @throws IOException if an I/O error occurs + */ + public long readVarInt(InputStream input) throws IOException { + long val, b; + b = input.read() & 0xff; + val = b & 0x7f; + if (b < 0x80) { + return val; + } + b = input.read() & 0xff; + val |= (b & 0x7f) << 7; + if (b < 0x80) { + return val; + } + b = input.read() & 0xff; + val |= (b & 0x7f) << 14; + if (b < 0x80) { + return val; + } + b = input.read() & 0xff; + val |= (b & 0x7f) << 21; + if (b < 0x80) { + return val; + } + val |= (b & 0x0f) << 28; + return readVarIntRemainder(input, val); + } +} diff --git a/baremaps-pmtiles/src/test/java/org/apache/baremaps/pmtiles/DirectorySerializerTest.java b/baremaps-pmtiles/src/test/java/org/apache/baremaps/pmtiles/DirectorySerializerTest.java new file mode 100644 index 000000000..ef2d62119 --- /dev/null +++ b/baremaps-pmtiles/src/test/java/org/apache/baremaps/pmtiles/DirectorySerializerTest.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.baremaps.pmtiles; + +import static org.junit.jupiter.api.Assertions.*; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Random; +import org.junit.jupiter.api.Test; + +/** + * Tests for the DirectorySerializer class. + */ +class DirectorySerializerTest { + + private final DirectorySerializer directorySerializer = new DirectorySerializer(); + + @Test + void buildRootLeaves() throws IOException { + var entries = List.of(Entry.builder().tileId(100).offset(1).length(1).runLength(0).build()); + var directories = directorySerializer.buildRootLeaves(entries, 1, Compression.NONE); + assertEquals(1, directories.getNumLeaves()); + } + + @Test + void optimizeDirectories() throws IOException { + var random = new Random(3857); + var entries = new ArrayList<Entry>(); + entries.add(Entry.builder().tileId(0).offset(0).length(100).runLength(1).build()); + var directories = directorySerializer.optimizeDirectories(entries, 100, Compression.NONE); + assertFalse(directories.getLeaves().length > 0); + assertEquals(0, directories.getNumLeaves()); + + entries = new ArrayList<>(); + int offset = 0; + for (var i = 0; i < 1000; i++) { + var randTileSize = random.nextInt(1000000); + entries + .add(Entry.builder().tileId(i).offset(offset).length(randTileSize).runLength(1).build()); + offset += randTileSize; + } + directories = directorySerializer.optimizeDirectories(entries, 1024, Compression.NONE); + assertFalse(directories.getRoot().length > 1024); + assertNotEquals(0, directories.getNumLeaves()); + assertNotEquals(0, directories.getLeaves().length); + } +} diff --git a/baremaps-pmtiles/src/test/java/org/apache/baremaps/pmtiles/EntrySerializerTest.java b/baremaps-pmtiles/src/test/java/org/apache/baremaps/pmtiles/EntrySerializerTest.java new file mode 100644 index 000000000..c13ab44cf --- /dev/null +++ b/baremaps-pmtiles/src/test/java/org/apache/baremaps/pmtiles/EntrySerializerTest.java @@ -0,0 +1,87 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.baremaps.pmtiles; + +import static org.junit.jupiter.api.Assertions.*; + +import java.util.ArrayList; +import org.junit.jupiter.api.Test; + +/** + * Tests for the EntrySerializer class. + */ +class EntrySerializerTest { + + private final EntrySerializer entrySerializer = new EntrySerializer(); + + @Test + void searchForMissingEntry() { + var entries = new ArrayList<Entry>(); + assertNull(entrySerializer.findTile(entries, 101)); + } + + @Test + void searchForFirstEntry() { + var entry = Entry.builder().tileId(100).offset(1).length(1).runLength(1).build(); + var entries = new ArrayList<Entry>(); + entries.add(entry); + assertEquals(entry, entrySerializer.findTile(entries, 100)); + } + + @Test + void searchWithRunLength() { + var entry = Entry.builder().tileId(3).offset(3).length(1).runLength(2).build(); + var entries = new ArrayList<Entry>(); + entries.add(entry); + entries.add(Entry.builder().tileId(5).offset(5).length(1).runLength(2).build()); + assertEquals(entry, entrySerializer.findTile(entries, 4)); + } + + @Test + void searchWithMultipleTileEntries() { + var entries = new ArrayList<Entry>(); + entries.add(Entry.builder().tileId(100).offset(1).length(1).runLength(2).build()); + var entry = entrySerializer.findTile(entries, 101); + assertEquals(1, entry.getOffset()); + assertEquals(1, entry.getLength()); + + entries = new ArrayList<Entry>(); + entries.add(Entry.builder().tileId(100).offset(1).length(1).runLength(1).build()); + entries.add(Entry.builder().tileId(150).offset(2).length(2).runLength(2).build()); + entry = entrySerializer.findTile(entries, 151); + assertEquals(2, entry.getOffset()); + assertEquals(2, entry.getLength()); + + entries = new ArrayList<>(); + entries.add(Entry.builder().tileId(50).offset(1).length(1).runLength(2).build()); + entries.add(Entry.builder().tileId(100).offset(2).length(2).runLength(1).build()); + entries.add(Entry.builder().tileId(150).offset(3).length(3).runLength(1).build()); + entry = entrySerializer.findTile(entries, 51); + assertEquals(1, entry.getOffset()); + assertEquals(1, entry.getLength()); + } + + @Test + void leafSearch() { + var entries = new ArrayList<Entry>(); + entries.add(Entry.builder().tileId(100).offset(1).length(1).runLength(0).build()); + var entry = entrySerializer.findTile(entries, 150); + assertEquals(1, entry.getOffset()); + assertEquals(1, entry.getLength()); + } +} diff --git a/baremaps-pmtiles/src/test/java/org/apache/baremaps/pmtiles/HeaderSerializerTest.java b/baremaps-pmtiles/src/test/java/org/apache/baremaps/pmtiles/HeaderSerializerTest.java new file mode 100644 index 000000000..20c57f040 --- /dev/null +++ b/baremaps-pmtiles/src/test/java/org/apache/baremaps/pmtiles/HeaderSerializerTest.java @@ -0,0 +1,100 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.baremaps.pmtiles; + +import static org.junit.jupiter.api.Assertions.*; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.nio.channels.Channels; +import java.nio.channels.FileChannel; +import org.apache.baremaps.testing.TestFiles; +import org.junit.jupiter.api.Test; + +/** + * Tests for the HeaderSerializer class. + */ +class HeaderSerializerTest { + + private final HeaderSerializer headerSerializer = new HeaderSerializer(); + + @Test + void decodeHeader() throws IOException { + var file = TestFiles.resolve("baremaps-testing/data/pmtiles/test_fixture_1.pmtiles"); + try (var channel = FileChannel.open(file)) { + var input = Channels.newInputStream(channel); + var header = headerSerializer.deserialize(input); + assertEquals(127, header.getRootDirectoryOffset()); + assertEquals(25, header.getRootDirectoryLength()); + assertEquals(152, header.getJsonMetadataOffset()); + assertEquals(247, header.getJsonMetadataLength()); + assertEquals(0, header.getLeafDirectoryOffset()); + assertEquals(0, header.getLeafDirectoryLength()); + assertEquals(399, header.getTileDataOffset()); + assertEquals(69, header.getTileDataLength()); + assertEquals(1, header.getNumAddressedTiles()); + assertEquals(1, header.getNumTileEntries()); + assertEquals(1, header.getNumTileContents()); + assertFalse(header.isClustered()); + assertEquals(Compression.GZIP, header.getInternalCompression()); + assertEquals(Compression.GZIP, header.getTileCompression()); + assertEquals(TileType.MVT, header.getTileType()); + assertEquals(0, header.getMinZoom()); + assertEquals(0, header.getMaxZoom()); + assertEquals(0, header.getMinLon()); + assertEquals(0, header.getMinLat()); + assertEquals(1, Math.round(header.getMaxLon())); + assertEquals(1, Math.round(header.getMaxLat())); + } + } + + @Test + void encodeHeader() throws IOException { + var header = Header.builder() + .specVersion(127) + .rootDirectoryOffset(25) + .rootDirectoryLength(152) + .jsonMetadataOffset(247) + .jsonMetadataLength(0) + .leafDirectoryOffset(0) + .leafDirectoryLength(399) + .tileDataOffset(69) + .tileDataLength(1) + .numAddressedTiles(1) + .numTileEntries(1) + .numTileContents(10) + .clustered(false) + .internalCompression(Compression.GZIP) + .tileCompression(Compression.GZIP) + .tileType(TileType.MVT) + .minZoom(0) + .maxZoom(0) + .bounds(0, 1, 1, 0) + .center(0, 0, 0) + .build(); + + var array = new ByteArrayOutputStream(); + headerSerializer.serialize(header, array); + + var input = new ByteArrayInputStream(array.toByteArray()); + var header2 = headerSerializer.deserialize(input); + + assertEquals(header, header2); + } +} diff --git a/baremaps-pmtiles/src/test/java/org/apache/baremaps/pmtiles/PMTilesUtilsTest.java b/baremaps-pmtiles/src/test/java/org/apache/baremaps/pmtiles/PMTilesSerializerTest.java similarity index 54% rename from baremaps-pmtiles/src/test/java/org/apache/baremaps/pmtiles/PMTilesUtilsTest.java rename to baremaps-pmtiles/src/test/java/org/apache/baremaps/pmtiles/PMTilesSerializerTest.java index affc6f502..b5650373c 100644 --- a/baremaps-pmtiles/src/test/java/org/apache/baremaps/pmtiles/PMTilesUtilsTest.java +++ b/baremaps-pmtiles/src/test/java/org/apache/baremaps/pmtiles/PMTilesSerializerTest.java @@ -32,7 +32,12 @@ import java.util.Random; import org.apache.baremaps.testing.TestFiles; import org.junit.jupiter.api.Test; -class PMTilesUtilsTest { +class PMTilesSerializerTest { + + private final VarIntSerializer varIntSerializer = new VarIntSerializer(); + private final HeaderSerializer headerSerializer = new HeaderSerializer(); + private final EntrySerializer entrySerializer = new EntrySerializer(); + private final DirectorySerializer directorySerializer = new DirectorySerializer(); @Test void decodeVarInt() throws IOException { @@ -41,53 +46,53 @@ class PMTilesUtilsTest { (byte) 127, (byte) 0xe5, (byte) 0x8e, (byte) 0x26 })); - assertEquals(0, PMTilesUtils.readVarInt(b)); - assertEquals(1, PMTilesUtils.readVarInt(b)); - assertEquals(127, PMTilesUtils.readVarInt(b)); - assertEquals(624485, PMTilesUtils.readVarInt(b)); + assertEquals(0, varIntSerializer.readVarInt(b)); + assertEquals(1, varIntSerializer.readVarInt(b)); + assertEquals(127, varIntSerializer.readVarInt(b)); + assertEquals(624485, varIntSerializer.readVarInt(b)); b = new LittleEndianDataInputStream(new ByteArrayInputStream(new byte[] { (byte) 0xff, (byte) 0xff, (byte) 0xff, (byte) 0xff, (byte) 0xff, (byte) 0xff, (byte) 0xff, (byte) 0x0f, })); - assertEquals(9007199254740991L, PMTilesUtils.readVarInt(b)); + assertEquals(9007199254740991L, varIntSerializer.readVarInt(b)); } @Test void encodeVarInt() throws IOException { for (long i = 0; i < 1000; i++) { var array = new ByteArrayOutputStream(); - PMTilesUtils.writeVarInt(array, i); + varIntSerializer.writeVarInt(array, i); var input = new LittleEndianDataInputStream(new ByteArrayInputStream(array.toByteArray())); - assertEquals(i, PMTilesUtils.readVarInt(input)); + assertEquals(i, varIntSerializer.readVarInt(input)); } for (long i = Long.MAX_VALUE - 1000; i < Long.MAX_VALUE; i++) { var array = new ByteArrayOutputStream(); - PMTilesUtils.writeVarInt(array, i); + varIntSerializer.writeVarInt(array, i); var input = new LittleEndianDataInputStream(new ByteArrayInputStream(array.toByteArray())); - assertEquals(i, PMTilesUtils.readVarInt(input)); + assertEquals(i, varIntSerializer.readVarInt(input)); } } @Test void zxyToTileId() { - assertEquals(0, PMTilesUtils.zxyToTileId(0, 0, 0)); - assertEquals(1, PMTilesUtils.zxyToTileId(1, 0, 0)); - assertEquals(2, PMTilesUtils.zxyToTileId(1, 0, 1)); - assertEquals(3, PMTilesUtils.zxyToTileId(1, 1, 1)); - assertEquals(4, PMTilesUtils.zxyToTileId(1, 1, 0)); - assertEquals(5, PMTilesUtils.zxyToTileId(2, 0, 0)); + assertEquals(0, TileIdConverter.zxyToTileId(0, 0, 0)); + assertEquals(1, TileIdConverter.zxyToTileId(1, 0, 0)); + assertEquals(2, TileIdConverter.zxyToTileId(1, 0, 1)); + assertEquals(3, TileIdConverter.zxyToTileId(1, 1, 1)); + assertEquals(4, TileIdConverter.zxyToTileId(1, 1, 0)); + assertEquals(5, TileIdConverter.zxyToTileId(2, 0, 0)); } @Test void tileIdToZxy() { - assertArrayEquals(new long[] {0, 0, 0}, PMTilesUtils.tileIdToZxy(0)); - assertArrayEquals(new long[] {1, 0, 0}, PMTilesUtils.tileIdToZxy(1)); - assertArrayEquals(new long[] {1, 0, 1}, PMTilesUtils.tileIdToZxy(2)); - assertArrayEquals(new long[] {1, 1, 1}, PMTilesUtils.tileIdToZxy(3)); - assertArrayEquals(new long[] {1, 1, 0}, PMTilesUtils.tileIdToZxy(4)); - assertArrayEquals(new long[] {2, 0, 0}, PMTilesUtils.tileIdToZxy(5)); + assertArrayEquals(new long[] {0, 0, 0}, TileIdConverter.tileIdToZxy(0)); + assertArrayEquals(new long[] {1, 0, 0}, TileIdConverter.tileIdToZxy(1)); + assertArrayEquals(new long[] {1, 0, 1}, TileIdConverter.tileIdToZxy(2)); + assertArrayEquals(new long[] {1, 1, 1}, TileIdConverter.tileIdToZxy(3)); + assertArrayEquals(new long[] {1, 1, 0}, TileIdConverter.tileIdToZxy(4)); + assertArrayEquals(new long[] {2, 0, 0}, TileIdConverter.tileIdToZxy(5)); } @Test @@ -95,7 +100,7 @@ class PMTilesUtilsTest { for (int z = 0; z < 9; z++) { for (long x = 0; x < 1 << z; x++) { for (long y = 0; y < 1 << z; y++) { - var result = PMTilesUtils.tileIdToZxy(PMTilesUtils.zxyToTileId(z, x, y)); + var result = TileIdConverter.tileIdToZxy(TileIdConverter.zxyToTileId(z, x, y)); if (result[0] != z || result[1] != x || result[2] != y) { fail("roundtrip failed"); } @@ -108,30 +113,30 @@ class PMTilesUtilsTest { void tileExtremes() { for (var z = 0; z < 27; z++) { var dim = LongMath.pow(2, z) - 1; - var tl = PMTilesUtils.tileIdToZxy(PMTilesUtils.zxyToTileId(z, 0, 0)); + var tl = TileIdConverter.tileIdToZxy(TileIdConverter.zxyToTileId(z, 0, 0)); assertArrayEquals(new long[] {z, 0, 0}, tl); - var tr = PMTilesUtils.tileIdToZxy(PMTilesUtils.zxyToTileId(z, dim, 0)); + var tr = TileIdConverter.tileIdToZxy(TileIdConverter.zxyToTileId(z, dim, 0)); assertArrayEquals(new long[] {z, dim, 0}, tr); - var bl = PMTilesUtils.tileIdToZxy(PMTilesUtils.zxyToTileId(z, 0, dim)); + var bl = TileIdConverter.tileIdToZxy(TileIdConverter.zxyToTileId(z, 0, dim)); assertArrayEquals(new long[] {z, 0, dim}, bl); - var br = PMTilesUtils.tileIdToZxy(PMTilesUtils.zxyToTileId(z, dim, dim)); + var br = TileIdConverter.tileIdToZxy(TileIdConverter.zxyToTileId(z, dim, dim)); assertArrayEquals(new long[] {z, dim, dim}, br); } } @Test void invalidTiles() { - assertThrows(RuntimeException.class, () -> PMTilesUtils.tileIdToZxy(9007199254740991L)); - assertThrows(RuntimeException.class, () -> PMTilesUtils.zxyToTileId(27, 0, 0)); - assertThrows(RuntimeException.class, () -> PMTilesUtils.zxyToTileId(0, 1, 1)); + assertThrows(RuntimeException.class, () -> TileIdConverter.tileIdToZxy(9007199254740991L)); + assertThrows(RuntimeException.class, () -> TileIdConverter.zxyToTileId(27, 0, 0)); + assertThrows(RuntimeException.class, () -> TileIdConverter.zxyToTileId(0, 1, 1)); } @Test void decodeHeader() throws IOException { var file = TestFiles.resolve("baremaps-testing/data/pmtiles/test_fixture_1.pmtiles"); try (var channel = FileChannel.open(file)) { - var input = new LittleEndianDataInputStream(Channels.newInputStream(channel)); - var header = PMTilesUtils.deserializeHeader(input); + var input = Channels.newInputStream(channel); + var header = headerSerializer.deserialize(input); assertEquals(127, header.getRootDirectoryOffset()); assertEquals(25, header.getRootDirectoryLength()); assertEquals(152, header.getJsonMetadataOffset()); @@ -158,38 +163,34 @@ class PMTilesUtilsTest { @Test void encodeHeader() throws IOException { - var header = new Header( - 127, - 25, - 152, - 247, - 0, - 0, - 399, - 69, - 1, - 1, - 1, - 10, - false, - Compression.GZIP, - Compression.GZIP, - TileType.MVT, - 0, - 0, - 0, - 1, - 1, - 0, - 0, - 0, - 0); + var header = Header.builder() + .specVersion(127) + .rootDirectoryOffset(25) + .rootDirectoryLength(152) + .jsonMetadataOffset(247) + .jsonMetadataLength(0) + .leafDirectoryOffset(0) + .leafDirectoryLength(399) + .tileDataOffset(69) + .tileDataLength(1) + .numAddressedTiles(1) + .numTileEntries(1) + .numTileContents(10) + .clustered(false) + .internalCompression(Compression.GZIP) + .tileCompression(Compression.GZIP) + .tileType(TileType.MVT) + .minZoom(0) + .maxZoom(0) + .bounds(0, 1, 1, 0) + .center(0, 0, 0) + .build(); var array = new ByteArrayOutputStream(); - array.write(PMTilesUtils.serializeHeader(header)); + headerSerializer.serialize(header, array); - var input = new LittleEndianDataInputStream(new ByteArrayInputStream(array.toByteArray())); - var header2 = PMTilesUtils.deserializeHeader(input); + var input = new ByteArrayInputStream(array.toByteArray()); + var header2 = headerSerializer.deserialize(input); assertEquals(header, header2); } @@ -197,46 +198,46 @@ class PMTilesUtilsTest { @Test void searchForMissingEntry() { var entries = new ArrayList<Entry>(); - assertNull(PMTilesUtils.findTile(entries, 101)); + assertNull(entrySerializer.findTile(entries, 101)); } @Test void searchForFirstEntry() { - var entry = new Entry(100, 1, 1, 1); + var entry = Entry.builder().tileId(100).offset(1).length(1).runLength(1).build(); var entries = new ArrayList<Entry>(); entries.add(entry); - assertEquals(entry, PMTilesUtils.findTile(entries, 100)); + assertEquals(entry, entrySerializer.findTile(entries, 100)); } @Test void searchWithRunLength() { - var entry = new Entry(3, 3, 1, 2); + var entry = Entry.builder().tileId(3).offset(3).length(1).runLength(2).build(); var entries = new ArrayList<Entry>(); entries.add(entry); - entries.add(new Entry(5, 5, 1, 2)); - assertEquals(entry, PMTilesUtils.findTile(entries, 4)); + entries.add(Entry.builder().tileId(5).offset(5).length(1).runLength(2).build()); + assertEquals(entry, entrySerializer.findTile(entries, 4)); } @Test void searchWithMultipleTileEntries() { var entries = new ArrayList<Entry>(); - entries.add(new Entry(100, 1, 1, 2)); - var entry = PMTilesUtils.findTile(entries, 101); + entries.add(Entry.builder().tileId(100).offset(1).length(1).runLength(2).build()); + var entry = entrySerializer.findTile(entries, 101); assertEquals(1, entry.getOffset()); assertEquals(1, entry.getLength()); entries = new ArrayList<Entry>(); - entries.add(new Entry(100, 1, 1, 1)); - entries.add(new Entry(150, 2, 2, 2)); - entry = PMTilesUtils.findTile(entries, 151); + entries.add(Entry.builder().tileId(100).offset(1).length(1).runLength(1).build()); + entries.add(Entry.builder().tileId(150).offset(2).length(2).runLength(2).build()); + entry = entrySerializer.findTile(entries, 151); assertEquals(2, entry.getOffset()); assertEquals(2, entry.getLength()); entries = new ArrayList<>(); - entries.add(new Entry(50, 1, 1, 2)); - entries.add(new Entry(100, 2, 2, 1)); - entries.add(new Entry(150, 3, 3, 1)); - entry = PMTilesUtils.findTile(entries, 51); + entries.add(Entry.builder().tileId(50).offset(1).length(1).runLength(2).build()); + entries.add(Entry.builder().tileId(100).offset(2).length(2).runLength(1).build()); + entries.add(Entry.builder().tileId(150).offset(3).length(3).runLength(1).build()); + entry = entrySerializer.findTile(entries, 51); assertEquals(1, entry.getOffset()); assertEquals(1, entry.getLength()); } @@ -244,26 +245,25 @@ class PMTilesUtilsTest { @Test void leafSearch() { var entries = new ArrayList<Entry>(); - entries.add(new Entry(100, 1, 1, 0)); - var entry = PMTilesUtils.findTile(entries, 150); + entries.add(Entry.builder().tileId(100).offset(1).length(1).runLength(0).build()); + var entry = entrySerializer.findTile(entries, 150); assertEquals(1, entry.getOffset()); assertEquals(1, entry.getLength()); } @Test void buildRootLeaves() throws IOException { - var entries = List.of(new Entry(100, 1, 1, 0)); - var directories = PMTilesUtils.buildRootLeaves(entries, 1, Compression.NONE); + var entries = List.of(Entry.builder().tileId(100).offset(1).length(1).runLength(0).build()); + var directories = directorySerializer.buildRootLeaves(entries, 1, Compression.NONE); assertEquals(1, directories.getNumLeaves()); - } @Test void optimizeDirectories() throws IOException { var random = new Random(3857); var entries = new ArrayList<Entry>(); - entries.add(new Entry(0, 0, 100, 1)); - var directories = PMTilesUtils.optimizeDirectories(entries, 100, Compression.NONE); + entries.add(Entry.builder().tileId(0).offset(0).length(100).runLength(1).build()); + var directories = directorySerializer.optimizeDirectories(entries, 100, Compression.NONE); assertFalse(directories.getLeaves().length > 0); assertEquals(0, directories.getNumLeaves()); @@ -271,10 +271,11 @@ class PMTilesUtilsTest { int offset = 0; for (var i = 0; i < 1000; i++) { var randTileSize = random.nextInt(1000000); - entries.add(new Entry(i, offset, randTileSize, 1)); + entries + .add(Entry.builder().tileId(i).offset(offset).length(randTileSize).runLength(1).build()); offset += randTileSize; } - directories = PMTilesUtils.optimizeDirectories(entries, 1024, Compression.NONE); + directories = directorySerializer.optimizeDirectories(entries, 1024, Compression.NONE); assertFalse(directories.getRoot().length > 1024); assertNotEquals(0, directories.getNumLeaves()); assertNotEquals(0, directories.getLeaves().length); diff --git a/baremaps-pmtiles/src/test/java/org/apache/baremaps/pmtiles/TileIdConverterTest.java b/baremaps-pmtiles/src/test/java/org/apache/baremaps/pmtiles/TileIdConverterTest.java new file mode 100644 index 000000000..85eab7272 --- /dev/null +++ b/baremaps-pmtiles/src/test/java/org/apache/baremaps/pmtiles/TileIdConverterTest.java @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.baremaps.pmtiles; + +import static org.junit.jupiter.api.Assertions.*; + +import com.google.common.math.LongMath; +import org.junit.jupiter.api.Test; + +/** + * Tests for the TileIdConverter class. + */ +class TileIdConverterTest { + + @Test + void zxyToTileId() { + assertEquals(0, TileIdConverter.zxyToTileId(0, 0, 0)); + assertEquals(1, TileIdConverter.zxyToTileId(1, 0, 0)); + assertEquals(2, TileIdConverter.zxyToTileId(1, 0, 1)); + assertEquals(3, TileIdConverter.zxyToTileId(1, 1, 1)); + assertEquals(4, TileIdConverter.zxyToTileId(1, 1, 0)); + assertEquals(5, TileIdConverter.zxyToTileId(2, 0, 0)); + } + + @Test + void tileIdToZxy() { + assertArrayEquals(new long[] {0, 0, 0}, TileIdConverter.tileIdToZxy(0)); + assertArrayEquals(new long[] {1, 0, 0}, TileIdConverter.tileIdToZxy(1)); + assertArrayEquals(new long[] {1, 0, 1}, TileIdConverter.tileIdToZxy(2)); + assertArrayEquals(new long[] {1, 1, 1}, TileIdConverter.tileIdToZxy(3)); + assertArrayEquals(new long[] {1, 1, 0}, TileIdConverter.tileIdToZxy(4)); + assertArrayEquals(new long[] {2, 0, 0}, TileIdConverter.tileIdToZxy(5)); + } + + @Test + void aLotOfTiles() { + for (int z = 0; z < 9; z++) { + for (long x = 0; x < 1 << z; x++) { + for (long y = 0; y < 1 << z; y++) { + var result = TileIdConverter.tileIdToZxy(TileIdConverter.zxyToTileId(z, x, y)); + if (result[0] != z || result[1] != x || result[2] != y) { + fail("roundtrip failed"); + } + } + } + } + } + + @Test + void tileExtremes() { + for (var z = 0; z < 27; z++) { + var dim = LongMath.pow(2, z) - 1; + var tl = TileIdConverter.tileIdToZxy(TileIdConverter.zxyToTileId(z, 0, 0)); + assertArrayEquals(new long[] {z, 0, 0}, tl); + var tr = TileIdConverter.tileIdToZxy(TileIdConverter.zxyToTileId(z, dim, 0)); + assertArrayEquals(new long[] {z, dim, 0}, tr); + var bl = TileIdConverter.tileIdToZxy(TileIdConverter.zxyToTileId(z, 0, dim)); + assertArrayEquals(new long[] {z, 0, dim}, bl); + var br = TileIdConverter.tileIdToZxy(TileIdConverter.zxyToTileId(z, dim, dim)); + assertArrayEquals(new long[] {z, dim, dim}, br); + } + } + + @Test + void invalidTiles() { + assertThrows(RuntimeException.class, () -> TileIdConverter.tileIdToZxy(9007199254740991L)); + assertThrows(RuntimeException.class, () -> TileIdConverter.zxyToTileId(27, 0, 0)); + assertThrows(RuntimeException.class, () -> TileIdConverter.zxyToTileId(0, 1, 1)); + } +} diff --git a/baremaps-pmtiles/src/test/java/org/apache/baremaps/pmtiles/VarIntSerializerTest.java b/baremaps-pmtiles/src/test/java/org/apache/baremaps/pmtiles/VarIntSerializerTest.java new file mode 100644 index 000000000..af9376f00 --- /dev/null +++ b/baremaps-pmtiles/src/test/java/org/apache/baremaps/pmtiles/VarIntSerializerTest.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.baremaps.pmtiles; + +import static org.junit.jupiter.api.Assertions.*; + +import com.google.common.io.LittleEndianDataInputStream; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import org.junit.jupiter.api.Test; + +/** + * Tests for the VarIntSerializer class. + */ +class VarIntSerializerTest { + + private final VarIntSerializer varIntSerializer = new VarIntSerializer(); + + @Test + void decodeVarInt() throws IOException { + var b = new LittleEndianDataInputStream(new ByteArrayInputStream(new byte[] { + (byte) 0, (byte) 1, + (byte) 127, (byte) 0xe5, + (byte) 0x8e, (byte) 0x26 + })); + assertEquals(0, varIntSerializer.readVarInt(b)); + assertEquals(1, varIntSerializer.readVarInt(b)); + assertEquals(127, varIntSerializer.readVarInt(b)); + assertEquals(624485, varIntSerializer.readVarInt(b)); + b = new LittleEndianDataInputStream(new ByteArrayInputStream(new byte[] { + (byte) 0xff, (byte) 0xff, + (byte) 0xff, (byte) 0xff, + (byte) 0xff, (byte) 0xff, + (byte) 0xff, (byte) 0x0f, + })); + assertEquals(9007199254740991L, varIntSerializer.readVarInt(b)); + } + + @Test + void encodeVarInt() throws IOException { + for (long i = 0; i < 1000; i++) { + var array = new ByteArrayOutputStream(); + varIntSerializer.writeVarInt(array, i); + var input = new LittleEndianDataInputStream(new ByteArrayInputStream(array.toByteArray())); + assertEquals(i, varIntSerializer.readVarInt(input)); + } + for (long i = Long.MAX_VALUE - 1000; i < Long.MAX_VALUE; i++) { + var array = new ByteArrayOutputStream(); + varIntSerializer.writeVarInt(array, i); + var input = new LittleEndianDataInputStream(new ByteArrayInputStream(array.toByteArray())); + assertEquals(i, varIntSerializer.readVarInt(input)); + } + } +}
