This is an automated email from the ASF dual-hosted git repository. bchapuis pushed a commit to branch pmtiles in repository https://gitbox.apache.org/repos/asf/incubator-baremaps.git
commit ade0fab5feb687e12266c8f295439f6e651a534c Author: Bertil Chapuis <[email protected]> AuthorDate: Tue Sep 5 23:24:54 2023 +0200 Implement header serialization and deserialization --- .../apache/baremaps/tilestore/pmtiles/PMTiles.java | 116 +++++++++++++++++++++ .../baremaps/tilestore/pmtiles/PMTilesTest.java | 73 +++++++++++++ .../src/test/resources/pmtiles/empty.pmtiles | 0 .../src/test/resources/pmtiles/invalid.pmtiles | 1 + .../src/test/resources/pmtiles/invalid_v4.pmtiles | Bin 0 -> 468 bytes .../test/resources/pmtiles/test_fixture_1.pmtiles | Bin 0 -> 468 bytes .../test/resources/pmtiles/test_fixture_2.pmtiles | Bin 0 -> 466 bytes 7 files changed, 190 insertions(+) diff --git a/baremaps-core/src/main/java/org/apache/baremaps/tilestore/pmtiles/PMTiles.java b/baremaps-core/src/main/java/org/apache/baremaps/tilestore/pmtiles/PMTiles.java index c84b19b2..f773c7eb 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/tilestore/pmtiles/PMTiles.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/tilestore/pmtiles/PMTiles.java @@ -3,6 +3,7 @@ package org.apache.baremaps.tilestore.pmtiles; import com.google.common.math.LongMath; import java.nio.ByteBuffer; +import java.nio.ByteOrder; public class PMTiles { @@ -142,4 +143,119 @@ public class PMTiles { } throw new RuntimeException("Tile zoom level exceeds max safe number limit (26)"); } + + enum Compression { + Unknown, + None, + Gzip, + Brotli, + Zstd, + } + + enum TileType { + Unknown, + Mvt, + Png, + Jpeg, + Webp, + Avif, + } + + public record Header( + int specVersion, + long rootDirectoryOffset, + long rootDirectoryLength, + long jsonMetadataOffset, + long jsonMetadataLength, + long leafDirectoryOffset, + long leafDirectoryLength, + long tileDataOffset, + long tileDataLength, + long numAddressedTiles, + long numTileEntries, + long numTileContents, + boolean clustered, + Compression internalCompression, + Compression tileCompression, + TileType tileType, + int minZoom, + int maxZoom, + double minLon, + double minLat, + double maxLon, + double maxLat, + int centerZoom, + double centerLon, + double centerLat, + String etag + ) {} + + public static Header bytesToHeader(ByteBuffer buf, String etag) { + buf.order(ByteOrder.LITTLE_ENDIAN); + return new Header( + buf.get(7), + buf.getLong(8), + buf.getLong(16), + buf.getLong(24), + buf.getLong(32), + buf.getLong(40), + buf.getLong(48), + buf.getLong(56), + buf.getLong(64), + buf.getLong(72), + buf.getLong(80), + buf.getLong(88), + buf.get(96) == 1, + Compression.values()[buf.get(97)], + Compression.values()[buf.get(98)], + TileType.values()[buf.get(99)], + buf.get(100), + buf.get(101), + (double) buf.getInt(102) / 10000000, + (double) buf.getInt(106) / 10000000, + (double) buf.getInt(110) / 10000000, + (double) buf.getInt(114) / 10000000, + buf.get(118), + (double) buf.getInt(119) / 10000000, + (double) buf.getInt(123) / 10000000, + etag + ); + } + + public static void headerToBytes(Header header, ByteBuffer buf) { + buf.order(ByteOrder.LITTLE_ENDIAN); + buf.put(0, (byte) 0x4d); + buf.put(1, (byte) 0x42); + buf.put(2, (byte) 0x54); + buf.put(3, (byte) 0x42); + buf.put(4, (byte) 0x49); + buf.put(5, (byte) 0x4e); + buf.put(6, (byte) 0x41); + buf.put(7, (byte) header.specVersion); + buf.putLong(8, header.rootDirectoryOffset); + buf.putLong(16, header.rootDirectoryLength); + buf.putLong(24, header.jsonMetadataOffset); + buf.putLong(32, header.jsonMetadataLength); + buf.putLong(40, header.leafDirectoryOffset); + buf.putLong(48, header.leafDirectoryLength); + buf.putLong(56, header.tileDataOffset); + buf.putLong(64, header.tileDataLength); + buf.putLong(72, header.numAddressedTiles); + buf.putLong(80, header.numTileEntries); + buf.putLong(88, header.numTileContents); + buf.put(96, (byte) (header.clustered ? 1 : 0)); + buf.put(97, (byte) header.internalCompression.ordinal()); + buf.put(98, (byte) header.tileCompression.ordinal()); + buf.put(99, (byte) header.tileType.ordinal()); + buf.put(100, (byte) header.minZoom); + buf.put(101, (byte) header.maxZoom); + buf.putInt(102, (int) (header.minLon * 10000000)); + buf.putInt(106, (int) (header.minLat * 10000000)); + buf.putInt(110, (int) (header.maxLon * 10000000)); + buf.putInt(114, (int) (header.maxLat * 10000000)); + buf.put(118, (byte) header.centerZoom); + buf.putInt(119, (int) (header.centerLon * 10000000)); + buf.putInt(123, (int) (header.centerLat * 10000000)); + } + } diff --git a/baremaps-core/src/test/java/org/apache/baremaps/tilestore/pmtiles/PMTilesTest.java b/baremaps-core/src/test/java/org/apache/baremaps/tilestore/pmtiles/PMTilesTest.java index c119a882..3eda409b 100644 --- a/baremaps-core/src/test/java/org/apache/baremaps/tilestore/pmtiles/PMTilesTest.java +++ b/baremaps-core/src/test/java/org/apache/baremaps/tilestore/pmtiles/PMTilesTest.java @@ -1,9 +1,14 @@ package org.apache.baremaps.tilestore.pmtiles; import com.google.common.math.LongMath; +import org.apache.baremaps.testing.TestFiles; +import org.apache.baremaps.tilestore.pmtiles.PMTiles.Compression; +import org.apache.baremaps.tilestore.pmtiles.PMTiles.TileType; import org.junit.jupiter.api.Test; +import java.io.IOException; import java.nio.ByteBuffer; +import java.nio.file.Files; import static org.junit.jupiter.api.Assertions.*; @@ -85,4 +90,72 @@ class PMTilesTest { assertThrows(RuntimeException.class, () -> PMTiles.zxyToTileId(0, 1, 1)); } + @Test + void bytesToHeader() throws IOException { + var file = TestFiles.resolve("pmtiles/test_fixture_1.pmtiles"); + try (var channel = Files.newByteChannel(file)) { + var buffer = ByteBuffer.allocate(127); + channel.read(buffer); + buffer.flip(); + var header = PMTiles.bytesToHeader(buffer, "1"); + assertEquals(header.rootDirectoryOffset(), 127); + assertEquals(header.rootDirectoryLength(), 25); + assertEquals(header.jsonMetadataOffset(), 152); + assertEquals(header.jsonMetadataLength(), 247); + assertEquals(header.leafDirectoryOffset(), 0); + assertEquals(header.leafDirectoryLength(), 0); + assertEquals(header.tileDataOffset(), 399); + assertEquals(header.tileDataLength(), 69); + assertEquals(header.numAddressedTiles(), 1); + assertEquals(header.numTileEntries(), 1); + assertEquals(header.numTileContents(), 1); + assertFalse(header.clustered()); + assertEquals(header.internalCompression(), Compression.Gzip); + assertEquals(header.tileCompression(), Compression.Gzip); + assertEquals(header.tileType(), TileType.Mvt); + assertEquals(header.minZoom(), 0); + assertEquals(header.maxZoom(), 0); + assertEquals(header.minLon(), 0); + assertEquals(header.minLat(), 0); + assertEquals(Math.round(header.maxLon()), 1); + assertEquals(Math.round(header.maxLat()), 1); + } + } + + @Test + void headerToBytes() throws IOException { + var etag = "1"; + var buffer = ByteBuffer.allocate(127); + var header = new PMTiles.Header( + 127, + 25, + 152, + 247, + 0, + 0, + 399, + 69, + 1, + 1, + 1, + 10, + false, + Compression.Gzip, + Compression.Gzip, + TileType.Mvt, + 0, + 0, + 0, + 1, + 1, + 0, + 0, + 0, + 0, + etag); + PMTiles.headerToBytes(header, buffer); + var header2 = PMTiles.bytesToHeader(buffer, etag); + assertEquals(header, header2); + } + } \ No newline at end of file diff --git a/baremaps-core/src/test/resources/pmtiles/empty.pmtiles b/baremaps-core/src/test/resources/pmtiles/empty.pmtiles new file mode 100644 index 00000000..e69de29b diff --git a/baremaps-core/src/test/resources/pmtiles/invalid.pmtiles b/baremaps-core/src/test/resources/pmtiles/invalid.pmtiles new file mode 100644 index 00000000..f2b720ba --- /dev/null +++ b/baremaps-core/src/test/resources/pmtiles/invalid.pmtiles @@ -0,0 +1 @@ +This is an invalid tile archive, a test case to make sure that the code throws an error, but it needs to be the minimum size to pass the first test diff --git a/baremaps-core/src/test/resources/pmtiles/invalid_v4.pmtiles b/baremaps-core/src/test/resources/pmtiles/invalid_v4.pmtiles new file mode 100644 index 00000000..1871cb27 Binary files /dev/null and b/baremaps-core/src/test/resources/pmtiles/invalid_v4.pmtiles differ diff --git a/baremaps-core/src/test/resources/pmtiles/test_fixture_1.pmtiles b/baremaps-core/src/test/resources/pmtiles/test_fixture_1.pmtiles new file mode 100644 index 00000000..c86db1f2 Binary files /dev/null and b/baremaps-core/src/test/resources/pmtiles/test_fixture_1.pmtiles differ diff --git a/baremaps-core/src/test/resources/pmtiles/test_fixture_2.pmtiles b/baremaps-core/src/test/resources/pmtiles/test_fixture_2.pmtiles new file mode 100644 index 00000000..cb19dd5f Binary files /dev/null and b/baremaps-core/src/test/resources/pmtiles/test_fixture_2.pmtiles differ
