This is an automated email from the ASF dual-hosted git repository.
bchapuis pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-baremaps.git
The following commit(s) were added to refs/heads/main by this push:
new 1764a6a4 Add support for PMTiles (#794)
1764a6a4 is described below
commit 1764a6a4a69a0da60969765c221d53626cbb666e
Author: Bertil Chapuis <[email protected]>
AuthorDate: Sun Nov 26 13:51:28 2023 +0100
Add support for PMTiles (#794)
Enable the export of vector tiles in the PMTiles format. Most of the
necessary functions have been ported from
[go-pmtiles](https://github.com/protomaps/go-pmtiles) to Java.
---
.run/basemap-dev.run.xml | 7 +-
...-dev.run.xml => basemap-export-mbtiles.run.xml} | 4 +-
...-dev.run.xml => basemap-export-pmtiles.run.xml} | 4 +-
.run/basemap-workflow.run.xml | 2 +-
.../org/apache/baremaps/tilestore/TileCache.java | 6 +
.../org/apache/baremaps/tilestore/TileStore.java | 2 +-
.../baremaps/tilestore/TileStoreException.java | 10 +
.../baremaps/tilestore/file/FileTileStore.java | 5 +
.../baremaps/tilestore/mbtiles/MBTilesStore.java | 5 +
.../baremaps/tilestore/pmtiles/Compression.java | 74 ++++
.../Directories.java} | 37 +-
.../apache/baremaps/tilestore/pmtiles/Entry.java | 66 ++++
.../apache/baremaps/tilestore/pmtiles/Header.java | 341 ++++++++++++++++++
.../apache/baremaps/tilestore/pmtiles/PMTiles.java | 391 +++++++++++++++++++++
.../baremaps/tilestore/pmtiles/PMTilesReader.java | 95 +++++
.../baremaps/tilestore/pmtiles/PMTilesStore.java | 95 +++++
.../baremaps/tilestore/pmtiles/PMTilesWriter.java | 208 +++++++++++
.../TileType.java} | 29 +-
.../tilestore/postgres/PostgresTileStore.java | 20 +-
.../baremaps/workflow/tasks/ExportVectorTiles.java | 79 +++--
.../baremaps/tilestore/pmtiles/PMTilesTest.java | 288 +++++++++++++++
.../tilestore/postgres/PostgresTileStoreTest.java | 2 +-
.../baremaps/vectortile/VectorTileViewer.java | 2 +-
.../src/test/resources/pmtiles/empty.pmtiles | 0
.../src/test/resources/pmtiles/invalid.pmtiles | 1 +
.../src/test/resources/pmtiles/invalid_v4.pmtiles | Bin 0 -> 468 bytes
.../test/resources/pmtiles/test_fixture_1.pmtiles | Bin 0 -> 468 bytes
.../test/resources/pmtiles/test_fixture_2.pmtiles | Bin 0 -> 466 bytes
basemap/import.js | 16 +
basemap/layers/building/extrusion.js | 9 -
basemap/layers/building/tileset.js | 68 ++--
basemap/queries/assertions.sql | 48 +++
basemap/queries/functions.sql | 49 +++
33 files changed, 1836 insertions(+), 127 deletions(-)
diff --git a/.run/basemap-dev.run.xml b/.run/basemap-dev.run.xml
index 4b61bfcd..af9438a4 100644
--- a/.run/basemap-dev.run.xml
+++ b/.run/basemap-dev.run.xml
@@ -2,8 +2,13 @@
<configuration default="false" name="basemap-dev" type="Application"
factoryName="Application">
<option name="MAIN_CLASS_NAME" value="org.apache.baremaps.cli.Baremaps" />
<module name="baremaps-cli" />
- <option name="PROGRAM_PARAMETERS" value="map dev --tileset tileset.js
--style style.js" />
+ <option name="PROGRAM_PARAMETERS" value="map dev --tileset tileset.js
--style style.js --log-level DEBUG" />
<option name="WORKING_DIRECTORY" value="$PROJECT_DIR$/basemap" />
+ <extension
name="software.aws.toolkits.jetbrains.core.execution.JavaAwsConnectionExtension">
+ <option name="credential" />
+ <option name="region" />
+ <option name="useCurrentConnection" value="false" />
+ </extension>
<method v="2">
<option name="Make" enabled="true" />
</method>
diff --git a/.run/basemap-dev.run.xml b/.run/basemap-export-mbtiles.run.xml
similarity index 57%
copy from .run/basemap-dev.run.xml
copy to .run/basemap-export-mbtiles.run.xml
index 4b61bfcd..d04d8a2a 100644
--- a/.run/basemap-dev.run.xml
+++ b/.run/basemap-export-mbtiles.run.xml
@@ -1,8 +1,8 @@
<component name="ProjectRunConfigurationManager">
- <configuration default="false" name="basemap-dev" type="Application"
factoryName="Application">
+ <configuration default="false" name="basemap-export-mbtiles"
type="Application" factoryName="Application">
<option name="MAIN_CLASS_NAME" value="org.apache.baremaps.cli.Baremaps" />
<module name="baremaps-cli" />
- <option name="PROGRAM_PARAMETERS" value="map dev --tileset tileset.js
--style style.js" />
+ <option name="PROGRAM_PARAMETERS" value="map export --tileset tileset.js
--repository tiles.mbtiles --format mbtiles" />
<option name="WORKING_DIRECTORY" value="$PROJECT_DIR$/basemap" />
<method v="2">
<option name="Make" enabled="true" />
diff --git a/.run/basemap-dev.run.xml b/.run/basemap-export-pmtiles.run.xml
similarity index 57%
copy from .run/basemap-dev.run.xml
copy to .run/basemap-export-pmtiles.run.xml
index 4b61bfcd..840a4774 100644
--- a/.run/basemap-dev.run.xml
+++ b/.run/basemap-export-pmtiles.run.xml
@@ -1,8 +1,8 @@
<component name="ProjectRunConfigurationManager">
- <configuration default="false" name="basemap-dev" type="Application"
factoryName="Application">
+ <configuration default="false" name="basemap-export-pmtiles"
type="Application" factoryName="Application">
<option name="MAIN_CLASS_NAME" value="org.apache.baremaps.cli.Baremaps" />
<module name="baremaps-cli" />
- <option name="PROGRAM_PARAMETERS" value="map dev --tileset tileset.js
--style style.js" />
+ <option name="PROGRAM_PARAMETERS" value="map export --tileset tileset.js
--repository tiles.mbtiles --format pmtiles" />
<option name="WORKING_DIRECTORY" value="$PROJECT_DIR$/basemap" />
<method v="2">
<option name="Make" enabled="true" />
diff --git a/.run/basemap-workflow.run.xml b/.run/basemap-workflow.run.xml
index f0fde5c2..663c874a 100644
--- a/.run/basemap-workflow.run.xml
+++ b/.run/basemap-workflow.run.xml
@@ -2,7 +2,7 @@
<configuration default="false" name="basemap-workflow" type="Application"
factoryName="Application">
<option name="MAIN_CLASS_NAME" value="org.apache.baremaps.cli.Baremaps" />
<module name="baremaps-cli" />
- <option name="PROGRAM_PARAMETERS" value="workflow execute --file
workflow.js" />
+ <option name="PROGRAM_PARAMETERS" value="workflow execute --file
import.js" />
<option name="WORKING_DIRECTORY" value="$PROJECT_DIR$/basemap" />
<method v="2">
<option name="Make" enabled="true" />
diff --git
a/baremaps-core/src/main/java/org/apache/baremaps/tilestore/TileCache.java
b/baremaps-core/src/main/java/org/apache/baremaps/tilestore/TileCache.java
index 7381b6e9..199734a7 100644
--- a/baremaps-core/src/main/java/org/apache/baremaps/tilestore/TileCache.java
+++ b/baremaps-core/src/main/java/org/apache/baremaps/tilestore/TileCache.java
@@ -84,4 +84,10 @@ public class TileCache implements TileStore {
tileStore.delete(tileCoord);
cache.invalidate(tileCoord);
}
+
+ @Override
+ public void close() throws Exception {
+ tileStore.close();
+ cache.cleanUp();
+ }
}
diff --git
a/baremaps-core/src/main/java/org/apache/baremaps/tilestore/TileStore.java
b/baremaps-core/src/main/java/org/apache/baremaps/tilestore/TileStore.java
index c0332150..e7f34b11 100644
--- a/baremaps-core/src/main/java/org/apache/baremaps/tilestore/TileStore.java
+++ b/baremaps-core/src/main/java/org/apache/baremaps/tilestore/TileStore.java
@@ -24,7 +24,7 @@ import java.util.ArrayList;
import java.util.List;
/** Represents a store for tiles. */
-public interface TileStore {
+public interface TileStore extends AutoCloseable {
/**
* Reads the content of a tile.
diff --git
a/baremaps-core/src/main/java/org/apache/baremaps/tilestore/TileStoreException.java
b/baremaps-core/src/main/java/org/apache/baremaps/tilestore/TileStoreException.java
index d67bc50c..1ad72e1e 100644
---
a/baremaps-core/src/main/java/org/apache/baremaps/tilestore/TileStoreException.java
+++
b/baremaps-core/src/main/java/org/apache/baremaps/tilestore/TileStoreException.java
@@ -37,4 +37,14 @@ public class TileStoreException extends Exception {
public TileStoreException(Throwable cause) {
super(cause);
}
+
+ /**
+ * Constructs a {@code BlobStoreException} with the specified detail message
and cause.
+ *
+ * @param message the message
+ * @param cause the cause
+ */
+ public TileStoreException(String message, Throwable cause) {
+ super(message, cause);
+ }
}
diff --git
a/baremaps-core/src/main/java/org/apache/baremaps/tilestore/file/FileTileStore.java
b/baremaps-core/src/main/java/org/apache/baremaps/tilestore/file/FileTileStore.java
index 41dd412b..5afcdafa 100644
---
a/baremaps-core/src/main/java/org/apache/baremaps/tilestore/file/FileTileStore.java
+++
b/baremaps-core/src/main/java/org/apache/baremaps/tilestore/file/FileTileStore.java
@@ -77,4 +77,9 @@ public class FileTileStore implements TileStore {
public Path resolve(TileCoord tileCoord) {
return path.resolve(String.format("%s/%s/%s.mvt", tileCoord.z(),
tileCoord.x(), tileCoord.y()));
}
+
+ @Override
+ public void close() {
+ // Do nothing
+ }
}
diff --git
a/baremaps-core/src/main/java/org/apache/baremaps/tilestore/mbtiles/MBTilesStore.java
b/baremaps-core/src/main/java/org/apache/baremaps/tilestore/mbtiles/MBTilesStore.java
index b6560778..9787cbeb 100644
---
a/baremaps-core/src/main/java/org/apache/baremaps/tilestore/mbtiles/MBTilesStore.java
+++
b/baremaps-core/src/main/java/org/apache/baremaps/tilestore/mbtiles/MBTilesStore.java
@@ -219,4 +219,9 @@ public class MBTilesStore implements TileStore {
private static int reverseY(int y, int z) {
return (int) (Math.pow(2.0, z) - 1 - y);
}
+
+ @Override
+ public void close() throws Exception {
+ // Do nothing
+ }
}
diff --git
a/baremaps-core/src/main/java/org/apache/baremaps/tilestore/pmtiles/Compression.java
b/baremaps-core/src/main/java/org/apache/baremaps/tilestore/pmtiles/Compression.java
new file mode 100644
index 00000000..30da86fa
--- /dev/null
+++
b/baremaps-core/src/main/java/org/apache/baremaps/tilestore/pmtiles/Compression.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.baremaps.tilestore.pmtiles;
+
+import java.io.*;
+import java.util.zip.GZIPInputStream;
+import java.util.zip.GZIPOutputStream;
+
+enum Compression {
+ Unknown,
+ None,
+ Gzip,
+ Brotli,
+ Zstd;
+
+ InputStream decompress(InputStream inputStream) throws IOException {
+ return switch (this) {
+ case None -> inputStream;
+ case Gzip -> decompressGzip(inputStream);
+ case Brotli -> decompressBrotli(inputStream);
+ case Zstd -> decompressZstd(inputStream);
+ default -> throw new RuntimeException("Unknown compression");
+ };
+ }
+
+ static InputStream decompressGzip(InputStream inputStream) throws
IOException {
+ return new GZIPInputStream(inputStream);
+ }
+
+ static InputStream decompressBrotli(InputStream buffer) {
+ throw new RuntimeException("Brotli compression not implemented");
+ }
+
+ static InputStream decompressZstd(InputStream buffer) {
+ throw new RuntimeException("Zstd compression not implemented");
+ }
+
+ OutputStream compress(OutputStream outputStream) throws IOException {
+ return switch (this) {
+ case None -> outputStream;
+ case Gzip -> compressGzip(outputStream);
+ case Brotli -> compressBrotli(outputStream);
+ case Zstd -> compressZstd(outputStream);
+ default -> throw new RuntimeException("Unknown compression");
+ };
+ }
+
+ static OutputStream compressGzip(OutputStream outputStream) throws
IOException {
+ return new GZIPOutputStream(outputStream);
+ }
+
+ static OutputStream compressBrotli(OutputStream outputStream) {
+ throw new RuntimeException("Brotli compression not implemented");
+ }
+
+ static OutputStream compressZstd(OutputStream outputStream) {
+ throw new RuntimeException("Zstd compression not implemented");
+ }
+}
diff --git
a/baremaps-core/src/main/java/org/apache/baremaps/tilestore/TileStoreException.java
b/baremaps-core/src/main/java/org/apache/baremaps/tilestore/pmtiles/Directories.java
similarity index 60%
copy from
baremaps-core/src/main/java/org/apache/baremaps/tilestore/TileStoreException.java
copy to
baremaps-core/src/main/java/org/apache/baremaps/tilestore/pmtiles/Directories.java
index d67bc50c..ea91b1fa 100644
---
a/baremaps-core/src/main/java/org/apache/baremaps/tilestore/TileStoreException.java
+++
b/baremaps-core/src/main/java/org/apache/baremaps/tilestore/pmtiles/Directories.java
@@ -15,26 +15,29 @@
* limitations under the License.
*/
-package org.apache.baremaps.tilestore;
+package org.apache.baremaps.tilestore.pmtiles;
-/** Signals that an exception occurred in a {@code TileStore}. */
-public class TileStoreException extends Exception {
+class Directories {
- /**
- * Constructs an {@code BlobStoreException} with the specified detail
message.
- *
- * @param message the message
- */
- public TileStoreException(String message) {
- super(message);
+ private final byte[] root;
+ private final byte[] leaves;
+ private final int numLeaves;
+
+ public Directories(byte[] root, byte[] leaves, int numLeaves) {
+ this.root = root;
+ this.leaves = leaves;
+ this.numLeaves = numLeaves;
+ }
+
+ public byte[] getRoot() {
+ return root;
+ }
+
+ public byte[] getLeaves() {
+ return leaves;
}
- /**
- * Constructs a {@code BlobStoreException} with the specified cause.
- *
- * @param cause the cause
- */
- public TileStoreException(Throwable cause) {
- super(cause);
+ public int getNumLeaves() {
+ return numLeaves;
}
}
diff --git
a/baremaps-core/src/main/java/org/apache/baremaps/tilestore/pmtiles/Entry.java
b/baremaps-core/src/main/java/org/apache/baremaps/tilestore/pmtiles/Entry.java
new file mode 100644
index 00000000..a8b0947e
--- /dev/null
+++
b/baremaps-core/src/main/java/org/apache/baremaps/tilestore/pmtiles/Entry.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.baremaps.tilestore.pmtiles;
+
+class Entry {
+ private long tileId;
+ private long offset;
+ private long length;
+ private long runLength;
+
+ public Entry() {}
+
+ public Entry(long tileId, long offset, long length, long runLength) {
+ this.tileId = tileId;
+ this.offset = offset;
+ this.length = length;
+ this.runLength = runLength;
+ }
+
+ public long getTileId() {
+ return tileId;
+ }
+
+ public void setTileId(long tileId) {
+ this.tileId = tileId;
+ }
+
+ public long getOffset() {
+ return offset;
+ }
+
+ public void setOffset(long offset) {
+ this.offset = offset;
+ }
+
+ public long getLength() {
+ return length;
+ }
+
+ public void setLength(long length) {
+ this.length = length;
+ }
+
+ public long getRunLength() {
+ return runLength;
+ }
+
+ public void setRunLength(long runLength) {
+ this.runLength = runLength;
+ }
+}
diff --git
a/baremaps-core/src/main/java/org/apache/baremaps/tilestore/pmtiles/Header.java
b/baremaps-core/src/main/java/org/apache/baremaps/tilestore/pmtiles/Header.java
new file mode 100644
index 00000000..b1e3bddf
--- /dev/null
+++
b/baremaps-core/src/main/java/org/apache/baremaps/tilestore/pmtiles/Header.java
@@ -0,0 +1,341 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.baremaps.tilestore.pmtiles;
+
+import java.util.Objects;
+
+class Header {
+
+ private int specVersion;
+ private long rootDirectoryOffset;
+ private long rootDirectoryLength;
+ private long jsonMetadataOffset;
+ private long jsonMetadataLength;
+ private long leafDirectoryOffset;
+ private long leafDirectoryLength;
+ private long tileDataOffset;
+ private long tileDataLength;
+ private long numAddressedTiles;
+ private long numTileEntries;
+ private long numTileContents;
+ private boolean clustered;
+ private Compression internalCompression;
+ private Compression tileCompression;
+ private TileType tileType;
+ private int minZoom;
+ private int maxZoom;
+ private double minLon;
+ private double minLat;
+ private double maxLon;
+ private double maxLat;
+ private int centerZoom;
+ private double centerLon;
+ private double centerLat;
+
+ public Header() {
+ this.specVersion = 3;
+ }
+
+ public Header(
+ int specVersion,
+ long rootDirectoryOffset,
+ long rootDirectoryLength,
+ long jsonMetadataOffset,
+ long jsonMetadataLength,
+ long leafDirectoryOffset,
+ long leafDirectoryLength,
+ long tileDataOffset,
+ long tileDataLength,
+ long numAddressedTiles,
+ long numTileEntries,
+ long numTileContents,
+ boolean clustered,
+ Compression internalCompression,
+ Compression tileCompression,
+ TileType tileType,
+ int minZoom,
+ int maxZoom,
+ double minLon,
+ double minLat, double maxLon,
+ double maxLat,
+ int centerZoom,
+ double centerLon,
+ double centerLat) {
+ this.specVersion = specVersion;
+ this.rootDirectoryOffset = rootDirectoryOffset;
+ this.rootDirectoryLength = rootDirectoryLength;
+ this.jsonMetadataOffset = jsonMetadataOffset;
+ this.jsonMetadataLength = jsonMetadataLength;
+ this.leafDirectoryOffset = leafDirectoryOffset;
+ this.leafDirectoryLength = leafDirectoryLength;
+ this.tileDataOffset = tileDataOffset;
+ this.tileDataLength = tileDataLength;
+ this.numAddressedTiles = numAddressedTiles;
+ this.numTileEntries = numTileEntries;
+ this.numTileContents = numTileContents;
+ this.clustered = clustered;
+ this.internalCompression = internalCompression;
+ this.tileCompression = tileCompression;
+ this.tileType = tileType;
+ this.minZoom = minZoom;
+ this.maxZoom = maxZoom;
+ this.minLon = minLon;
+ this.minLat = minLat;
+ this.maxLon = maxLon;
+ this.maxLat = maxLat;
+ this.centerZoom = centerZoom;
+ this.centerLon = centerLon;
+ this.centerLat = centerLat;
+ }
+
+ public int getSpecVersion() {
+ return specVersion;
+ }
+
+ public void setSpecVersion(int specVersion) {
+ this.specVersion = specVersion;
+ }
+
+ public long getRootDirectoryOffset() {
+ return rootDirectoryOffset;
+ }
+
+ public void setRootOffset(long rootDirectoryOffset) {
+ this.rootDirectoryOffset = rootDirectoryOffset;
+ }
+
+ public long getRootDirectoryLength() {
+ return rootDirectoryLength;
+ }
+
+ public void setRootLength(long rootDirectoryLength) {
+ this.rootDirectoryLength = rootDirectoryLength;
+ }
+
+ public long getJsonMetadataOffset() {
+ return jsonMetadataOffset;
+ }
+
+ public void setMetadataOffset(long jsonMetadataOffset) {
+ this.jsonMetadataOffset = jsonMetadataOffset;
+ }
+
+ public long getJsonMetadataLength() {
+ return jsonMetadataLength;
+ }
+
+ public void setMetadataLength(long jsonMetadataLength) {
+ this.jsonMetadataLength = jsonMetadataLength;
+ }
+
+ public long getLeafDirectoryOffset() {
+ return leafDirectoryOffset;
+ }
+
+ public void setLeavesOffset(long leafDirectoryOffset) {
+ this.leafDirectoryOffset = leafDirectoryOffset;
+ }
+
+ public long getLeafDirectoryLength() {
+ return leafDirectoryLength;
+ }
+
+ public void setLeavesLength(long leafDirectoryLength) {
+ this.leafDirectoryLength = leafDirectoryLength;
+ }
+
+ public long getTileDataOffset() {
+ return tileDataOffset;
+ }
+
+ public void setTilesOffset(long tileDataOffset) {
+ this.tileDataOffset = tileDataOffset;
+ }
+
+ public long getTileDataLength() {
+ return tileDataLength;
+ }
+
+ public void setTilesLength(long tileDataLength) {
+ this.tileDataLength = tileDataLength;
+ }
+
+ public long getNumAddressedTiles() {
+ return numAddressedTiles;
+ }
+
+ public void setNumAddressedTiles(long numAddressedTiles) {
+ this.numAddressedTiles = numAddressedTiles;
+ }
+
+ public long getNumTileEntries() {
+ return numTileEntries;
+ }
+
+ public void setNumTileEntries(long numTileEntries) {
+ this.numTileEntries = numTileEntries;
+ }
+
+ public long getNumTileContents() {
+ return numTileContents;
+ }
+
+ public void setNumTileContents(long numTileContents) {
+ this.numTileContents = numTileContents;
+ }
+
+ public boolean isClustered() {
+ return clustered;
+ }
+
+ public void setClustered(boolean clustered) {
+ this.clustered = clustered;
+ }
+
+ public Compression getInternalCompression() {
+ return internalCompression;
+ }
+
+ public void setInternalCompression(Compression internalCompression) {
+ this.internalCompression = internalCompression;
+ }
+
+ public Compression getTileCompression() {
+ return tileCompression;
+ }
+
+ public void setTileCompression(Compression tileCompression) {
+ this.tileCompression = tileCompression;
+ }
+
+ public TileType getTileType() {
+ return tileType;
+ }
+
+ public void setTileType(TileType tileType) {
+ this.tileType = tileType;
+ }
+
+ public int getMinZoom() {
+ return minZoom;
+ }
+
+ public void setMinZoom(int minZoom) {
+ this.minZoom = minZoom;
+ }
+
+ public int getMaxZoom() {
+ return maxZoom;
+ }
+
+ public void setMaxZoom(int maxZoom) {
+ this.maxZoom = maxZoom;
+ }
+
+ public double getMinLon() {
+ return minLon;
+ }
+
+ public void setMinLon(double minLon) {
+ this.minLon = minLon;
+ }
+
+ public double getMinLat() {
+ return minLat;
+ }
+
+ public void setMinLat(double minLat) {
+ this.minLat = minLat;
+ }
+
+ public double getMaxLon() {
+ return maxLon;
+ }
+
+ public void setMaxLon(double maxLon) {
+ this.maxLon = maxLon;
+ }
+
+ public double getMaxLat() {
+ return maxLat;
+ }
+
+ public void setMaxLat(double maxLat) {
+ this.maxLat = maxLat;
+ }
+
+ public int getCenterZoom() {
+ return centerZoom;
+ }
+
+ public void setCenterZoom(int centerZoom) {
+ this.centerZoom = centerZoom;
+ }
+
+ public double getCenterLon() {
+ return centerLon;
+ }
+
+ public void setCenterLon(double centerLon) {
+ this.centerLon = centerLon;
+ }
+
+ public double getCenterLat() {
+ return centerLat;
+ }
+
+ public void setCenterLat(double centerLat) {
+ this.centerLat = centerLat;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ Header header = (Header) o;
+ return specVersion == header.specVersion && rootDirectoryOffset ==
header.rootDirectoryOffset
+ && rootDirectoryLength == header.rootDirectoryLength
+ && jsonMetadataOffset == header.jsonMetadataOffset
+ && jsonMetadataLength == header.jsonMetadataLength
+ && leafDirectoryOffset == header.leafDirectoryOffset
+ && leafDirectoryLength == header.leafDirectoryLength
+ && tileDataOffset == header.tileDataOffset && tileDataLength ==
header.tileDataLength
+ && numAddressedTiles == header.numAddressedTiles && numTileEntries ==
header.numTileEntries
+ && numTileContents == header.numTileContents && clustered ==
header.clustered
+ && minZoom == header.minZoom && maxZoom == header.maxZoom
+ && Double.compare(header.minLon, minLon) == 0 &&
Double.compare(header.minLat, minLat) == 0
+ && Double.compare(header.maxLon, maxLon) == 0 &&
Double.compare(header.maxLat, maxLat) == 0
+ && centerZoom == header.centerZoom && Double.compare(header.centerLon,
centerLon) == 0
+ && Double.compare(header.centerLat, centerLat) == 0
+ && internalCompression == header.internalCompression
+ && tileCompression == header.tileCompression && tileType ==
header.tileType;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(specVersion, rootDirectoryOffset, rootDirectoryLength,
jsonMetadataOffset,
+ jsonMetadataLength, leafDirectoryOffset, leafDirectoryLength,
tileDataOffset,
+ tileDataLength, numAddressedTiles, numTileEntries, numTileContents,
clustered,
+ internalCompression, tileCompression, tileType, minZoom, maxZoom,
minLon, minLat, maxLon,
+ maxLat, centerZoom, centerLon, centerLat);
+ }
+}
diff --git
a/baremaps-core/src/main/java/org/apache/baremaps/tilestore/pmtiles/PMTiles.java
b/baremaps-core/src/main/java/org/apache/baremaps/tilestore/pmtiles/PMTiles.java
new file mode 100644
index 00000000..e8bf2f30
--- /dev/null
+++
b/baremaps-core/src/main/java/org/apache/baremaps/tilestore/pmtiles/PMTiles.java
@@ -0,0 +1,391 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.baremaps.tilestore.pmtiles;
+
+import com.google.common.io.LittleEndianDataInputStream;
+import com.google.common.io.LittleEndianDataOutputStream;
+import com.google.common.math.LongMath;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+public class PMTiles {
+
+ public static long toNum(long low, long high) {
+ return high * 0x100000000L + low;
+ }
+
+ public static long readVarIntRemainder(LittleEndianDataInputStream input,
long l)
+ throws IOException {
+ long h, b;
+ b = input.readByte() & 0xff;
+ h = (b & 0x70) >> 4;
+ if (b < 0x80) {
+ return toNum(l, h);
+ }
+ b = input.readByte() & 0xff;
+ h |= (b & 0x7f) << 3;
+ if (b < 0x80) {
+ return toNum(l, h);
+ }
+ b = input.readByte() & 0xff;
+ h |= (b & 0x7f) << 10;
+ if (b < 0x80) {
+ return toNum(l, h);
+ }
+ b = input.readByte() & 0xff;
+ h |= (b & 0x7f) << 17;
+ if (b < 0x80) {
+ return toNum(l, h);
+ }
+ b = input.readByte() & 0xff;
+ h |= (b & 0x7f) << 24;
+ if (b < 0x80) {
+ return toNum(l, h);
+ }
+ b = input.readByte() & 0xff;
+ h |= (b & 0x01) << 31;
+ if (b < 0x80) {
+ return toNum(l, h);
+ }
+ throw new RuntimeException("Expected varint not more than 10 bytes");
+ }
+
+ public static int writeVarInt(LittleEndianDataOutputStream output, long
value)
+ throws IOException {
+ int n = 1;
+ while (value >= 0x80) {
+ output.writeByte((byte) (value | 0x80));
+ value >>>= 7;
+ n++;
+ }
+ output.writeByte((byte) value);
+ return n;
+ }
+
+ public static long readVarInt(LittleEndianDataInputStream input) throws
IOException {
+ long val, b;
+ b = input.readByte() & 0xff;
+ val = b & 0x7f;
+ if (b < 0x80) {
+ return val;
+ }
+ b = input.readByte() & 0xff;
+ val |= (b & 0x7f) << 7;
+ if (b < 0x80) {
+ return val;
+ }
+ b = input.readByte() & 0xff;
+ val |= (b & 0x7f) << 14;
+ if (b < 0x80) {
+ return val;
+ }
+ b = input.readByte() & 0xff;
+ val |= (b & 0x7f) << 21;
+ if (b < 0x80) {
+ return val;
+ }
+ val |= (b & 0x0f) << 28;
+ return readVarIntRemainder(input, val);
+ }
+
+ public static void rotate(long n, long[] xy, long rx, long ry) {
+ if (ry == 0) {
+ if (rx == 1) {
+ xy[0] = n - 1 - xy[0];
+ xy[1] = n - 1 - xy[1];
+ }
+ long t = xy[0];
+ xy[0] = xy[1];
+ xy[1] = t;
+ }
+ }
+
+ public static long[] idOnLevel(int z, long pos) {
+ long n = LongMath.pow(2, z);
+ long rx, ry, t = pos;
+ long[] xy = new long[] {0, 0};
+ long s = 1;
+ while (s < n) {
+ rx = 1 & (t / 2);
+ ry = 1 & (t ^ rx);
+ rotate(s, xy, rx, ry);
+ xy[0] += s * rx;
+ xy[1] += s * ry;
+ t = t / 4;
+ s *= 2;
+ }
+ return new long[] {z, xy[0], xy[1]};
+ }
+
+ private static long[] tzValues = new long[] {
+ 0, 1, 5, 21, 85, 341, 1365, 5461, 21845, 87381, 349525, 1398101, 5592405,
+ 22369621, 89478485, 357913941, 1431655765, 5726623061L, 22906492245L,
+ 91625968981L, 366503875925L, 1466015503701L, 5864062014805L,
23456248059221L,
+ 93824992236885L, 375299968947541L, 1501199875790165L,
+ };
+
+ public static long zxyToTileId(int z, long x, long y) {
+ if (z > 26) {
+ throw new RuntimeException("Tile zoom level exceeds max safe number
limit (26)");
+ }
+ if (x > Math.pow(2, z) - 1 || y > Math.pow(2, z) - 1) {
+ throw new RuntimeException("tile x/y outside zoom level bounds");
+ }
+ long acc = tzValues[z];
+ long n = LongMath.pow(2, z);
+ long rx = 0;
+ long ry = 0;
+ long d = 0;
+ long[] xy = new long[] {x, y};
+ long s = n / 2;
+ while (s > 0) {
+ rx = (xy[0] & s) > 0 ? 1 : 0;
+ ry = (xy[1] & s) > 0 ? 1 : 0;
+ d += s * s * ((3 * rx) ^ ry);
+ rotate(s, xy, rx, ry);
+ s = s / 2;
+ }
+ return acc + d;
+ }
+
+ public static long[] tileIdToZxy(long i) {
+ long acc = 0;
+ for (int z = 0; z < 27; z++) {
+ long numTiles = (0x1L << z) * (0x1L << z);
+ if (acc + numTiles > i) {
+ return idOnLevel(z, i - acc);
+ }
+ acc += numTiles;
+ }
+ throw new RuntimeException("Tile zoom level exceeds max safe number limit
(26)");
+ }
+
+ private static final int HEADER_SIZE_BYTES = 127;
+
+ public static Header deserializeHeader(LittleEndianDataInputStream input)
throws IOException {
+ input.skipBytes(7);
+ return new Header(
+ input.readByte(),
+ input.readLong(),
+ input.readLong(),
+ input.readLong(),
+ input.readLong(),
+ input.readLong(),
+ input.readLong(),
+ input.readLong(),
+ input.readLong(),
+ input.readLong(),
+ input.readLong(),
+ input.readLong(),
+ input.readByte() == 1,
+ Compression.values()[input.readByte()],
+ Compression.values()[input.readByte()],
+ TileType.values()[input.readByte()],
+ input.readByte(),
+ input.readByte(),
+ (double) input.readInt() / 10000000,
+ (double) input.readInt() / 10000000,
+ (double) input.readInt() / 10000000,
+ (double) input.readInt() / 10000000,
+ input.readByte(),
+ (double) input.readInt() / 10000000,
+ (double) input.readInt() / 10000000);
+ }
+
+ public static void serializeHeader(LittleEndianDataOutputStream output,
Header header)
+ throws IOException {
+ output.writeByte((byte) 0x50);
+ output.writeByte((byte) 0x4D);
+ output.writeByte((byte) 0x54);
+ output.writeByte((byte) 0x69);
+ output.writeByte((byte) 0x6C);
+ output.writeByte((byte) 0x65);
+ output.writeByte((byte) 0x73);
+ output.writeByte((byte) header.getSpecVersion());
+ output.writeLong(header.getRootDirectoryOffset());
+ output.writeLong(header.getRootDirectoryLength());
+ output.writeLong(header.getJsonMetadataOffset());
+ output.writeLong(header.getJsonMetadataLength());
+ output.writeLong(header.getLeafDirectoryOffset());
+ output.writeLong(header.getLeafDirectoryLength());
+ output.writeLong(header.getTileDataOffset());
+ output.writeLong(header.getTileDataLength());
+ output.writeLong(header.getNumAddressedTiles());
+ output.writeLong(header.getNumTileEntries());
+ output.writeLong(header.getNumTileContents());
+ output.writeByte((byte) (header.isClustered() ? 1 : 0));
+ output.writeByte((byte) header.getInternalCompression().ordinal());
+ output.writeByte((byte) header.getTileCompression().ordinal());
+ output.writeByte((byte) header.getTileType().ordinal());
+ output.writeByte((byte) header.getMinZoom());
+ output.writeByte((byte) header.getMaxZoom());
+ output.writeInt((int) (header.getMinLon() * 10000000));
+ output.writeInt((int) (header.getMinLat() * 10000000));
+ output.writeInt((int) (header.getMaxLon() * 10000000));
+ output.writeInt((int) (header.getMaxLat() * 10000000));
+ output.writeByte((byte) header.getCenterZoom());
+ output.writeInt((int) (header.getCenterLon() * 10000000));
+ output.writeInt((int) (header.getCenterLat() * 10000000));
+ }
+
+ public static void serializeEntries(LittleEndianDataOutputStream output,
List<Entry> entries)
+ throws IOException {
+ writeVarInt(output, entries.size());
+ long lastId = 0;
+ for (Entry entry : entries) {
+ writeVarInt(output, entry.getTileId() - lastId);
+ lastId = entry.getTileId();
+ }
+ for (Entry entry : entries) {
+ writeVarInt(output, entry.getRunLength());
+ }
+ for (Entry entry : entries) {
+ writeVarInt(output, entry.getLength());
+ }
+ for (int i = 0; i < entries.size(); i++) {
+ Entry entry = entries.get(i);
+ if (i > 0
+ && entry.getOffset() == entries.get(i - 1).getOffset() +
entries.get(i - 1).getLength()) {
+ writeVarInt(output, 0);
+ } else {
+ writeVarInt(output, entry.getOffset() + 1);
+ }
+ }
+ }
+
+ public static List<Entry> deserializeEntries(LittleEndianDataInputStream
buffer)
+ throws IOException {
+ long numEntries = readVarInt(buffer);
+ List<Entry> entries = new ArrayList<>((int) numEntries);
+ long lastId = 0;
+ for (int i = 0; i < numEntries; i++) {
+ long value = readVarInt(buffer);
+ lastId = lastId + value;
+ Entry entry = new Entry();
+ entry.setTileId(lastId);
+ entries.add(entry);
+ }
+ for (int i = 0; i < numEntries; i++) {
+ long value = readVarInt(buffer);
+ entries.get(i).setRunLength(value);
+ }
+ for (int i = 0; i < numEntries; i++) {
+ long value = readVarInt(buffer);
+ entries.get(i).setLength(value);
+ }
+ for (int i = 0; i < numEntries; i++) {
+ long value = readVarInt(buffer);
+ if (value == 0 && i > 0) {
+ Entry prevEntry = entries.get(i - 1);
+ entries.get(i).setOffset(prevEntry.getOffset() +
prevEntry.getLength());;
+ } else {
+ entries.get(i).setOffset(value - 1);
+ }
+ }
+ return entries;
+ }
+
+ public static Entry findTile(List<Entry> entries, long tileId) {
+ int m = 0;
+ int n = entries.size() - 1;
+ while (m <= n) {
+ int k = (n + m) >> 1;
+ long cmp = tileId - entries.get(k).getTileId();
+ if (cmp > 0) {
+ m = k + 1;
+ } else if (cmp < 0) {
+ n = k - 1;
+ } else {
+ return entries.get(k);
+ }
+ }
+
+ // at this point, m > n
+ if (n >= 0) {
+ if (entries.get(n).getRunLength() == 0) {
+ return entries.get(n);
+ }
+ if (tileId - entries.get(n).getTileId() < entries.get(n).getRunLength())
{
+ return entries.get(n);
+ }
+ }
+ return null;
+ }
+
+ public static Directories buildRootLeaves(List<Entry> entries, int leafSize)
throws IOException {
+ var rootEntries = new ArrayList<Entry>();
+ var numLeaves = 0;
+ byte[] leavesBytes;
+ byte[] rootBytes;
+
+ try (var leavesOutput = new ByteArrayOutputStream();
+ var leavesDataOutput = new LittleEndianDataOutputStream(leavesOutput))
{
+ for (var i = 0; i < entries.size(); i += leafSize) {
+ numLeaves++;
+ var end = i + leafSize;
+ if (i + leafSize > entries.size()) {
+ end = entries.size();
+ }
+
+ var offset = leavesOutput.size();
+ serializeEntries(leavesDataOutput, entries.subList(i, end));
+ var length = leavesOutput.size();
+ rootEntries.add(new Entry(entries.get(i).getTileId(), offset, length,
0));
+ }
+
+ leavesBytes = leavesOutput.toByteArray();
+ }
+
+ try (var rootOutput = new ByteArrayOutputStream();
+ var rootDataOutput = new LittleEndianDataOutputStream(rootOutput)) {
+ serializeEntries(rootDataOutput, rootEntries);
+ rootBytes = rootOutput.toByteArray();
+ }
+
+ return new Directories(rootBytes, leavesBytes, numLeaves);
+ }
+
+ public static Directories optimizeDirectories(List<Entry> entries, int
targetRootLenght)
+ throws IOException {
+ if (entries.size() < 16384) {
+ byte[] rootBytes;
+ try (var rootOutput = new ByteArrayOutputStream();
+ var rootDataOutput = new LittleEndianDataOutputStream(rootOutput)) {
+ serializeEntries(rootDataOutput, entries);
+ rootBytes = rootOutput.toByteArray();
+ }
+ if (rootBytes.length <= targetRootLenght) {
+ return new Directories(rootBytes, new byte[] {}, 0);
+ }
+ }
+
+ double leafSize = (double) entries.size() / 3500;
+ if (leafSize < 4096) {
+ leafSize = 4096;
+ }
+ for (;;) {
+ var directories = buildRootLeaves(entries, (int) leafSize);
+ if (directories.getRoot().length <= targetRootLenght) {
+ return directories;
+ }
+ leafSize = leafSize * 1.2;
+ }
+ }
+}
diff --git
a/baremaps-core/src/main/java/org/apache/baremaps/tilestore/pmtiles/PMTilesReader.java
b/baremaps-core/src/main/java/org/apache/baremaps/tilestore/pmtiles/PMTilesReader.java
new file mode 100644
index 00000000..42a43af4
--- /dev/null
+++
b/baremaps-core/src/main/java/org/apache/baremaps/tilestore/pmtiles/PMTilesReader.java
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.baremaps.tilestore.pmtiles;
+
+import com.google.common.io.LittleEndianDataInputStream;
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.channels.FileChannel;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.List;
+
+public class PMTilesReader {
+
+ private final Path path;
+
+ private Header header;
+
+ private List<Entry> rootEntries;
+
+ public PMTilesReader(Path path) {
+ this.path = path;
+ }
+
+ public Header getHeader() {
+ if (header == null) {
+ try (var inputStream = new
LittleEndianDataInputStream(Files.newInputStream(path))) {
+ header = PMTiles.deserializeHeader(inputStream);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ return header;
+ }
+
+ public List<Entry> getRootDirectory() {
+ if (rootEntries == null) {
+ var header = getHeader();
+ rootEntries = getDirectory(header.getRootDirectoryOffset());
+ }
+ return rootEntries;
+ }
+
+ public List<Entry> getDirectory(long offset) {
+ var header = getHeader();
+ try (var input = Files.newInputStream(path)) {
+ input.skip(offset);
+ try (var decompressed =
+ new
LittleEndianDataInputStream(header.getInternalCompression().decompress(input)))
{
+ return PMTiles.deserializeEntries(decompressed);
+ }
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ public ByteBuffer getTile(int z, long x, long y) {
+ var tileId = PMTiles.zxyToTileId(z, x, y);
+ var header = getHeader();
+ var entries = getRootDirectory();
+ var entry = PMTiles.findTile(entries, tileId);
+
+ if (entry == null) {
+ return null;
+ }
+
+ try (var channel = FileChannel.open(path)) {
+ var compressed = ByteBuffer.allocate((int) entry.getLength());
+ channel.position(header.getTileDataOffset() + entry.getOffset());
+ channel.read(compressed);
+ compressed.flip();
+ try (var tile = new ByteArrayInputStream(compressed.array())) {
+ return ByteBuffer.wrap(tile.readAllBytes());
+ }
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+}
diff --git
a/baremaps-core/src/main/java/org/apache/baremaps/tilestore/pmtiles/PMTilesStore.java
b/baremaps-core/src/main/java/org/apache/baremaps/tilestore/pmtiles/PMTilesStore.java
new file mode 100644
index 00000000..c6319e6c
--- /dev/null
+++
b/baremaps-core/src/main/java/org/apache/baremaps/tilestore/pmtiles/PMTilesStore.java
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.baremaps.tilestore.pmtiles;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.file.Path;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Optional;
+import org.apache.baremaps.tilestore.TileCoord;
+import org.apache.baremaps.tilestore.TileStore;
+import org.apache.baremaps.tilestore.TileStoreException;
+import org.apache.baremaps.vectortile.tileset.Tileset;
+
+public class PMTilesStore implements TileStore {
+
+ private final PMTilesWriter writer;
+
+ public PMTilesStore(Path path, Tileset tileset) {
+ try {
+ var metadata = new HashMap<String, Object>();
+ metadata.put("name", tileset.getName());
+ metadata.put("type", "baselayer");
+ metadata.put("version", tileset.getVersion());
+ metadata.put("description", tileset.getDescription());
+ metadata.put("attribution", tileset.getAttribution());
+ metadata.put("vector_layers", tileset.getVectorLayers());
+
+ var minZoom = Optional.ofNullable(tileset.getMinzoom()).orElse(0);
+ var maxZoom = Optional.ofNullable(tileset.getMaxzoom()).orElse(14);
+ var bounds =
Optional.ofNullable(tileset.getBounds()).orElse(List.of(-180d, -90d, 180d,
90d));
+ var center = Optional.ofNullable(tileset.getCenter()).orElse(List.of(0d,
0d, 3d));
+
+ writer = new PMTilesWriter(path);
+ writer.setMetadata(metadata);
+ writer.setMinZoom(minZoom);
+ writer.setMaxZoom(maxZoom);
+ writer.setMinLon(bounds.get(0));
+ writer.setMinLat(bounds.get(1));
+ writer.setMaxLon(bounds.get(2));
+ writer.setMaxLat(bounds.get(3));
+ writer.setCenterLon(center.get(0));
+ writer.setCenterLat(center.get(1));
+ writer.setMinZoom(tileset.getMinzoom());
+ writer.setMaxZoom(tileset.getMaxzoom());
+
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public ByteBuffer read(TileCoord tileCoord) throws TileStoreException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void write(TileCoord tileCoord, ByteBuffer blob) throws
TileStoreException {
+ try {
+ writer.setTile(tileCoord.z(), tileCoord.x(), tileCoord.y(),
blob.array());
+ } catch (IOException e) {
+ throw new TileStoreException(e);
+ }
+ }
+
+ @Override
+ public void delete(TileCoord tileCoord) throws TileStoreException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void close() throws TileStoreException {
+ try {
+ writer.write();
+ } catch (IOException e) {
+ throw new TileStoreException(e);
+ }
+ }
+}
diff --git
a/baremaps-core/src/main/java/org/apache/baremaps/tilestore/pmtiles/PMTilesWriter.java
b/baremaps-core/src/main/java/org/apache/baremaps/tilestore/pmtiles/PMTilesWriter.java
new file mode 100644
index 00000000..f01aaf3e
--- /dev/null
+++
b/baremaps-core/src/main/java/org/apache/baremaps/tilestore/pmtiles/PMTilesWriter.java
@@ -0,0 +1,208 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.baremaps.tilestore.pmtiles;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.common.hash.Hashing;
+import com.google.common.io.LittleEndianDataOutputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.*;
+
+public class PMTilesWriter {
+
+ private Path path;
+
+ private Map<String, Object> metadata = new HashMap<>();
+
+ private List<Entry> entries;
+
+ private Map<Long, Long> tileHashToOffset;
+
+ private Long lastTileHash = null;
+
+ private Path tilePath;
+
+ private boolean clustered = true;
+
+ private int minZoom = 0;
+
+ private int maxZoom = 14;
+
+ private double minLon = -180;
+
+ private double minLat = -90;
+
+ private double maxLon = 180;
+
+ private double maxLat = 90;
+
+ private int centerZoom = 3;
+
+ private double centerLat = 0;
+
+ private double centerLon = 0;
+
+ public PMTilesWriter(Path path) throws IOException {
+ this(path, new ArrayList<>(), new HashMap<>());
+ }
+
+ public PMTilesWriter(Path path, List<Entry> entries, Map<Long, Long>
tileHashToOffset)
+ throws IOException {
+ this.path = path;
+ this.entries = entries;
+ this.tileHashToOffset = tileHashToOffset;
+ this.tilePath = Files.createTempFile(path.getParent(), "tiles_", ".tmp");
+ }
+
+ public void setMetadata(Map<String, Object> metadata) {
+ this.metadata = metadata;
+ }
+
+ public void setTile(int z, int x, int y, byte[] bytes) throws IOException {
+ // Write the tile
+ var tileId = PMTiles.zxyToTileId(z, x, y);
+ var tileLength = bytes.length;
+ Long tileHash = Hashing.farmHashFingerprint64().hashBytes(bytes).asLong();
+
+ // If the tile is not greater than the last one, the index is not clustered
+ if (entries.size() > 0 && tileId < entries.get(entries.size() -
1).getTileId()) {
+ clustered = false;
+ }
+
+ // If the tile is the same as the last one, increment the run length
+ if (clustered && tileHash.equals(lastTileHash)) {
+ var lastEntry = entries.get(entries.size() - 1);
+ lastEntry.setRunLength(lastEntry.getRunLength() + 1);
+ }
+
+ // Else, if the tile is the same as the last one, increment the run length
+ else if (tileHashToOffset.containsKey(tileHash)) {
+ var tileOffset = tileHashToOffset.get(tileHash);
+ entries.add(new Entry(tileId, tileOffset, tileLength, 1));
+ }
+
+ // Else, write the tile and add it to the index
+ else {
+ var tileOffset = Files.size(tilePath);
+ tileHashToOffset.put(tileHash, tileOffset);
+ lastTileHash = tileHash;
+ try (var output = new FileOutputStream(tilePath.toFile(), true)) {
+ output.write(bytes);
+ entries.add(new Entry(tileId, tileOffset, tileLength, 1));
+ }
+ }
+ }
+
+ public void setMinZoom(int minZoom) {
+ this.minZoom = minZoom;
+ }
+
+ public void setMaxZoom(int maxZoom) {
+ this.maxZoom = maxZoom;
+ }
+
+ public void setMinLon(double minLon) {
+ this.minLon = minLon;
+ }
+
+ public void setMinLat(double minLat) {
+ this.minLat = minLat;
+ }
+
+ public void setMaxLon(double maxLon) {
+ this.maxLon = maxLon;
+ }
+
+ public void setMaxLat(double maxLat) {
+ this.maxLat = maxLat;
+ }
+
+ public void setCenterZoom(int centerZoom) {
+ this.centerZoom = centerZoom;
+ }
+
+ public void setCenterLat(double centerLat) {
+ this.centerLat = centerLat;
+ }
+
+ public void setCenterLon(double centerLon) {
+ this.centerLon = centerLon;
+ }
+
+ public void write() throws IOException {
+ // Sort the entries by tile id
+ if (!clustered) {
+ entries.sort(Comparator.comparingLong(Entry::getTileId));
+ }
+
+ var metadataBytes = new ObjectMapper().writeValueAsBytes(metadata);
+
+ var directories = PMTiles.optimizeDirectories(entries, 16247);
+ var rootOffset = 127;
+ var rootLength = directories.getRoot().length;
+ var metadataOffset = rootOffset + rootLength;
+ var metadataLength = metadataBytes.length;
+ var leavesOffset = metadataOffset + metadataLength;
+ var leavesLength = directories.getLeaves().length;
+ var tilesOffset = leavesOffset + leavesLength;
+ var tilesLength = Files.size(tilePath);
+ var numTiles = entries.size();
+
+ var header = new Header();
+ header.setNumAddressedTiles(numTiles);
+ header.setNumTileEntries(numTiles);
+ header.setNumTileContents(numTiles);
+ header.setClustered(true);
+
+ header.setInternalCompression(Compression.None);
+ header.setTileCompression(Compression.Gzip);
+ header.setTileType(TileType.mvt);
+ header.setRootOffset(rootOffset);
+ header.setRootLength(rootLength);
+ header.setMetadataOffset(metadataOffset);
+ header.setMetadataLength(metadataLength);
+ header.setLeavesOffset(leavesOffset);
+ header.setLeavesLength(leavesLength);
+ header.setTilesOffset(tilesOffset);
+ header.setTilesLength(tilesLength);
+
+ header.setMinZoom(minZoom);
+ header.setMaxZoom(maxZoom);
+ header.setMinLon(minLon);
+ header.setMinLat(minLat);
+ header.setMaxLon(maxLon);
+ header.setMaxLat(maxLat);
+ header.setCenterZoom(centerZoom);
+ header.setCenterLat(centerLat);
+ header.setCenterLon(centerLon);
+
+ try (var output = new LittleEndianDataOutputStream(new
FileOutputStream(path.toFile()))) {
+ PMTiles.serializeHeader(output, header);
+ output.write(directories.getRoot());
+ output.write(metadataBytes);
+ output.write(directories.getLeaves());
+ Files.copy(tilePath, output);
+ } finally {
+ Files.delete(tilePath);
+ }
+ }
+
+}
diff --git
a/baremaps-core/src/main/java/org/apache/baremaps/tilestore/TileStoreException.java
b/baremaps-core/src/main/java/org/apache/baremaps/tilestore/pmtiles/TileType.java
similarity index 59%
copy from
baremaps-core/src/main/java/org/apache/baremaps/tilestore/TileStoreException.java
copy to
baremaps-core/src/main/java/org/apache/baremaps/tilestore/pmtiles/TileType.java
index d67bc50c..335bc488 100644
---
a/baremaps-core/src/main/java/org/apache/baremaps/tilestore/TileStoreException.java
+++
b/baremaps-core/src/main/java/org/apache/baremaps/tilestore/pmtiles/TileType.java
@@ -15,26 +15,13 @@
* limitations under the License.
*/
-package org.apache.baremaps.tilestore;
+package org.apache.baremaps.tilestore.pmtiles;
-/** Signals that an exception occurred in a {@code TileStore}. */
-public class TileStoreException extends Exception {
-
- /**
- * Constructs an {@code BlobStoreException} with the specified detail
message.
- *
- * @param message the message
- */
- public TileStoreException(String message) {
- super(message);
- }
-
- /**
- * Constructs a {@code BlobStoreException} with the specified cause.
- *
- * @param cause the cause
- */
- public TileStoreException(Throwable cause) {
- super(cause);
- }
+enum TileType {
+ unknown,
+ mvt,
+ png,
+ jpeg,
+ webp,
+ avif,
}
diff --git
a/baremaps-core/src/main/java/org/apache/baremaps/tilestore/postgres/PostgresTileStore.java
b/baremaps-core/src/main/java/org/apache/baremaps/tilestore/postgres/PostgresTileStore.java
index add70a06..382adb98 100644
---
a/baremaps-core/src/main/java/org/apache/baremaps/tilestore/postgres/PostgresTileStore.java
+++
b/baremaps-core/src/main/java/org/apache/baremaps/tilestore/postgres/PostgresTileStore.java
@@ -99,6 +99,9 @@ public class PostgresTileStore implements TileStore {
byte[] bytes = resultSet.getBytes(1);
gzip.write(bytes);
}
+ } catch (Exception e) {
+ throw new TileStoreException(String.format("Failed to execute
statement: %s", statement),
+ e);
}
// Log slow queries (> 10s)
@@ -125,7 +128,7 @@ public class PostgresTileStore implements TileStore {
protected static Query prepareQuery(Tileset tileset, int zoom) {
// Initialize a builder for the tile sql
var tileSql = new StringBuilder();
- tileSql.append("SELECT (");
+ tileSql.append("SELECT ");
// Iterate over the layers and keep track of the number of layers and
parameters included in the
// final sql
@@ -136,7 +139,7 @@ public class PostgresTileStore implements TileStore {
// Initialize a builder for the layer sql
var layerSql = new StringBuilder();
- var layerHead = "(WITH mvtGeom AS (";
+ var layerHead = String.format("(SELECT ST_AsMVT(mvtGeom.*, '%s') FROM
(", layer.getId());
layerSql.append(layerHead);
// Iterate over the queries and keep track of the number of queries
included in the final
@@ -154,7 +157,8 @@ public class PostgresTileStore implements TileStore {
}
// Add the sql to the layer sql
- var querySql = query.getSql()
+ var querySql = query.getSql().trim()
+ .replaceAll("\\s+", " ")
.replace(";", "")
.replace("?", "??")
.replace("$zoom", String.valueOf(zoom));
@@ -171,8 +175,7 @@ public class PostgresTileStore implements TileStore {
}
// Add the tail of the layer sql
- var layerQueryTail =
- String.format(") SELECT ST_AsMVT(mvtGeom.*, '%s') FROM mvtGeom)",
layer.getId());
+ var layerQueryTail = ") AS mvtGeom)";
layerSql.append(layerQueryTail);
// Only include the layer sql if queries were included for this layer
@@ -192,7 +195,7 @@ public class PostgresTileStore implements TileStore {
}
// Add the tail of the tile sql
- var tileQueryTail = ") mvtTile";
+ var tileQueryTail = " mvtTile";
tileSql.append(tileQueryTail);
// Format the sql query
@@ -216,4 +219,9 @@ public class PostgresTileStore implements TileStore {
public void delete(TileCoord tileCoord) {
throw new UnsupportedOperationException("The postgis tile store is read
only");
}
+
+ @Override
+ public void close() throws Exception {
+ // do nothing
+ }
}
diff --git
a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExportVectorTiles.java
b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExportVectorTiles.java
index 3f5e1854..93a27a2c 100644
---
a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExportVectorTiles.java
+++
b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExportVectorTiles.java
@@ -33,6 +33,7 @@ import org.apache.baremaps.stream.StreamUtils;
import org.apache.baremaps.tilestore.*;
import org.apache.baremaps.tilestore.file.FileTileStore;
import org.apache.baremaps.tilestore.mbtiles.MBTilesStore;
+import org.apache.baremaps.tilestore.pmtiles.PMTilesStore;
import org.apache.baremaps.tilestore.postgres.PostgresTileStore;
import org.apache.baremaps.utils.SqliteUtils;
import org.apache.baremaps.vectortile.tileset.Tileset;
@@ -52,7 +53,8 @@ public record ExportVectorTiles(
public enum Format {
file,
- mbtiles
+ mbtiles,
+ pmtiles
}
private static final Logger logger =
LoggerFactory.getLogger(ExportVectorTiles.class);
@@ -64,40 +66,43 @@ public record ExportVectorTiles(
var tileset = objectMapper.readValue(configReader.read(this.tileset),
Tileset.class);
var datasource = context.getDataSource(tileset.getDatabase());
- var sourceTileStore = sourceTileStore(tileset, datasource);
- var targetTileStore = targetTileStore(tileset);
-
- var envelope = tileset.getBounds().size() == 4
- ? new Envelope(
- tileset.getBounds().get(0), tileset.getBounds().get(2),
- tileset.getBounds().get(1), tileset.getBounds().get(3))
- : new Envelope(-180, 180, -85.0511, 85.0511);
-
- var count = TileCoord.count(envelope, tileset.getMinzoom(),
tileset.getMaxzoom());
- var start = System.currentTimeMillis();
-
- var tileCoordIterator =
- TileCoord.iterator(envelope, tileset.getMinzoom(),
tileset.getMaxzoom());
- var tileCoordStream =
- StreamUtils.stream(tileCoordIterator).peek(new ProgressLogger<>(count,
5000));
- var bufferedTileEntryStream =
StreamUtils.bufferInCompletionOrder(tileCoordStream, tile -> {
- try {
- return new TileEntry(tile, sourceTileStore.read(tile));
- } catch (TileStoreException e) {
- throw new RuntimeException(e);
- }
- }, 1000);
- var partitionedTileEntryStream =
StreamUtils.partition(bufferedTileEntryStream, 1000);
- partitionedTileEntryStream.forEach(batch -> {
- try {
- targetTileStore.write(batch);
- } catch (TileStoreException e) {
- throw new RuntimeException(e);
- }
- });
-
- var stop = System.currentTimeMillis();
- logger.info("Exported {} tiles in {}s", count, (stop - start) / 1000);
+ try (var sourceTileStore = sourceTileStore(tileset, datasource);
+ var targetTileStore = targetTileStore(tileset)) {
+
+ var envelope = tileset.getBounds().size() == 4
+ ? new Envelope(
+ tileset.getBounds().get(0), tileset.getBounds().get(2),
+ tileset.getBounds().get(1), tileset.getBounds().get(3))
+ : new Envelope(-180, 180, -85.0511, 85.0511);
+
+ var count = TileCoord.count(envelope, tileset.getMinzoom(),
tileset.getMaxzoom());
+ var start = System.currentTimeMillis();
+
+ var tileCoordIterator =
+ TileCoord.iterator(envelope, tileset.getMinzoom(),
tileset.getMaxzoom());
+ var tileCoordStream =
+ StreamUtils.stream(tileCoordIterator).peek(new
ProgressLogger<>(count, 5000));
+
+ var bufferedTileEntryStream =
StreamUtils.bufferInCompletionOrder(tileCoordStream, tile -> {
+ try {
+ return new TileEntry(tile, sourceTileStore.read(tile));
+ } catch (TileStoreException e) {
+ throw new RuntimeException(e);
+ }
+ }, 1000);
+
+ var partitionedTileEntryStream =
StreamUtils.partition(bufferedTileEntryStream, 1000);
+ partitionedTileEntryStream.forEach(batch -> {
+ try {
+ targetTileStore.write(batch);
+ } catch (TileStoreException e) {
+ throw new RuntimeException(e);
+ }
+ });
+
+ var stop = System.currentTimeMillis();
+ logger.info("Exported {} tiles in {}s", count, (stop - start) / 1000);
+ }
}
private TileStore sourceTileStore(Tileset tileset, DataSource datasource) {
@@ -115,6 +120,10 @@ public record ExportVectorTiles(
tilesStore.initializeDatabase();
tilesStore.writeMetadata(metadata(source));
return tilesStore;
+ case pmtiles:
+ Files.deleteIfExists(repository);
+ var tileStore = new PMTilesStore(repository, source);
+ return tileStore;
default:
throw new IllegalArgumentException("Unsupported format");
}
diff --git
a/baremaps-core/src/test/java/org/apache/baremaps/tilestore/pmtiles/PMTilesTest.java
b/baremaps-core/src/test/java/org/apache/baremaps/tilestore/pmtiles/PMTilesTest.java
new file mode 100644
index 00000000..e078d1d5
--- /dev/null
+++
b/baremaps-core/src/test/java/org/apache/baremaps/tilestore/pmtiles/PMTilesTest.java
@@ -0,0 +1,288 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.baremaps.tilestore.pmtiles;
+
+import static org.junit.jupiter.api.Assertions.*;
+
+import com.google.common.io.LittleEndianDataInputStream;
+import com.google.common.io.LittleEndianDataOutputStream;
+import com.google.common.math.LongMath;
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.nio.channels.Channels;
+import java.nio.channels.FileChannel;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Random;
+import org.apache.baremaps.testing.TestFiles;
+import org.junit.jupiter.api.Test;
+
+class PMTilesTest {
+
+ @Test
+ void decodeVarInt() throws IOException {
+ var b = new LittleEndianDataInputStream(new ByteArrayInputStream(new
byte[] {
+ (byte) 0, (byte) 1,
+ (byte) 127, (byte) 0xe5,
+ (byte) 0x8e, (byte) 0x26
+ }));
+ assertEquals(PMTiles.readVarInt(b), 0);
+ assertEquals(PMTiles.readVarInt(b), 1);
+ assertEquals(PMTiles.readVarInt(b), 127);
+ assertEquals(PMTiles.readVarInt(b), 624485);
+ b = new LittleEndianDataInputStream(new ByteArrayInputStream(new byte[] {
+ (byte) 0xff, (byte) 0xff,
+ (byte) 0xff, (byte) 0xff,
+ (byte) 0xff, (byte) 0xff,
+ (byte) 0xff, (byte) 0x0f,
+ }));
+ assertEquals(PMTiles.readVarInt(b), 9007199254740991L);
+ }
+
+ @Test
+ void encodeVarInt() throws IOException {
+ for (long i = 0; i < 1000; i++) {
+ var array = new ByteArrayOutputStream();
+ var output = new LittleEndianDataOutputStream(array);
+ PMTiles.writeVarInt(output, i);
+ var input = new LittleEndianDataInputStream(new
ByteArrayInputStream(array.toByteArray()));
+ assertEquals(i, PMTiles.readVarInt(input));
+ }
+ for (long i = Long.MAX_VALUE - 1000; i < Long.MAX_VALUE; i++) {
+ var array = new ByteArrayOutputStream();
+ var output = new LittleEndianDataOutputStream(array);
+ PMTiles.writeVarInt(output, i);
+ var input = new LittleEndianDataInputStream(new
ByteArrayInputStream(array.toByteArray()));
+ assertEquals(i, PMTiles.readVarInt(input));
+ }
+ }
+
+ @Test
+ void zxyToTileId() {
+ assertEquals(PMTiles.zxyToTileId(0, 0, 0), 0);
+ assertEquals(PMTiles.zxyToTileId(1, 0, 0), 1);
+ assertEquals(PMTiles.zxyToTileId(1, 0, 1), 2);
+ assertEquals(PMTiles.zxyToTileId(1, 1, 1), 3);
+ assertEquals(PMTiles.zxyToTileId(1, 1, 0), 4);
+ assertEquals(PMTiles.zxyToTileId(2, 0, 0), 5);
+ }
+
+ @Test
+ void tileIdToZxy() {
+ assertArrayEquals(PMTiles.tileIdToZxy(0), new long[] {0, 0, 0});
+ assertArrayEquals(PMTiles.tileIdToZxy(1), new long[] {1, 0, 0});
+ assertArrayEquals(PMTiles.tileIdToZxy(2), new long[] {1, 0, 1});
+ assertArrayEquals(PMTiles.tileIdToZxy(3), new long[] {1, 1, 1});
+ assertArrayEquals(PMTiles.tileIdToZxy(4), new long[] {1, 1, 0});
+ assertArrayEquals(PMTiles.tileIdToZxy(5), new long[] {2, 0, 0});
+ }
+
+ @Test
+ void aLotOfTiles() {
+ for (int z = 0; z < 9; z++) {
+ for (long x = 0; x < 1 << z; x++) {
+ for (long y = 0; y < 1 << z; y++) {
+ var result = PMTiles.tileIdToZxy(PMTiles.zxyToTileId(z, x, y));
+ if (result[0] != z || result[1] != x || result[2] != y) {
+ fail("roundtrip failed");
+ }
+ }
+ }
+ }
+ }
+
+ @Test
+ void tileExtremes() {
+ for (var z = 0; z < 27; z++) {
+ var dim = LongMath.pow(2, z) - 1;
+ var tl = PMTiles.tileIdToZxy(PMTiles.zxyToTileId(z, 0, 0));
+ assertArrayEquals(new long[] {z, 0, 0}, tl);
+ var tr = PMTiles.tileIdToZxy(PMTiles.zxyToTileId(z, dim, 0));
+ assertArrayEquals(new long[] {z, dim, 0}, tr);
+ var bl = PMTiles.tileIdToZxy(PMTiles.zxyToTileId(z, 0, dim));
+ assertArrayEquals(new long[] {z, 0, dim}, bl);
+ var br = PMTiles.tileIdToZxy(PMTiles.zxyToTileId(z, dim, dim));
+ assertArrayEquals(new long[] {z, dim, dim}, br);
+ }
+ }
+
+ @Test
+ void invalidTiles() {
+ assertThrows(RuntimeException.class, () ->
PMTiles.tileIdToZxy(9007199254740991L));
+ assertThrows(RuntimeException.class, () -> PMTiles.zxyToTileId(27, 0, 0));
+ assertThrows(RuntimeException.class, () -> PMTiles.zxyToTileId(0, 1, 1));
+ }
+
+ @Test
+ void decodeHeader() throws IOException {
+ var file = TestFiles.resolve("pmtiles/test_fixture_1.pmtiles");
+ try (var channel = FileChannel.open(file)) {
+ var input = new
LittleEndianDataInputStream(Channels.newInputStream(channel));
+ var header = PMTiles.deserializeHeader(input);
+ assertEquals(header.getRootDirectoryOffset(), 127);
+ assertEquals(header.getRootDirectoryLength(), 25);
+ assertEquals(header.getJsonMetadataOffset(), 152);
+ assertEquals(header.getJsonMetadataLength(), 247);
+ assertEquals(header.getLeafDirectoryOffset(), 0);
+ assertEquals(header.getLeafDirectoryLength(), 0);
+ assertEquals(header.getTileDataOffset(), 399);
+ assertEquals(header.getTileDataLength(), 69);
+ assertEquals(header.getNumAddressedTiles(), 1);
+ assertEquals(header.getNumTileEntries(), 1);
+ assertEquals(header.getNumTileContents(), 1);
+ assertFalse(header.isClustered());
+ assertEquals(header.getInternalCompression(), Compression.Gzip);
+ assertEquals(header.getTileCompression(), Compression.Gzip);
+ assertEquals(header.getTileType(), TileType.mvt);
+ assertEquals(header.getMinZoom(), 0);
+ assertEquals(header.getMaxZoom(), 0);
+ assertEquals(header.getMinLon(), 0);
+ assertEquals(header.getMinLat(), 0);
+ assertEquals(Math.round(header.getMaxLon()), 1);
+ assertEquals(Math.round(header.getMaxLat()), 1);
+ }
+ }
+
+ @Test
+ void encodeHeader() throws IOException {
+ var etag = "1";
+ var header = new Header(
+ 127,
+ 25,
+ 152,
+ 247,
+ 0,
+ 0,
+ 399,
+ 69,
+ 1,
+ 1,
+ 1,
+ 10,
+ false,
+ Compression.Gzip,
+ Compression.Gzip,
+ TileType.mvt,
+ 0,
+ 0,
+ 0,
+ 1,
+ 1,
+ 0,
+ 0,
+ 0,
+ 0);
+
+ var array = new ByteArrayOutputStream();
+
+ var output = new LittleEndianDataOutputStream(array);
+ PMTiles.serializeHeader(output, header);
+
+ var input = new LittleEndianDataInputStream(new
ByteArrayInputStream(array.toByteArray()));
+ var header2 = PMTiles.deserializeHeader(input);
+
+ assertEquals(header, header2);
+ }
+
+ @Test
+ void searchForMissingEntry() {
+ var entries = new ArrayList<Entry>();
+ assertEquals(PMTiles.findTile(entries, 101), null);
+ }
+
+ @Test
+ void searchForFirstEntry() {
+ var entry = new Entry(100, 1, 1, 1);
+ var entries = new ArrayList<Entry>();
+ entries.add(entry);
+ assertEquals(PMTiles.findTile(entries, 100), entry);
+ }
+
+ @Test
+ void searchWithRunLength() {
+ var entry = new Entry(3, 3, 1, 2);
+ var entries = new ArrayList<Entry>();
+ entries.add(entry);
+ entries.add(new Entry(5, 5, 1, 2));
+ assertEquals(PMTiles.findTile(entries, 4), entry);
+ }
+
+ @Test
+ void searchWithMultipleTileEntries() {
+ var entries = new ArrayList<Entry>();
+ entries.add(new Entry(100, 1, 1, 2));
+ var entry = PMTiles.findTile(entries, 101);
+ assertEquals(entry.getOffset(), 1);
+ assertEquals(entry.getLength(), 1);
+
+ entries = new ArrayList<Entry>();
+ entries.add(new Entry(100, 1, 1, 1));
+ entries.add(new Entry(150, 2, 2, 2));
+ entry = PMTiles.findTile(entries, 151);
+ assertEquals(entry.getOffset(), 2);
+ assertEquals(entry.getLength(), 2);
+
+ entries = new ArrayList<>();
+ entries.add(new Entry(50, 1, 1, 2));
+ entries.add(new Entry(100, 2, 2, 1));
+ entries.add(new Entry(150, 3, 3, 1));
+ entry = PMTiles.findTile(entries, 51);
+ assertEquals(entry.getOffset(), 1);
+ assertEquals(entry.getLength(), 1);
+ }
+
+ @Test
+ void leafSearch() {
+ var entries = new ArrayList<Entry>();
+ entries.add(new Entry(100, 1, 1, 0));
+ var entry = PMTiles.findTile(entries, 150);
+ assertEquals(entry.getOffset(), 1);
+ assertEquals(entry.getLength(), 1);
+ }
+
+ @Test
+ void buildRootLeaves() throws IOException {
+ var entries = List.of(new Entry(100, 1, 1, 0));
+ var directories = PMTiles.buildRootLeaves(entries, 1);
+ assertEquals(directories.getNumLeaves(), 1);
+
+ }
+
+ @Test
+ void optimizeDirectories() throws IOException {
+ var random = new Random(3857);
+ var entries = new ArrayList<Entry>();
+ entries.add(new Entry(0, 0, 100, 1));
+ var directories = PMTiles.optimizeDirectories(entries, 100);
+ assertFalse(directories.getLeaves().length > 0);
+ assertEquals(0, directories.getNumLeaves());
+
+ entries = new ArrayList<>();
+ int offset = 0;
+ for (var i = 0; i < 1000; i++) {
+ var randTileSize = random.nextInt(1000000);
+ entries.add(new Entry(i, offset, randTileSize, 1));
+ offset += randTileSize;
+ }
+ directories = PMTiles.optimizeDirectories(entries, 1024);
+ assertFalse(directories.getRoot().length > 1024);
+ assertFalse(directories.getNumLeaves() == 0);
+ assertFalse(directories.getLeaves().length == 0);
+ }
+}
diff --git
a/baremaps-core/src/test/java/org/apache/baremaps/tilestore/postgres/PostgresTileStoreTest.java
b/baremaps-core/src/test/java/org/apache/baremaps/tilestore/postgres/PostgresTileStoreTest.java
index f146a2cf..24892ac1 100644
---
a/baremaps-core/src/test/java/org/apache/baremaps/tilestore/postgres/PostgresTileStoreTest.java
+++
b/baremaps-core/src/test/java/org/apache/baremaps/tilestore/postgres/PostgresTileStoreTest.java
@@ -40,7 +40,7 @@ class PostgresTileStoreTest {
List.of(new TilesetQuery(0, 20, "SELECT id, tags, geom FROM
table")))));
var query = PostgresTileStore.prepareQuery(tileset, 10);
assertEquals(
- "SELECT ((WITH mvtGeom AS (SELECT ST_AsMVTGeom(t.geom,
ST_TileEnvelope(?, ?, ?)) AS geom, t.tags, t.id FROM (SELECT id, tags, geom
FROM table) AS t WHERE t.geom && ST_TileEnvelope(?, ?, ?, margin =>
(64.0/4096))) SELECT ST_AsMVT(mvtGeom.*, 'a') FROM mvtGeom) || (WITH mvtGeom AS
(SELECT ST_AsMVTGeom(t.geom, ST_TileEnvelope(?, ?, ?)) AS geom, t.tags, t.id
FROM (SELECT id, tags, geom FROM table) AS t WHERE t.geom && ST_TileEnvelope(?,
?, ?, margin => (64.0/4096))) SELECT ST_AsMVT(mv [...]
+ "SELECT (SELECT ST_AsMVT(mvtGeom.*, 'a') FROM (SELECT
ST_AsMVTGeom(t.geom, ST_TileEnvelope(?, ?, ?)) AS geom, t.tags, t.id FROM
(SELECT id, tags, geom FROM table) AS t WHERE t.geom && ST_TileEnvelope(?, ?,
?, margin => (64.0/4096))) AS mvtGeom) || (SELECT ST_AsMVT(mvtGeom.*, 'b') FROM
(SELECT ST_AsMVTGeom(t.geom, ST_TileEnvelope(?, ?, ?)) AS geom, t.tags, t.id
FROM (SELECT id, tags, geom FROM table) AS t WHERE t.geom && ST_TileEnvelope(?,
?, ?, margin => (64.0/4096))) AS mvtGeom) [...]
query.sql());
}
}
diff --git
a/baremaps-core/src/test/java/org/apache/baremaps/vectortile/VectorTileViewer.java
b/baremaps-core/src/test/java/org/apache/baremaps/vectortile/VectorTileViewer.java
index faada968..36e925f4 100644
---
a/baremaps-core/src/test/java/org/apache/baremaps/vectortile/VectorTileViewer.java
+++
b/baremaps-core/src/test/java/org/apache/baremaps/vectortile/VectorTileViewer.java
@@ -49,7 +49,7 @@ public class VectorTileViewer {
}
}
- static class TilePanel extends JPanel {
+ public static class TilePanel extends JPanel {
private final Tile tile;
diff --git a/baremaps-core/src/test/resources/pmtiles/empty.pmtiles
b/baremaps-core/src/test/resources/pmtiles/empty.pmtiles
new file mode 100644
index 00000000..e69de29b
diff --git a/baremaps-core/src/test/resources/pmtiles/invalid.pmtiles
b/baremaps-core/src/test/resources/pmtiles/invalid.pmtiles
new file mode 100644
index 00000000..f2b720ba
--- /dev/null
+++ b/baremaps-core/src/test/resources/pmtiles/invalid.pmtiles
@@ -0,0 +1 @@
+This is an invalid tile archive, a test case to make sure that the code throws
an error, but it needs to be the minimum size to pass the first test
diff --git a/baremaps-core/src/test/resources/pmtiles/invalid_v4.pmtiles
b/baremaps-core/src/test/resources/pmtiles/invalid_v4.pmtiles
new file mode 100644
index 00000000..1871cb27
Binary files /dev/null and
b/baremaps-core/src/test/resources/pmtiles/invalid_v4.pmtiles differ
diff --git a/baremaps-core/src/test/resources/pmtiles/test_fixture_1.pmtiles
b/baremaps-core/src/test/resources/pmtiles/test_fixture_1.pmtiles
new file mode 100644
index 00000000..c86db1f2
Binary files /dev/null and
b/baremaps-core/src/test/resources/pmtiles/test_fixture_1.pmtiles differ
diff --git a/baremaps-core/src/test/resources/pmtiles/test_fixture_2.pmtiles
b/baremaps-core/src/test/resources/pmtiles/test_fixture_2.pmtiles
new file mode 100644
index 00000000..cb19dd5f
Binary files /dev/null and
b/baremaps-core/src/test/resources/pmtiles/test_fixture_2.pmtiles differ
diff --git a/basemap/import.js b/basemap/import.js
index fec1e00a..75715991 100644
--- a/basemap/import.js
+++ b/basemap/import.js
@@ -18,6 +18,22 @@ import config from "./config.js";
export default {
"steps": [
+ {
+ "id": "initialize-database",
+ "needs": [],
+ "tasks": [
+ {
+ "type": "ExecuteSqlScript",
+ "file": "queries/assertions.sql",
+ "database": config.database,
+ },
+ {
+ "type": "ExecuteSqlScript",
+ "file": "queries/functions.sql",
+ "database": config.database,
+ },
+ ]
+ },
{
"id": "natural-earth",
"needs": [],
diff --git a/basemap/layers/building/extrusion.js
b/basemap/layers/building/extrusion.js
index 51081931..f5ebe305 100644
--- a/basemap/layers/building/extrusion.js
+++ b/basemap/layers/building/extrusion.js
@@ -45,15 +45,6 @@ export default {
16,
['get', "extrusion:height"]
],
- "fill-extrusion-opacity": [
- 'interpolate',
- ['linear'],
- ['zoom'],
- 15,
- 0,
- 16,
- 0.8
- ],
"fill-extrusion-color": theme.buildingShapeFillColor,
// Having muliple colors for building parts results in z-fighting
// https://github.com/maplibre/maplibre-gl-js/issues/3157
diff --git a/basemap/layers/building/tileset.js
b/basemap/layers/building/tileset.js
index e7fabd2f..12c9a8b7 100644
--- a/basemap/layers/building/tileset.js
+++ b/basemap/layers/building/tileset.js
@@ -25,23 +25,27 @@ export default {
SELECT
id,
tags
- || jsonb_build_object('extrusion:base', (CASE
- WHEN tags ? 'building:min_height'
- THEN (tags ->> 'building:min_height')::real
- WHEN tags ->> 'building:min_level' ~ '^[0-9.]+$'
- THEN (tags ->> 'building:min_level')::real * 3
- ELSE 0 END))
- || jsonb_build_object('extrusion:height', (CASE
- WHEN tags ? 'height'
- THEN (SUBSTRING(tags ->> 'height' FROM
'^[0-9]+'))::real
- WHEN tags ? 'building:height'
- THEN (tags ->> 'building:height')::real
- WHEN tags ->> 'building:levels' ~ '^[0-9.]+$'
- THEN (tags ->> 'building:levels')::real * 3
- ELSE 6 END)) as tags,
+ || jsonb_build_object('extrusion:base',
+ CASE
+ WHEN tags ? 'building:min_height'
+ THEN convert_to_number(tags ->>
'building:min_height', 0)
+ WHEN tags ? 'building:min_level'
+ THEN convert_to_number(tags ->>
'building:min_level', 0) * 3
+ ELSE 0
+ END)
+ || jsonb_build_object('extrusion:height',
+ CASE
+ WHEN tags ? 'height'
+ THEN convert_to_number(tags ->> 'height',
6)
+ WHEN tags ? 'building:height'
+ THEN convert_to_number(tags ->>
'building:height', 6)
+ WHEN tags ? 'building:levels'
+ THEN convert_to_number(tags ->>
'building:levels', 2) * 3
+ ELSE 6
+ END) as tags,
geom
FROM osm_ways
- WHERE (tags ? 'building' OR tags ? 'building:part') AND ((NOT
tags ? 'layer') OR (tags ->> 'layer')::real >= 0)`,
+ WHERE (tags ? 'building' OR tags ? 'building:part') AND ((NOT
tags ? 'layer') OR convert_to_number(tags ->> 'layer', 0) >= 0)`,
},
{
minzoom: 13,
@@ -50,23 +54,27 @@ export default {
SELECT
id,
tags
- || jsonb_build_object('extrusion:base', (CASE
- WHEN tags ? 'building:min_height'
- THEN (tags ->> 'building:min_height')::real
- WHEN tags ->> 'building:min_level' ~ '^[0-9.]+$'
- THEN (tags ->> 'building:min_level')::real * 3
- ELSE 0 END))
- || jsonb_build_object('extrusion:height', (CASE
- WHEN tags ? 'height'
- THEN (SUBSTRING(tags ->> 'height' FROM
'^[0-9]+'))::real
- WHEN tags ? 'building:height'
- THEN (tags ->> 'building:height')::real
- WHEN tags ->> 'building:levels' ~ '^[0-9.]+$'
- THEN (tags ->> 'building:levels')::real * 3
- ELSE 6 END)) as tags,
+ || jsonb_build_object('extrusion:base',
+ CASE
+ WHEN tags ? 'building:min_height'
+ THEN convert_to_number(tags ->>
'building:min_height', 0)
+ WHEN tags ? 'building:min_level'
+ THEN convert_to_number(tags ->>
'building:min_level', 0) * 3
+ ELSE 0
+ END)
+ || jsonb_build_object('extrusion:height',
+ CASE
+ WHEN tags ? 'height'
+ THEN convert_to_number(tags ->> 'height',
6)
+ WHEN tags ? 'building:height'
+ THEN convert_to_number(tags ->>
'building:height', 6)
+ WHEN tags ? 'building:levels'
+ THEN convert_to_number(tags ->>
'building:levels', 2) * 3
+ ELSE 6
+ END) as tags,
geom
FROM osm_relations
- WHERE (tags ? 'building' OR tags ? 'building:part') AND ((NOT
tags ? 'layer') OR (tags ->> 'layer')::real >= 0)`,
+ WHERE (tags ? 'building' OR tags ? 'building:part') AND ((NOT
tags ? 'layer') OR convert_to_number(tags ->> 'layer', 0) >= 0)`,
},
],
}
diff --git a/basemap/queries/assertions.sql b/basemap/queries/assertions.sql
new file mode 100644
index 00000000..7a2019c1
--- /dev/null
+++ b/basemap/queries/assertions.sql
@@ -0,0 +1,48 @@
+-- Licensed to the Apache Software Foundation (ASF) under one or more
+-- contributor license agreements. See the NOTICE file distributed with
+-- this work for additional information regarding copyright ownership.
+-- The ASF licenses this file to you under the Apache License, Version 2.0
+-- (the "License"); you may not use this file except in compliance with
+-- the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+
+-- Asserts that the test function raises the expected exception.
+CREATE OR REPLACE FUNCTION assert_exception(test_query text,
expected_exception text)
+ RETURNS void AS $$
+BEGIN
+ BEGIN
+ -- Execute the test query using dynamic SQL
+ EXECUTE test_query;
+
+ -- If no exception is raised, fail the test
+ RAISE EXCEPTION 'Assertion Failed: Expected exception %, but no
exception was raised', expected_exception;
+ EXCEPTION
+ WHEN OTHERS THEN
+ -- Check if the raised exception matches the expected exception
+ IF SQLERRM NOT LIKE expected_exception THEN
+ RAISE EXCEPTION 'Assertion Failed: Expected exception %, but
was %', expected_exception, SQLERRM;
+ END IF;
+ END;
+END;
+$$ LANGUAGE plpgsql;
+
+-- Asserts that the actual value is equal to the expected value.
+CREATE OR REPLACE FUNCTION assert_equals(actual numeric, expected numeric)
+ RETURNS void AS $$
+BEGIN
+ IF actual != expected THEN
+ RAISE EXCEPTION 'Assertion Failed: Expected %, but was %', expected,
actual;
+ END IF;
+END;
+$$ LANGUAGE plpgsql;
+
+-- Test cases for the assert_equals function
+SELECT assert_equals(1, 1);
+SELECT assert_exception('SELECT assert_equals(1, 2)', 'Assertion Failed:
Expected 2, but was 1');
\ No newline at end of file
diff --git a/basemap/queries/functions.sql b/basemap/queries/functions.sql
new file mode 100644
index 00000000..f07d3362
--- /dev/null
+++ b/basemap/queries/functions.sql
@@ -0,0 +1,49 @@
+-- Licensed to the Apache Software Foundation (ASF) under one or more
+-- contributor license agreements. See the NOTICE file distributed with
+-- this work for additional information regarding copyright ownership.
+-- The ASF licenses this file to you under the Apache License, Version 2.0
+-- (the "License"); you may not use this file except in compliance with
+-- the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+
+-- Converts a string to a number
+CREATE OR REPLACE FUNCTION convert_to_number(input_string text, default_value
numeric)
+ RETURNS numeric AS $$
+DECLARE
+ result numeric;
+BEGIN
+ -- Replace comma with dot
+ input_string := REPLACE(input_string, ',', '.');
+
+ -- Use a regular expression to extract the first number from the string
+ input_string := SUBSTRING(input_string FROM '^[0-9]+\.?[0-9]*');
+
+ -- Convert the extracted string to a numeric type
+ result := input_string::numeric;
+
+ IF result IS NULL THEN
+ RETURN default_value;
+ END IF;
+
+ RETURN result;
+EXCEPTION
+ WHEN others THEN
+ -- Return the default value in case of any error
+ RETURN default_value;
+END;
+$$ LANGUAGE plpgsql;
+
+-- Test cases for the convert_to_number function
+SELECT assert_equals(convert_to_number('1', 0), 1);
+SELECT assert_equals(convert_to_number('2.3', 0), 2.3);
+SELECT assert_equals(convert_to_number('3,4', 0), 3.4);
+SELECT assert_equals(convert_to_number('1.5m', 0), 1.5);
+SELECT assert_equals(convert_to_number('6.6 m', 0), 6.6);
+SELECT assert_equals(convert_to_number('abc', 0), 0);
\ No newline at end of file
