github-advanced-security[bot] commented on code in PR #794: URL: https://github.com/apache/incubator-baremaps/pull/794#discussion_r1404684078
########## baremaps-core/src/main/java/org/apache/baremaps/tilestore/pmtiles/PMTilesReader.java: ########## @@ -0,0 +1,96 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.baremaps.tilestore.pmtiles; + +import com.google.common.io.LittleEndianDataInputStream; +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.channels.FileChannel; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; + +public class PMTilesReader { + + private final Path path; + + private Header header; + + private List<Entry> rootEntries; + + public PMTilesReader(Path path) { + this.path = path; + } + + public Header getHeader() { + if (header == null) { + try (var inputStream = new LittleEndianDataInputStream(Files.newInputStream(path))) { + header = PMTiles.deserializeHeader(inputStream); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + return header; + } + + public List<Entry> getRootDirectory() { + if (rootEntries == null) { + var header = getHeader(); + rootEntries = + getDirectory(header.getRootDirectoryOffset(), (int) header.getRootDirectoryLength()); + } + return rootEntries; + } + + public List<Entry> getDirectory(long offset, int length) { + var header = getHeader(); + try (var input = Files.newInputStream(path)) { + input.skip(offset); Review Comment: ## Ignored error status of call Method getDirectory ignores exceptional return value of InputStream.skip. [Show more details](https://github.com/apache/incubator-baremaps/security/code-scanning/816) ########## baremaps-core/src/main/java/org/apache/baremaps/tilestore/pmtiles/PMTiles.java: ########## @@ -0,0 +1,391 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.baremaps.tilestore.pmtiles; + +import com.google.common.io.LittleEndianDataInputStream; +import com.google.common.io.LittleEndianDataOutputStream; +import com.google.common.math.LongMath; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +public class PMTiles { + + public static long toNum(long low, long high) { + return high * 0x100000000L + low; + } + + public static long readVarIntRemainder(LittleEndianDataInputStream input, long l) + throws IOException { + long h, b; + b = input.readByte() & 0xff; + h = (b & 0x70) >> 4; + if (b < 0x80) { + return toNum(l, h); + } + b = input.readByte() & 0xff; + h |= (b & 0x7f) << 3; + if (b < 0x80) { + return toNum(l, h); + } + b = input.readByte() & 0xff; + h |= (b & 0x7f) << 10; + if (b < 0x80) { + return toNum(l, h); + } + b = input.readByte() & 0xff; + h |= (b & 0x7f) << 17; + if (b < 0x80) { + return toNum(l, h); + } + b = input.readByte() & 0xff; + h |= (b & 0x7f) << 24; + if (b < 0x80) { + return toNum(l, h); + } + b = input.readByte() & 0xff; + h |= (b & 0x01) << 31; + if (b < 0x80) { + return toNum(l, h); + } + throw new RuntimeException("Expected varint not more than 10 bytes"); + } + + public static int writeVarInt(LittleEndianDataOutputStream output, long value) + throws IOException { + int n = 1; + while (value >= 0x80) { + output.writeByte((byte) (value | 0x80)); + value >>>= 7; + n++; + } + output.writeByte((byte) value); + return n; + } + + public static long readVarInt(LittleEndianDataInputStream input) throws IOException { + long val, b; + b = input.readByte() & 0xff; + val = b & 0x7f; + if (b < 0x80) { + return val; + } + b = input.readByte() & 0xff; + val |= (b & 0x7f) << 7; + if (b < 0x80) { + return val; + } + b = input.readByte() & 0xff; + val |= (b & 0x7f) << 14; + if (b < 0x80) { + return val; + } + b = input.readByte() & 0xff; + val |= (b & 0x7f) << 21; + if (b < 0x80) { + return val; + } + val |= (b & 0x0f) << 28; + return readVarIntRemainder(input, val); + } + + public static void rotate(long n, long[] xy, long rx, long ry) { + if (ry == 0) { + if (rx == 1) { + xy[0] = n - 1 - xy[0]; + xy[1] = n - 1 - xy[1]; + } + long t = xy[0]; + xy[0] = xy[1]; + xy[1] = t; + } + } + + public static long[] idOnLevel(int z, long pos) { + long n = LongMath.pow(2, z); + long rx, ry, t = pos; + long[] xy = new long[] {0, 0}; + long s = 1; + while (s < n) { + rx = 1 & (t / 2); + ry = 1 & (t ^ rx); + rotate(s, xy, rx, ry); + xy[0] += s * rx; + xy[1] += s * ry; + t = t / 4; + s *= 2; + } + return new long[] {z, xy[0], xy[1]}; + } + + private static long[] tzValues = new long[] { + 0, 1, 5, 21, 85, 341, 1365, 5461, 21845, 87381, 349525, 1398101, 5592405, + 22369621, 89478485, 357913941, 1431655765, 5726623061L, 22906492245L, + 91625968981L, 366503875925L, 1466015503701L, 5864062014805L, 23456248059221L, + 93824992236885L, 375299968947541L, 1501199875790165L, + }; + + public static long zxyToTileId(int z, long x, long y) { + if (z > 26) { + throw new RuntimeException("Tile zoom level exceeds max safe number limit (26)"); + } + if (x > Math.pow(2, z) - 1 || y > Math.pow(2, z) - 1) { + throw new RuntimeException("tile x/y outside zoom level bounds"); + } + long acc = tzValues[z]; + long n = LongMath.pow(2, z); + long rx = 0; + long ry = 0; + long d = 0; + long[] xy = new long[] {x, y}; + long s = n / 2; + while (s > 0) { + rx = (xy[0] & s) > 0 ? 1 : 0; + ry = (xy[1] & s) > 0 ? 1 : 0; + d += s * s * ((3 * rx) ^ ry); + rotate(s, xy, rx, ry); + s = s / 2; + } + return acc + d; + } + + public static long[] tileIdToZxy(long i) { + long acc = 0; + for (int z = 0; z < 27; z++) { + long numTiles = (0x1L << z) * (0x1L << z); + if (acc + numTiles > i) { + return idOnLevel(z, i - acc); + } + acc += numTiles; + } + throw new RuntimeException("Tile zoom level exceeds max safe number limit (26)"); + } + + private static final int HEADER_SIZE_BYTES = 127; + + public static Header deserializeHeader(LittleEndianDataInputStream input) throws IOException { + input.skipBytes(7); + return new Header( + input.readByte(), + input.readLong(), + input.readLong(), + input.readLong(), + input.readLong(), + input.readLong(), + input.readLong(), + input.readLong(), + input.readLong(), + input.readLong(), + input.readLong(), + input.readLong(), + input.readByte() == 1, + Compression.values()[input.readByte()], + Compression.values()[input.readByte()], + TileType.values()[input.readByte()], + input.readByte(), + input.readByte(), + (double) input.readInt() / 10000000, + (double) input.readInt() / 10000000, + (double) input.readInt() / 10000000, + (double) input.readInt() / 10000000, + input.readByte(), + (double) input.readInt() / 10000000, + (double) input.readInt() / 10000000); + } + + public static void serializeHeader(LittleEndianDataOutputStream output, Header header) + throws IOException { + output.writeByte((byte) 0x50); + output.writeByte((byte) 0x4D); + output.writeByte((byte) 0x54); + output.writeByte((byte) 0x69); + output.writeByte((byte) 0x6C); + output.writeByte((byte) 0x65); + output.writeByte((byte) 0x73); + output.writeByte((byte) header.getSpecVersion()); + output.writeLong(header.getRootDirectoryOffset()); + output.writeLong(header.getRootDirectoryLength()); + output.writeLong(header.getJsonMetadataOffset()); + output.writeLong(header.getJsonMetadataLength()); + output.writeLong(header.getLeafDirectoryOffset()); + output.writeLong(header.getLeafDirectoryLength()); + output.writeLong(header.getTileDataOffset()); + output.writeLong(header.getTileDataLength()); + output.writeLong(header.getNumAddressedTiles()); + output.writeLong(header.getNumTileEntries()); + output.writeLong(header.getNumTileContents()); + output.writeByte((byte) (header.isClustered() ? 1 : 0)); + output.writeByte((byte) header.getInternalCompression().ordinal()); + output.writeByte((byte) header.getTileCompression().ordinal()); + output.writeByte((byte) header.getTileType().ordinal()); + output.writeByte((byte) header.getMinZoom()); + output.writeByte((byte) header.getMaxZoom()); + output.writeInt((int) (header.getMinLon() * 10000000)); + output.writeInt((int) (header.getMinLat() * 10000000)); + output.writeInt((int) (header.getMaxLon() * 10000000)); + output.writeInt((int) (header.getMaxLat() * 10000000)); + output.writeByte((byte) header.getCenterZoom()); + output.writeInt((int) (header.getCenterLon() * 10000000)); + output.writeInt((int) (header.getCenterLat() * 10000000)); + } + + public static void serializeEntries(LittleEndianDataOutputStream output, List<Entry> entries) + throws IOException { + writeVarInt(output, entries.size()); + long lastId = 0; + for (Entry entry : entries) { + writeVarInt(output, entry.getTileId() - lastId); + lastId = entry.getTileId(); + } + for (Entry entry : entries) { + writeVarInt(output, entry.getRunLength()); + } + for (Entry entry : entries) { + writeVarInt(output, entry.getLength()); + } + for (int i = 0; i < entries.size(); i++) { + Entry entry = entries.get(i); + if (i > 0 + && entry.getOffset() == entries.get(i - 1).getOffset() + entries.get(i - 1).getLength()) { + writeVarInt(output, 0); + } else { + writeVarInt(output, entry.getOffset() + 1); + } + } + } + + public static List<Entry> deserializeEntries(LittleEndianDataInputStream buffer) + throws IOException { + long numEntries = readVarInt(buffer); + List<Entry> entries = new ArrayList<>((int) numEntries); + long lastId = 0; + for (int i = 0; i < numEntries; i++) { + long value = readVarInt(buffer); + lastId = lastId + value; + Entry entry = new Entry(); + entry.setTileId(lastId); + entries.add(entry); + } + for (int i = 0; i < numEntries; i++) { + long value = readVarInt(buffer); + entries.get(i).setRunLength(value); + } + for (int i = 0; i < numEntries; i++) { Review Comment: ## Comparison of narrow type with wide type in loop condition Comparison between [expression](1) of type int and [expression](2) of wider type long. [Show more details](https://github.com/apache/incubator-baremaps/security/code-scanning/825) ########## baremaps-core/src/test/java/org/apache/baremaps/tilestore/pmtiles/PMTilesTest.java: ########## @@ -0,0 +1,290 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.baremaps.tilestore.pmtiles; + +import static org.junit.jupiter.api.Assertions.*; + +import com.google.common.io.LittleEndianDataInputStream; +import com.google.common.io.LittleEndianDataOutputStream; +import com.google.common.math.LongMath; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.nio.channels.Channels; +import java.nio.channels.FileChannel; +import java.util.ArrayList; +import java.util.List; +import java.util.Random; +import org.apache.baremaps.testing.TestFiles; +import org.junit.jupiter.api.Test; + +class PMTilesTest { + + @Test + void decodeVarInt() throws IOException { + var b = new LittleEndianDataInputStream(new ByteArrayInputStream(new byte[] { + (byte) 0, (byte) 1, + (byte) 127, (byte) 0xe5, + (byte) 0x8e, (byte) 0x26 + })); + assertEquals(PMTiles.readVarInt(b), 0); + assertEquals(PMTiles.readVarInt(b), 1); + assertEquals(PMTiles.readVarInt(b), 127); + assertEquals(PMTiles.readVarInt(b), 624485); + b = new LittleEndianDataInputStream(new ByteArrayInputStream(new byte[] { + (byte) 0xff, (byte) 0xff, + (byte) 0xff, (byte) 0xff, + (byte) 0xff, (byte) 0xff, + (byte) 0xff, (byte) 0x0f, + })); + assertEquals(PMTiles.readVarInt(b), 9007199254740991L); + } + + @Test + void encodeVarInt() throws IOException { + for (long i = 0; i < 1000; i++) { + var array = new ByteArrayOutputStream(); + var output = new LittleEndianDataOutputStream(array); + PMTiles.writeVarInt(output, i); + var input = new LittleEndianDataInputStream(new ByteArrayInputStream(array.toByteArray())); + assertEquals(i, PMTiles.readVarInt(input)); + } + for (long i = Long.MAX_VALUE - 1000; i < Long.MAX_VALUE; i++) { + var array = new ByteArrayOutputStream(); + var output = new LittleEndianDataOutputStream(array); + PMTiles.writeVarInt(output, i); + var input = new LittleEndianDataInputStream(new ByteArrayInputStream(array.toByteArray())); + assertEquals(i, PMTiles.readVarInt(input)); + } + } + + @Test + void zxyToTileId() { + assertEquals(PMTiles.zxyToTileId(0, 0, 0), 0); + assertEquals(PMTiles.zxyToTileId(1, 0, 0), 1); + assertEquals(PMTiles.zxyToTileId(1, 0, 1), 2); + assertEquals(PMTiles.zxyToTileId(1, 1, 1), 3); + assertEquals(PMTiles.zxyToTileId(1, 1, 0), 4); + assertEquals(PMTiles.zxyToTileId(2, 0, 0), 5); + } + + @Test + void tileIdToZxy() { + assertArrayEquals(PMTiles.tileIdToZxy(0), new long[] {0, 0, 0}); + assertArrayEquals(PMTiles.tileIdToZxy(1), new long[] {1, 0, 0}); + assertArrayEquals(PMTiles.tileIdToZxy(2), new long[] {1, 0, 1}); + assertArrayEquals(PMTiles.tileIdToZxy(3), new long[] {1, 1, 1}); + assertArrayEquals(PMTiles.tileIdToZxy(4), new long[] {1, 1, 0}); + assertArrayEquals(PMTiles.tileIdToZxy(5), new long[] {2, 0, 0}); + } + + @Test + void aLotOfTiles() { + for (int z = 0; z < 9; z++) { + for (long x = 0; x < 1 << z; x++) { + for (long y = 0; y < 1 << z; y++) { + var result = PMTiles.tileIdToZxy(PMTiles.zxyToTileId(z, x, y)); + if (result[0] != z || result[1] != x || result[2] != y) { + fail("roundtrip failed"); + } + } + } + } + } + + @Test + void tileExtremes() { + for (var z = 0; z < 27; z++) { + var dim = LongMath.pow(2, z) - 1; + var tl = PMTiles.tileIdToZxy(PMTiles.zxyToTileId(z, 0, 0)); + assertArrayEquals(new long[] {z, 0, 0}, tl); + var tr = PMTiles.tileIdToZxy(PMTiles.zxyToTileId(z, dim, 0)); + assertArrayEquals(new long[] {z, dim, 0}, tr); + var bl = PMTiles.tileIdToZxy(PMTiles.zxyToTileId(z, 0, dim)); + assertArrayEquals(new long[] {z, 0, dim}, bl); + var br = PMTiles.tileIdToZxy(PMTiles.zxyToTileId(z, dim, dim)); + assertArrayEquals(new long[] {z, dim, dim}, br); + } + } + + @Test + void invalidTiles() { + assertThrows(RuntimeException.class, () -> PMTiles.tileIdToZxy(9007199254740991L)); + assertThrows(RuntimeException.class, () -> PMTiles.zxyToTileId(27, 0, 0)); + assertThrows(RuntimeException.class, () -> PMTiles.zxyToTileId(0, 1, 1)); + } + + @Test + void decodeHeader() throws IOException { + var file = TestFiles.resolve("pmtiles/test_fixture_1.pmtiles"); + try (var channel = FileChannel.open(file)) { + var input = new LittleEndianDataInputStream(Channels.newInputStream(channel)); Review Comment: ## Potential input resource leak This LittleEndianDataInputStream is not always closed on method exit. [Show more details](https://github.com/apache/incubator-baremaps/security/code-scanning/822) ########## baremaps-core/src/main/java/org/apache/baremaps/tilestore/pmtiles/PMTiles.java: ########## @@ -0,0 +1,391 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.baremaps.tilestore.pmtiles; + +import com.google.common.io.LittleEndianDataInputStream; +import com.google.common.io.LittleEndianDataOutputStream; +import com.google.common.math.LongMath; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +public class PMTiles { + + public static long toNum(long low, long high) { + return high * 0x100000000L + low; + } + + public static long readVarIntRemainder(LittleEndianDataInputStream input, long l) + throws IOException { + long h, b; + b = input.readByte() & 0xff; + h = (b & 0x70) >> 4; + if (b < 0x80) { + return toNum(l, h); + } + b = input.readByte() & 0xff; + h |= (b & 0x7f) << 3; + if (b < 0x80) { + return toNum(l, h); + } + b = input.readByte() & 0xff; + h |= (b & 0x7f) << 10; + if (b < 0x80) { + return toNum(l, h); + } + b = input.readByte() & 0xff; + h |= (b & 0x7f) << 17; + if (b < 0x80) { + return toNum(l, h); + } + b = input.readByte() & 0xff; + h |= (b & 0x7f) << 24; + if (b < 0x80) { + return toNum(l, h); + } + b = input.readByte() & 0xff; + h |= (b & 0x01) << 31; + if (b < 0x80) { + return toNum(l, h); + } + throw new RuntimeException("Expected varint not more than 10 bytes"); + } + + public static int writeVarInt(LittleEndianDataOutputStream output, long value) + throws IOException { + int n = 1; + while (value >= 0x80) { + output.writeByte((byte) (value | 0x80)); + value >>>= 7; + n++; + } + output.writeByte((byte) value); + return n; + } + + public static long readVarInt(LittleEndianDataInputStream input) throws IOException { + long val, b; + b = input.readByte() & 0xff; + val = b & 0x7f; + if (b < 0x80) { + return val; + } + b = input.readByte() & 0xff; + val |= (b & 0x7f) << 7; + if (b < 0x80) { + return val; + } + b = input.readByte() & 0xff; + val |= (b & 0x7f) << 14; + if (b < 0x80) { + return val; + } + b = input.readByte() & 0xff; + val |= (b & 0x7f) << 21; + if (b < 0x80) { + return val; + } + val |= (b & 0x0f) << 28; + return readVarIntRemainder(input, val); + } + + public static void rotate(long n, long[] xy, long rx, long ry) { + if (ry == 0) { + if (rx == 1) { + xy[0] = n - 1 - xy[0]; + xy[1] = n - 1 - xy[1]; + } + long t = xy[0]; + xy[0] = xy[1]; + xy[1] = t; + } + } + + public static long[] idOnLevel(int z, long pos) { + long n = LongMath.pow(2, z); + long rx, ry, t = pos; + long[] xy = new long[] {0, 0}; + long s = 1; + while (s < n) { + rx = 1 & (t / 2); + ry = 1 & (t ^ rx); + rotate(s, xy, rx, ry); + xy[0] += s * rx; + xy[1] += s * ry; + t = t / 4; + s *= 2; + } + return new long[] {z, xy[0], xy[1]}; + } + + private static long[] tzValues = new long[] { + 0, 1, 5, 21, 85, 341, 1365, 5461, 21845, 87381, 349525, 1398101, 5592405, + 22369621, 89478485, 357913941, 1431655765, 5726623061L, 22906492245L, + 91625968981L, 366503875925L, 1466015503701L, 5864062014805L, 23456248059221L, + 93824992236885L, 375299968947541L, 1501199875790165L, + }; + + public static long zxyToTileId(int z, long x, long y) { + if (z > 26) { + throw new RuntimeException("Tile zoom level exceeds max safe number limit (26)"); + } + if (x > Math.pow(2, z) - 1 || y > Math.pow(2, z) - 1) { + throw new RuntimeException("tile x/y outside zoom level bounds"); + } + long acc = tzValues[z]; + long n = LongMath.pow(2, z); + long rx = 0; + long ry = 0; + long d = 0; + long[] xy = new long[] {x, y}; + long s = n / 2; + while (s > 0) { + rx = (xy[0] & s) > 0 ? 1 : 0; + ry = (xy[1] & s) > 0 ? 1 : 0; + d += s * s * ((3 * rx) ^ ry); + rotate(s, xy, rx, ry); + s = s / 2; + } + return acc + d; + } + + public static long[] tileIdToZxy(long i) { + long acc = 0; + for (int z = 0; z < 27; z++) { + long numTiles = (0x1L << z) * (0x1L << z); + if (acc + numTiles > i) { + return idOnLevel(z, i - acc); + } + acc += numTiles; + } + throw new RuntimeException("Tile zoom level exceeds max safe number limit (26)"); + } + + private static final int HEADER_SIZE_BYTES = 127; + + public static Header deserializeHeader(LittleEndianDataInputStream input) throws IOException { + input.skipBytes(7); + return new Header( + input.readByte(), + input.readLong(), + input.readLong(), + input.readLong(), + input.readLong(), + input.readLong(), + input.readLong(), + input.readLong(), + input.readLong(), + input.readLong(), + input.readLong(), + input.readLong(), + input.readByte() == 1, + Compression.values()[input.readByte()], + Compression.values()[input.readByte()], + TileType.values()[input.readByte()], + input.readByte(), + input.readByte(), + (double) input.readInt() / 10000000, + (double) input.readInt() / 10000000, + (double) input.readInt() / 10000000, + (double) input.readInt() / 10000000, + input.readByte(), + (double) input.readInt() / 10000000, + (double) input.readInt() / 10000000); + } + + public static void serializeHeader(LittleEndianDataOutputStream output, Header header) + throws IOException { + output.writeByte((byte) 0x50); + output.writeByte((byte) 0x4D); + output.writeByte((byte) 0x54); + output.writeByte((byte) 0x69); + output.writeByte((byte) 0x6C); + output.writeByte((byte) 0x65); + output.writeByte((byte) 0x73); + output.writeByte((byte) header.getSpecVersion()); + output.writeLong(header.getRootDirectoryOffset()); + output.writeLong(header.getRootDirectoryLength()); + output.writeLong(header.getJsonMetadataOffset()); + output.writeLong(header.getJsonMetadataLength()); + output.writeLong(header.getLeafDirectoryOffset()); + output.writeLong(header.getLeafDirectoryLength()); + output.writeLong(header.getTileDataOffset()); + output.writeLong(header.getTileDataLength()); + output.writeLong(header.getNumAddressedTiles()); + output.writeLong(header.getNumTileEntries()); + output.writeLong(header.getNumTileContents()); + output.writeByte((byte) (header.isClustered() ? 1 : 0)); + output.writeByte((byte) header.getInternalCompression().ordinal()); + output.writeByte((byte) header.getTileCompression().ordinal()); + output.writeByte((byte) header.getTileType().ordinal()); + output.writeByte((byte) header.getMinZoom()); + output.writeByte((byte) header.getMaxZoom()); + output.writeInt((int) (header.getMinLon() * 10000000)); + output.writeInt((int) (header.getMinLat() * 10000000)); + output.writeInt((int) (header.getMaxLon() * 10000000)); + output.writeInt((int) (header.getMaxLat() * 10000000)); + output.writeByte((byte) header.getCenterZoom()); + output.writeInt((int) (header.getCenterLon() * 10000000)); + output.writeInt((int) (header.getCenterLat() * 10000000)); + } + + public static void serializeEntries(LittleEndianDataOutputStream output, List<Entry> entries) + throws IOException { + writeVarInt(output, entries.size()); + long lastId = 0; + for (Entry entry : entries) { + writeVarInt(output, entry.getTileId() - lastId); + lastId = entry.getTileId(); + } + for (Entry entry : entries) { + writeVarInt(output, entry.getRunLength()); + } + for (Entry entry : entries) { + writeVarInt(output, entry.getLength()); + } + for (int i = 0; i < entries.size(); i++) { + Entry entry = entries.get(i); + if (i > 0 + && entry.getOffset() == entries.get(i - 1).getOffset() + entries.get(i - 1).getLength()) { + writeVarInt(output, 0); + } else { + writeVarInt(output, entry.getOffset() + 1); + } + } + } + + public static List<Entry> deserializeEntries(LittleEndianDataInputStream buffer) + throws IOException { + long numEntries = readVarInt(buffer); + List<Entry> entries = new ArrayList<>((int) numEntries); + long lastId = 0; + for (int i = 0; i < numEntries; i++) { + long value = readVarInt(buffer); + lastId = lastId + value; + Entry entry = new Entry(); + entry.setTileId(lastId); + entries.add(entry); + } + for (int i = 0; i < numEntries; i++) { + long value = readVarInt(buffer); + entries.get(i).setRunLength(value); + } + for (int i = 0; i < numEntries; i++) { + long value = readVarInt(buffer); + entries.get(i).setLength(value); + } + for (int i = 0; i < numEntries; i++) { Review Comment: ## Comparison of narrow type with wide type in loop condition Comparison between [expression](1) of type int and [expression](2) of wider type long. [Show more details](https://github.com/apache/incubator-baremaps/security/code-scanning/826) ########## baremaps-core/src/test/java/org/apache/baremaps/tilestore/pmtiles/PMTilesTest.java: ########## @@ -0,0 +1,290 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.baremaps.tilestore.pmtiles; + +import static org.junit.jupiter.api.Assertions.*; + +import com.google.common.io.LittleEndianDataInputStream; +import com.google.common.io.LittleEndianDataOutputStream; +import com.google.common.math.LongMath; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.nio.channels.Channels; +import java.nio.channels.FileChannel; +import java.util.ArrayList; +import java.util.List; +import java.util.Random; +import org.apache.baremaps.testing.TestFiles; +import org.junit.jupiter.api.Test; + +class PMTilesTest { + + @Test + void decodeVarInt() throws IOException { + var b = new LittleEndianDataInputStream(new ByteArrayInputStream(new byte[] { + (byte) 0, (byte) 1, + (byte) 127, (byte) 0xe5, + (byte) 0x8e, (byte) 0x26 + })); + assertEquals(PMTiles.readVarInt(b), 0); + assertEquals(PMTiles.readVarInt(b), 1); + assertEquals(PMTiles.readVarInt(b), 127); + assertEquals(PMTiles.readVarInt(b), 624485); + b = new LittleEndianDataInputStream(new ByteArrayInputStream(new byte[] { + (byte) 0xff, (byte) 0xff, + (byte) 0xff, (byte) 0xff, + (byte) 0xff, (byte) 0xff, + (byte) 0xff, (byte) 0x0f, + })); + assertEquals(PMTiles.readVarInt(b), 9007199254740991L); + } + + @Test + void encodeVarInt() throws IOException { + for (long i = 0; i < 1000; i++) { + var array = new ByteArrayOutputStream(); + var output = new LittleEndianDataOutputStream(array); + PMTiles.writeVarInt(output, i); + var input = new LittleEndianDataInputStream(new ByteArrayInputStream(array.toByteArray())); + assertEquals(i, PMTiles.readVarInt(input)); + } + for (long i = Long.MAX_VALUE - 1000; i < Long.MAX_VALUE; i++) { + var array = new ByteArrayOutputStream(); + var output = new LittleEndianDataOutputStream(array); + PMTiles.writeVarInt(output, i); + var input = new LittleEndianDataInputStream(new ByteArrayInputStream(array.toByteArray())); + assertEquals(i, PMTiles.readVarInt(input)); + } + } + + @Test + void zxyToTileId() { + assertEquals(PMTiles.zxyToTileId(0, 0, 0), 0); + assertEquals(PMTiles.zxyToTileId(1, 0, 0), 1); + assertEquals(PMTiles.zxyToTileId(1, 0, 1), 2); + assertEquals(PMTiles.zxyToTileId(1, 1, 1), 3); + assertEquals(PMTiles.zxyToTileId(1, 1, 0), 4); + assertEquals(PMTiles.zxyToTileId(2, 0, 0), 5); + } + + @Test + void tileIdToZxy() { + assertArrayEquals(PMTiles.tileIdToZxy(0), new long[] {0, 0, 0}); + assertArrayEquals(PMTiles.tileIdToZxy(1), new long[] {1, 0, 0}); + assertArrayEquals(PMTiles.tileIdToZxy(2), new long[] {1, 0, 1}); + assertArrayEquals(PMTiles.tileIdToZxy(3), new long[] {1, 1, 1}); + assertArrayEquals(PMTiles.tileIdToZxy(4), new long[] {1, 1, 0}); + assertArrayEquals(PMTiles.tileIdToZxy(5), new long[] {2, 0, 0}); + } + + @Test + void aLotOfTiles() { + for (int z = 0; z < 9; z++) { + for (long x = 0; x < 1 << z; x++) { + for (long y = 0; y < 1 << z; y++) { + var result = PMTiles.tileIdToZxy(PMTiles.zxyToTileId(z, x, y)); + if (result[0] != z || result[1] != x || result[2] != y) { + fail("roundtrip failed"); + } + } + } + } + } + + @Test + void tileExtremes() { + for (var z = 0; z < 27; z++) { + var dim = LongMath.pow(2, z) - 1; + var tl = PMTiles.tileIdToZxy(PMTiles.zxyToTileId(z, 0, 0)); + assertArrayEquals(new long[] {z, 0, 0}, tl); + var tr = PMTiles.tileIdToZxy(PMTiles.zxyToTileId(z, dim, 0)); + assertArrayEquals(new long[] {z, dim, 0}, tr); + var bl = PMTiles.tileIdToZxy(PMTiles.zxyToTileId(z, 0, dim)); + assertArrayEquals(new long[] {z, 0, dim}, bl); + var br = PMTiles.tileIdToZxy(PMTiles.zxyToTileId(z, dim, dim)); + assertArrayEquals(new long[] {z, dim, dim}, br); + } + } + + @Test + void invalidTiles() { + assertThrows(RuntimeException.class, () -> PMTiles.tileIdToZxy(9007199254740991L)); + assertThrows(RuntimeException.class, () -> PMTiles.zxyToTileId(27, 0, 0)); + assertThrows(RuntimeException.class, () -> PMTiles.zxyToTileId(0, 1, 1)); + } + + @Test + void decodeHeader() throws IOException { + var file = TestFiles.resolve("pmtiles/test_fixture_1.pmtiles"); + try (var channel = FileChannel.open(file)) { + var input = new LittleEndianDataInputStream(Channels.newInputStream(channel)); + var header = PMTiles.deserializeHeader(input); + assertEquals(header.getRootDirectoryOffset(), 127); + assertEquals(header.getRootDirectoryLength(), 25); + assertEquals(header.getJsonMetadataOffset(), 152); + assertEquals(header.getJsonMetadataLength(), 247); + assertEquals(header.getLeafDirectoryOffset(), 0); + assertEquals(header.getLeafDirectoryLength(), 0); + assertEquals(header.getTileDataOffset(), 399); + assertEquals(header.getTileDataLength(), 69); + assertEquals(header.getNumAddressedTiles(), 1); + assertEquals(header.getNumTileEntries(), 1); + assertEquals(header.getNumTileContents(), 1); + assertFalse(header.isClustered()); + assertEquals(header.getInternalCompression(), Compression.Gzip); + assertEquals(header.getTileCompression(), Compression.Gzip); + assertEquals(header.getTileType(), TileType.mvt); + assertEquals(header.getMinZoom(), 0); + assertEquals(header.getMaxZoom(), 0); + assertEquals(header.getMinLon(), 0); + assertEquals(header.getMinLat(), 0); + assertEquals(Math.round(header.getMaxLon()), 1); + assertEquals(Math.round(header.getMaxLat()), 1); + } + } + + @Test + void encodeHeader() throws IOException { + var etag = "1"; + var header = new Header( + 127, + 25, + 152, + 247, + 0, + 0, + 399, + 69, + 1, + 1, + 1, + 10, + false, + Compression.Gzip, + Compression.Gzip, + TileType.mvt, + 0, + 0, + 0, + 1, + 1, + 0, + 0, + 0, + 0); + + var array = new ByteArrayOutputStream(); + + var output = new LittleEndianDataOutputStream(array); + PMTiles.serializeHeader(output, header); + + var array2 = array.toByteArray(); Review Comment: ## Unread local variable Variable 'byte\[\] array2' is never read. [Show more details](https://github.com/apache/incubator-baremaps/security/code-scanning/819) ########## baremaps-core/src/main/java/org/apache/baremaps/tilestore/pmtiles/PMTilesWriter.java: ########## @@ -0,0 +1,208 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.baremaps.tilestore.pmtiles; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.hash.Hashing; +import com.google.common.io.LittleEndianDataOutputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.*; + +public class PMTilesWriter { + + private Path path; + + private Map<String, Object> metadata = new HashMap<>(); + + private List<Entry> entries; + + private Map<Long, Long> tileHashToOffset; + + private Long lastTileHash = null; + + private Path tilePath; + + private boolean clustered = true; + + private int minZoom = 0; + + private int maxZoom = 14; + + private double minLon = -180; + + private double minLat = -90; + + private double maxLon = 180; + + private double maxLat = 90; + + private int centerZoom = 3; + + private double centerLat = 0; + + private double centerLon = 0; + + public PMTilesWriter(Path path) throws IOException { + this(path, new ArrayList<>(), new HashMap<>()); + } + + public PMTilesWriter(Path path, List<Entry> entries, Map<Long, Long> tileHashToOffset) + throws IOException { + this.path = path; + this.entries = entries; + this.tileHashToOffset = tileHashToOffset; + this.tilePath = Files.createTempFile(path.getParent(), "tiles_", ".tmp"); + } + + public void setMetadata(Map<String, Object> metadata) { + this.metadata = metadata; + } + + public void setTile(int z, int x, int y, byte[] bytes) throws IOException { + // Write the tile + var tileId = PMTiles.zxyToTileId(z, x, y); + var tileLength = bytes.length; + Long tileHash = Hashing.farmHashFingerprint64().hashBytes(bytes).asLong(); Review Comment: ## Boxed variable is never null The variable 'tileHash' is only assigned values of primitive type and is never 'null', but it is declared with the boxed type 'Long'. [Show more details](https://github.com/apache/incubator-baremaps/security/code-scanning/817) ########## baremaps-core/src/main/java/org/apache/baremaps/tilestore/pmtiles/PMTiles.java: ########## @@ -0,0 +1,391 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.baremaps.tilestore.pmtiles; + +import com.google.common.io.LittleEndianDataInputStream; +import com.google.common.io.LittleEndianDataOutputStream; +import com.google.common.math.LongMath; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +public class PMTiles { + + public static long toNum(long low, long high) { + return high * 0x100000000L + low; + } + + public static long readVarIntRemainder(LittleEndianDataInputStream input, long l) + throws IOException { + long h, b; + b = input.readByte() & 0xff; + h = (b & 0x70) >> 4; + if (b < 0x80) { + return toNum(l, h); + } + b = input.readByte() & 0xff; + h |= (b & 0x7f) << 3; + if (b < 0x80) { + return toNum(l, h); + } + b = input.readByte() & 0xff; + h |= (b & 0x7f) << 10; + if (b < 0x80) { + return toNum(l, h); + } + b = input.readByte() & 0xff; + h |= (b & 0x7f) << 17; + if (b < 0x80) { + return toNum(l, h); + } + b = input.readByte() & 0xff; + h |= (b & 0x7f) << 24; + if (b < 0x80) { + return toNum(l, h); + } + b = input.readByte() & 0xff; + h |= (b & 0x01) << 31; + if (b < 0x80) { + return toNum(l, h); + } + throw new RuntimeException("Expected varint not more than 10 bytes"); + } + + public static int writeVarInt(LittleEndianDataOutputStream output, long value) + throws IOException { + int n = 1; + while (value >= 0x80) { + output.writeByte((byte) (value | 0x80)); + value >>>= 7; + n++; + } + output.writeByte((byte) value); + return n; + } + + public static long readVarInt(LittleEndianDataInputStream input) throws IOException { + long val, b; + b = input.readByte() & 0xff; + val = b & 0x7f; + if (b < 0x80) { + return val; + } + b = input.readByte() & 0xff; + val |= (b & 0x7f) << 7; + if (b < 0x80) { + return val; + } + b = input.readByte() & 0xff; + val |= (b & 0x7f) << 14; + if (b < 0x80) { + return val; + } + b = input.readByte() & 0xff; + val |= (b & 0x7f) << 21; + if (b < 0x80) { + return val; + } + val |= (b & 0x0f) << 28; + return readVarIntRemainder(input, val); + } + + public static void rotate(long n, long[] xy, long rx, long ry) { + if (ry == 0) { + if (rx == 1) { + xy[0] = n - 1 - xy[0]; + xy[1] = n - 1 - xy[1]; + } + long t = xy[0]; + xy[0] = xy[1]; + xy[1] = t; + } + } + + public static long[] idOnLevel(int z, long pos) { + long n = LongMath.pow(2, z); + long rx, ry, t = pos; + long[] xy = new long[] {0, 0}; + long s = 1; + while (s < n) { + rx = 1 & (t / 2); + ry = 1 & (t ^ rx); + rotate(s, xy, rx, ry); + xy[0] += s * rx; + xy[1] += s * ry; + t = t / 4; + s *= 2; + } + return new long[] {z, xy[0], xy[1]}; + } + + private static long[] tzValues = new long[] { + 0, 1, 5, 21, 85, 341, 1365, 5461, 21845, 87381, 349525, 1398101, 5592405, + 22369621, 89478485, 357913941, 1431655765, 5726623061L, 22906492245L, + 91625968981L, 366503875925L, 1466015503701L, 5864062014805L, 23456248059221L, + 93824992236885L, 375299968947541L, 1501199875790165L, + }; + + public static long zxyToTileId(int z, long x, long y) { + if (z > 26) { + throw new RuntimeException("Tile zoom level exceeds max safe number limit (26)"); + } + if (x > Math.pow(2, z) - 1 || y > Math.pow(2, z) - 1) { + throw new RuntimeException("tile x/y outside zoom level bounds"); + } + long acc = tzValues[z]; + long n = LongMath.pow(2, z); + long rx = 0; + long ry = 0; + long d = 0; + long[] xy = new long[] {x, y}; + long s = n / 2; + while (s > 0) { + rx = (xy[0] & s) > 0 ? 1 : 0; + ry = (xy[1] & s) > 0 ? 1 : 0; + d += s * s * ((3 * rx) ^ ry); + rotate(s, xy, rx, ry); + s = s / 2; + } + return acc + d; + } + + public static long[] tileIdToZxy(long i) { + long acc = 0; + for (int z = 0; z < 27; z++) { + long numTiles = (0x1L << z) * (0x1L << z); + if (acc + numTiles > i) { + return idOnLevel(z, i - acc); + } + acc += numTiles; + } + throw new RuntimeException("Tile zoom level exceeds max safe number limit (26)"); + } + + private static final int HEADER_SIZE_BYTES = 127; + + public static Header deserializeHeader(LittleEndianDataInputStream input) throws IOException { + input.skipBytes(7); + return new Header( + input.readByte(), + input.readLong(), + input.readLong(), + input.readLong(), + input.readLong(), + input.readLong(), + input.readLong(), + input.readLong(), + input.readLong(), + input.readLong(), + input.readLong(), + input.readLong(), + input.readByte() == 1, + Compression.values()[input.readByte()], + Compression.values()[input.readByte()], + TileType.values()[input.readByte()], + input.readByte(), + input.readByte(), + (double) input.readInt() / 10000000, + (double) input.readInt() / 10000000, + (double) input.readInt() / 10000000, + (double) input.readInt() / 10000000, + input.readByte(), + (double) input.readInt() / 10000000, + (double) input.readInt() / 10000000); + } + + public static void serializeHeader(LittleEndianDataOutputStream output, Header header) + throws IOException { + output.writeByte((byte) 0x50); + output.writeByte((byte) 0x4D); + output.writeByte((byte) 0x54); + output.writeByte((byte) 0x69); + output.writeByte((byte) 0x6C); + output.writeByte((byte) 0x65); + output.writeByte((byte) 0x73); + output.writeByte((byte) header.getSpecVersion()); + output.writeLong(header.getRootDirectoryOffset()); + output.writeLong(header.getRootDirectoryLength()); + output.writeLong(header.getJsonMetadataOffset()); + output.writeLong(header.getJsonMetadataLength()); + output.writeLong(header.getLeafDirectoryOffset()); + output.writeLong(header.getLeafDirectoryLength()); + output.writeLong(header.getTileDataOffset()); + output.writeLong(header.getTileDataLength()); + output.writeLong(header.getNumAddressedTiles()); + output.writeLong(header.getNumTileEntries()); + output.writeLong(header.getNumTileContents()); + output.writeByte((byte) (header.isClustered() ? 1 : 0)); + output.writeByte((byte) header.getInternalCompression().ordinal()); + output.writeByte((byte) header.getTileCompression().ordinal()); + output.writeByte((byte) header.getTileType().ordinal()); + output.writeByte((byte) header.getMinZoom()); + output.writeByte((byte) header.getMaxZoom()); + output.writeInt((int) (header.getMinLon() * 10000000)); + output.writeInt((int) (header.getMinLat() * 10000000)); + output.writeInt((int) (header.getMaxLon() * 10000000)); + output.writeInt((int) (header.getMaxLat() * 10000000)); + output.writeByte((byte) header.getCenterZoom()); + output.writeInt((int) (header.getCenterLon() * 10000000)); + output.writeInt((int) (header.getCenterLat() * 10000000)); + } + + public static void serializeEntries(LittleEndianDataOutputStream output, List<Entry> entries) + throws IOException { + writeVarInt(output, entries.size()); + long lastId = 0; + for (Entry entry : entries) { + writeVarInt(output, entry.getTileId() - lastId); + lastId = entry.getTileId(); + } + for (Entry entry : entries) { + writeVarInt(output, entry.getRunLength()); + } + for (Entry entry : entries) { + writeVarInt(output, entry.getLength()); + } + for (int i = 0; i < entries.size(); i++) { + Entry entry = entries.get(i); + if (i > 0 + && entry.getOffset() == entries.get(i - 1).getOffset() + entries.get(i - 1).getLength()) { + writeVarInt(output, 0); + } else { + writeVarInt(output, entry.getOffset() + 1); + } + } + } + + public static List<Entry> deserializeEntries(LittleEndianDataInputStream buffer) + throws IOException { + long numEntries = readVarInt(buffer); + List<Entry> entries = new ArrayList<>((int) numEntries); + long lastId = 0; + for (int i = 0; i < numEntries; i++) { + long value = readVarInt(buffer); + lastId = lastId + value; + Entry entry = new Entry(); + entry.setTileId(lastId); + entries.add(entry); + } + for (int i = 0; i < numEntries; i++) { Review Comment: ## Comparison of narrow type with wide type in loop condition Comparison between [expression](1) of type int and [expression](2) of wider type long. [Show more details](https://github.com/apache/incubator-baremaps/security/code-scanning/824) ########## baremaps-core/src/main/java/org/apache/baremaps/tilestore/pmtiles/PMTilesReader.java: ########## @@ -0,0 +1,96 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.baremaps.tilestore.pmtiles; + +import com.google.common.io.LittleEndianDataInputStream; +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.channels.FileChannel; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; + +public class PMTilesReader { + + private final Path path; + + private Header header; + + private List<Entry> rootEntries; + + public PMTilesReader(Path path) { + this.path = path; + } + + public Header getHeader() { + if (header == null) { + try (var inputStream = new LittleEndianDataInputStream(Files.newInputStream(path))) { + header = PMTiles.deserializeHeader(inputStream); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + return header; + } + + public List<Entry> getRootDirectory() { + if (rootEntries == null) { + var header = getHeader(); + rootEntries = + getDirectory(header.getRootDirectoryOffset(), (int) header.getRootDirectoryLength()); + } + return rootEntries; + } + + public List<Entry> getDirectory(long offset, int length) { Review Comment: ## Useless parameter The parameter 'length' is never used. [Show more details](https://github.com/apache/incubator-baremaps/security/code-scanning/821) ########## baremaps-core/src/test/java/org/apache/baremaps/tilestore/pmtiles/PMTilesTest.java: ########## @@ -0,0 +1,290 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.baremaps.tilestore.pmtiles; + +import static org.junit.jupiter.api.Assertions.*; + +import com.google.common.io.LittleEndianDataInputStream; +import com.google.common.io.LittleEndianDataOutputStream; +import com.google.common.math.LongMath; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.nio.channels.Channels; +import java.nio.channels.FileChannel; +import java.util.ArrayList; +import java.util.List; +import java.util.Random; +import org.apache.baremaps.testing.TestFiles; +import org.junit.jupiter.api.Test; + +class PMTilesTest { + + @Test + void decodeVarInt() throws IOException { + var b = new LittleEndianDataInputStream(new ByteArrayInputStream(new byte[] { + (byte) 0, (byte) 1, + (byte) 127, (byte) 0xe5, + (byte) 0x8e, (byte) 0x26 + })); + assertEquals(PMTiles.readVarInt(b), 0); + assertEquals(PMTiles.readVarInt(b), 1); + assertEquals(PMTiles.readVarInt(b), 127); + assertEquals(PMTiles.readVarInt(b), 624485); + b = new LittleEndianDataInputStream(new ByteArrayInputStream(new byte[] { + (byte) 0xff, (byte) 0xff, + (byte) 0xff, (byte) 0xff, + (byte) 0xff, (byte) 0xff, + (byte) 0xff, (byte) 0x0f, + })); + assertEquals(PMTiles.readVarInt(b), 9007199254740991L); + } + + @Test + void encodeVarInt() throws IOException { + for (long i = 0; i < 1000; i++) { + var array = new ByteArrayOutputStream(); + var output = new LittleEndianDataOutputStream(array); + PMTiles.writeVarInt(output, i); + var input = new LittleEndianDataInputStream(new ByteArrayInputStream(array.toByteArray())); + assertEquals(i, PMTiles.readVarInt(input)); + } + for (long i = Long.MAX_VALUE - 1000; i < Long.MAX_VALUE; i++) { + var array = new ByteArrayOutputStream(); + var output = new LittleEndianDataOutputStream(array); + PMTiles.writeVarInt(output, i); + var input = new LittleEndianDataInputStream(new ByteArrayInputStream(array.toByteArray())); + assertEquals(i, PMTiles.readVarInt(input)); + } + } + + @Test + void zxyToTileId() { + assertEquals(PMTiles.zxyToTileId(0, 0, 0), 0); + assertEquals(PMTiles.zxyToTileId(1, 0, 0), 1); + assertEquals(PMTiles.zxyToTileId(1, 0, 1), 2); + assertEquals(PMTiles.zxyToTileId(1, 1, 1), 3); + assertEquals(PMTiles.zxyToTileId(1, 1, 0), 4); + assertEquals(PMTiles.zxyToTileId(2, 0, 0), 5); + } + + @Test + void tileIdToZxy() { + assertArrayEquals(PMTiles.tileIdToZxy(0), new long[] {0, 0, 0}); + assertArrayEquals(PMTiles.tileIdToZxy(1), new long[] {1, 0, 0}); + assertArrayEquals(PMTiles.tileIdToZxy(2), new long[] {1, 0, 1}); + assertArrayEquals(PMTiles.tileIdToZxy(3), new long[] {1, 1, 1}); + assertArrayEquals(PMTiles.tileIdToZxy(4), new long[] {1, 1, 0}); + assertArrayEquals(PMTiles.tileIdToZxy(5), new long[] {2, 0, 0}); + } + + @Test + void aLotOfTiles() { + for (int z = 0; z < 9; z++) { + for (long x = 0; x < 1 << z; x++) { + for (long y = 0; y < 1 << z; y++) { + var result = PMTiles.tileIdToZxy(PMTiles.zxyToTileId(z, x, y)); + if (result[0] != z || result[1] != x || result[2] != y) { + fail("roundtrip failed"); + } + } + } + } + } + + @Test + void tileExtremes() { + for (var z = 0; z < 27; z++) { + var dim = LongMath.pow(2, z) - 1; + var tl = PMTiles.tileIdToZxy(PMTiles.zxyToTileId(z, 0, 0)); + assertArrayEquals(new long[] {z, 0, 0}, tl); + var tr = PMTiles.tileIdToZxy(PMTiles.zxyToTileId(z, dim, 0)); + assertArrayEquals(new long[] {z, dim, 0}, tr); + var bl = PMTiles.tileIdToZxy(PMTiles.zxyToTileId(z, 0, dim)); + assertArrayEquals(new long[] {z, 0, dim}, bl); + var br = PMTiles.tileIdToZxy(PMTiles.zxyToTileId(z, dim, dim)); + assertArrayEquals(new long[] {z, dim, dim}, br); + } + } + + @Test + void invalidTiles() { + assertThrows(RuntimeException.class, () -> PMTiles.tileIdToZxy(9007199254740991L)); + assertThrows(RuntimeException.class, () -> PMTiles.zxyToTileId(27, 0, 0)); + assertThrows(RuntimeException.class, () -> PMTiles.zxyToTileId(0, 1, 1)); + } + + @Test + void decodeHeader() throws IOException { + var file = TestFiles.resolve("pmtiles/test_fixture_1.pmtiles"); + try (var channel = FileChannel.open(file)) { + var input = new LittleEndianDataInputStream(Channels.newInputStream(channel)); + var header = PMTiles.deserializeHeader(input); + assertEquals(header.getRootDirectoryOffset(), 127); + assertEquals(header.getRootDirectoryLength(), 25); + assertEquals(header.getJsonMetadataOffset(), 152); + assertEquals(header.getJsonMetadataLength(), 247); + assertEquals(header.getLeafDirectoryOffset(), 0); + assertEquals(header.getLeafDirectoryLength(), 0); + assertEquals(header.getTileDataOffset(), 399); + assertEquals(header.getTileDataLength(), 69); + assertEquals(header.getNumAddressedTiles(), 1); + assertEquals(header.getNumTileEntries(), 1); + assertEquals(header.getNumTileContents(), 1); + assertFalse(header.isClustered()); + assertEquals(header.getInternalCompression(), Compression.Gzip); + assertEquals(header.getTileCompression(), Compression.Gzip); + assertEquals(header.getTileType(), TileType.mvt); + assertEquals(header.getMinZoom(), 0); + assertEquals(header.getMaxZoom(), 0); + assertEquals(header.getMinLon(), 0); + assertEquals(header.getMinLat(), 0); + assertEquals(Math.round(header.getMaxLon()), 1); + assertEquals(Math.round(header.getMaxLat()), 1); + } + } + + @Test + void encodeHeader() throws IOException { + var etag = "1"; Review Comment: ## Unread local variable Variable 'String etag' is never read. [Show more details](https://github.com/apache/incubator-baremaps/security/code-scanning/818) ########## baremaps-core/src/main/java/org/apache/baremaps/tilestore/pmtiles/PMTiles.java: ########## @@ -0,0 +1,391 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.baremaps.tilestore.pmtiles; + +import com.google.common.io.LittleEndianDataInputStream; +import com.google.common.io.LittleEndianDataOutputStream; +import com.google.common.math.LongMath; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +public class PMTiles { + + public static long toNum(long low, long high) { + return high * 0x100000000L + low; + } + + public static long readVarIntRemainder(LittleEndianDataInputStream input, long l) + throws IOException { + long h, b; + b = input.readByte() & 0xff; + h = (b & 0x70) >> 4; + if (b < 0x80) { + return toNum(l, h); + } + b = input.readByte() & 0xff; + h |= (b & 0x7f) << 3; + if (b < 0x80) { + return toNum(l, h); + } + b = input.readByte() & 0xff; + h |= (b & 0x7f) << 10; + if (b < 0x80) { + return toNum(l, h); + } + b = input.readByte() & 0xff; + h |= (b & 0x7f) << 17; + if (b < 0x80) { + return toNum(l, h); + } + b = input.readByte() & 0xff; + h |= (b & 0x7f) << 24; + if (b < 0x80) { + return toNum(l, h); + } + b = input.readByte() & 0xff; + h |= (b & 0x01) << 31; + if (b < 0x80) { + return toNum(l, h); + } + throw new RuntimeException("Expected varint not more than 10 bytes"); + } + + public static int writeVarInt(LittleEndianDataOutputStream output, long value) + throws IOException { + int n = 1; + while (value >= 0x80) { + output.writeByte((byte) (value | 0x80)); + value >>>= 7; + n++; + } + output.writeByte((byte) value); + return n; + } + + public static long readVarInt(LittleEndianDataInputStream input) throws IOException { + long val, b; + b = input.readByte() & 0xff; + val = b & 0x7f; + if (b < 0x80) { + return val; + } + b = input.readByte() & 0xff; + val |= (b & 0x7f) << 7; + if (b < 0x80) { + return val; + } + b = input.readByte() & 0xff; + val |= (b & 0x7f) << 14; + if (b < 0x80) { + return val; + } + b = input.readByte() & 0xff; + val |= (b & 0x7f) << 21; + if (b < 0x80) { + return val; + } + val |= (b & 0x0f) << 28; + return readVarIntRemainder(input, val); + } + + public static void rotate(long n, long[] xy, long rx, long ry) { + if (ry == 0) { + if (rx == 1) { + xy[0] = n - 1 - xy[0]; + xy[1] = n - 1 - xy[1]; + } + long t = xy[0]; + xy[0] = xy[1]; + xy[1] = t; + } + } + + public static long[] idOnLevel(int z, long pos) { + long n = LongMath.pow(2, z); + long rx, ry, t = pos; + long[] xy = new long[] {0, 0}; + long s = 1; + while (s < n) { + rx = 1 & (t / 2); + ry = 1 & (t ^ rx); + rotate(s, xy, rx, ry); + xy[0] += s * rx; + xy[1] += s * ry; + t = t / 4; + s *= 2; + } + return new long[] {z, xy[0], xy[1]}; + } + + private static long[] tzValues = new long[] { + 0, 1, 5, 21, 85, 341, 1365, 5461, 21845, 87381, 349525, 1398101, 5592405, + 22369621, 89478485, 357913941, 1431655765, 5726623061L, 22906492245L, + 91625968981L, 366503875925L, 1466015503701L, 5864062014805L, 23456248059221L, + 93824992236885L, 375299968947541L, 1501199875790165L, + }; + + public static long zxyToTileId(int z, long x, long y) { + if (z > 26) { + throw new RuntimeException("Tile zoom level exceeds max safe number limit (26)"); + } + if (x > Math.pow(2, z) - 1 || y > Math.pow(2, z) - 1) { + throw new RuntimeException("tile x/y outside zoom level bounds"); + } + long acc = tzValues[z]; + long n = LongMath.pow(2, z); + long rx = 0; + long ry = 0; + long d = 0; + long[] xy = new long[] {x, y}; + long s = n / 2; + while (s > 0) { + rx = (xy[0] & s) > 0 ? 1 : 0; + ry = (xy[1] & s) > 0 ? 1 : 0; + d += s * s * ((3 * rx) ^ ry); + rotate(s, xy, rx, ry); + s = s / 2; + } + return acc + d; + } + + public static long[] tileIdToZxy(long i) { + long acc = 0; + for (int z = 0; z < 27; z++) { + long numTiles = (0x1L << z) * (0x1L << z); + if (acc + numTiles > i) { + return idOnLevel(z, i - acc); + } + acc += numTiles; + } + throw new RuntimeException("Tile zoom level exceeds max safe number limit (26)"); + } + + private static final int HEADER_SIZE_BYTES = 127; + + public static Header deserializeHeader(LittleEndianDataInputStream input) throws IOException { + input.skipBytes(7); + return new Header( + input.readByte(), + input.readLong(), + input.readLong(), + input.readLong(), + input.readLong(), + input.readLong(), + input.readLong(), + input.readLong(), + input.readLong(), + input.readLong(), + input.readLong(), + input.readLong(), + input.readByte() == 1, + Compression.values()[input.readByte()], + Compression.values()[input.readByte()], + TileType.values()[input.readByte()], + input.readByte(), + input.readByte(), + (double) input.readInt() / 10000000, + (double) input.readInt() / 10000000, + (double) input.readInt() / 10000000, + (double) input.readInt() / 10000000, + input.readByte(), + (double) input.readInt() / 10000000, + (double) input.readInt() / 10000000); + } + + public static void serializeHeader(LittleEndianDataOutputStream output, Header header) + throws IOException { + output.writeByte((byte) 0x50); + output.writeByte((byte) 0x4D); + output.writeByte((byte) 0x54); + output.writeByte((byte) 0x69); + output.writeByte((byte) 0x6C); + output.writeByte((byte) 0x65); + output.writeByte((byte) 0x73); + output.writeByte((byte) header.getSpecVersion()); + output.writeLong(header.getRootDirectoryOffset()); + output.writeLong(header.getRootDirectoryLength()); + output.writeLong(header.getJsonMetadataOffset()); + output.writeLong(header.getJsonMetadataLength()); + output.writeLong(header.getLeafDirectoryOffset()); + output.writeLong(header.getLeafDirectoryLength()); + output.writeLong(header.getTileDataOffset()); + output.writeLong(header.getTileDataLength()); + output.writeLong(header.getNumAddressedTiles()); + output.writeLong(header.getNumTileEntries()); + output.writeLong(header.getNumTileContents()); + output.writeByte((byte) (header.isClustered() ? 1 : 0)); + output.writeByte((byte) header.getInternalCompression().ordinal()); + output.writeByte((byte) header.getTileCompression().ordinal()); + output.writeByte((byte) header.getTileType().ordinal()); + output.writeByte((byte) header.getMinZoom()); + output.writeByte((byte) header.getMaxZoom()); + output.writeInt((int) (header.getMinLon() * 10000000)); + output.writeInt((int) (header.getMinLat() * 10000000)); + output.writeInt((int) (header.getMaxLon() * 10000000)); + output.writeInt((int) (header.getMaxLat() * 10000000)); + output.writeByte((byte) header.getCenterZoom()); + output.writeInt((int) (header.getCenterLon() * 10000000)); + output.writeInt((int) (header.getCenterLat() * 10000000)); + } + + public static void serializeEntries(LittleEndianDataOutputStream output, List<Entry> entries) + throws IOException { + writeVarInt(output, entries.size()); + long lastId = 0; + for (Entry entry : entries) { + writeVarInt(output, entry.getTileId() - lastId); + lastId = entry.getTileId(); + } + for (Entry entry : entries) { + writeVarInt(output, entry.getRunLength()); + } + for (Entry entry : entries) { + writeVarInt(output, entry.getLength()); + } + for (int i = 0; i < entries.size(); i++) { + Entry entry = entries.get(i); + if (i > 0 + && entry.getOffset() == entries.get(i - 1).getOffset() + entries.get(i - 1).getLength()) { + writeVarInt(output, 0); + } else { + writeVarInt(output, entry.getOffset() + 1); + } + } + } + + public static List<Entry> deserializeEntries(LittleEndianDataInputStream buffer) + throws IOException { + long numEntries = readVarInt(buffer); + List<Entry> entries = new ArrayList<>((int) numEntries); + long lastId = 0; + for (int i = 0; i < numEntries; i++) { Review Comment: ## Comparison of narrow type with wide type in loop condition Comparison between [expression](1) of type int and [expression](2) of wider type long. [Show more details](https://github.com/apache/incubator-baremaps/security/code-scanning/823) -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
