This is an automated email from the ASF dual-hosted git repository. bchapuis pushed a commit to branch calcite in repository https://gitbox.apache.org/repos/asf/incubator-baremaps.git
commit b138761b527f6439ab3be5caa12293baebfcaf17 Author: Bertil Chapuis <[email protected]> AuthorDate: Tue Jun 20 11:58:49 2023 +0200 Prepare the Table and Store abstractions and remove collection tasks --- baremaps-core/pom.xml | 1 - .../java/org/apache/baremaps/calcite/Calcite.java | 116 ++++++++++++++++ .../collection/algorithm/ExternalMergeSort.java | 4 +- .../baremaps/collection/type/BooleanDataType.java | 38 ++++++ .../collection/type/geometry/GeometryDataType.java | 2 +- .../collection/type/geometry/LonLatDataType.java | 67 +++++++++ .../collection/type/geometry/WKBDataType.java | 54 ++++++++ .../main/java/org/apache/baremaps/storage/Row.java | 16 +++ .../org/apache/baremaps/storage/RowDataType.java | 85 ++++++++++++ .../java/org/apache/baremaps/storage/RowImpl.java | 13 ++ .../org/apache/baremaps/storage/SchemaImpl.java | 29 +++- .../java/org/apache/baremaps/workflow/Task.java | 2 - .../workflow/tasks/CreateEntityCollection.java | 93 ------------- .../org/apache/baremaps/workflow/tasks/Entity.java | 83 ------------ .../baremaps/workflow/tasks/EntityDataType.java | 66 --------- .../workflow/tasks/ImportOpenStreetMap.java | 2 +- .../workflow/tasks/TransformEntityCollection.java | 150 --------------------- .../baremaps/collection/type/DataTypeProvider.java | 9 ++ .../baremaps/tilestore/file/FileTileStoreTest.java | 2 +- .../baremaps/vectortile/ExpressionsTest.java | 33 +++-- .../CreateAndTransformEntityCollectionTest.java | 48 ------- 21 files changed, 455 insertions(+), 458 deletions(-) diff --git a/baremaps-core/pom.xml b/baremaps-core/pom.xml index 3b370c4b..b23bb1f3 100644 --- a/baremaps-core/pom.xml +++ b/baremaps-core/pom.xml @@ -84,7 +84,6 @@ <dependency> <groupId>org.apache.calcite</groupId> <artifactId>calcite-core</artifactId> - <version>${version.lib.calcite}</version> </dependency> <dependency> <groupId>org.apache.commons</groupId> diff --git a/baremaps-core/src/main/java/org/apache/baremaps/calcite/Calcite.java b/baremaps-core/src/main/java/org/apache/baremaps/calcite/Calcite.java new file mode 100644 index 00000000..300f9af4 --- /dev/null +++ b/baremaps-core/src/main/java/org/apache/baremaps/calcite/Calcite.java @@ -0,0 +1,116 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + */ + +package org.apache.baremaps.calcite; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.Arrays; +import java.util.List; +import java.util.Properties; +import org.apache.calcite.DataContext; +import org.apache.calcite.jdbc.CalciteConnection; +import org.apache.calcite.jdbc.JavaTypeFactoryImpl; +import org.apache.calcite.linq4j.Enumerable; +import org.apache.calcite.linq4j.Linq4j; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rel.type.RelDataTypeFactory; +import org.apache.calcite.schema.ScannableTable; +import org.apache.calcite.schema.SchemaPlus; +import org.apache.calcite.schema.impl.AbstractTable; +import org.apache.calcite.sql.type.SqlTypeName; + +public class Calcite { + + public static final List<Object[]> PLAYER_DATA_AS_OBJECT_ARRAY = Arrays.asList( + new Object[] {1, "Wizard", 5}, + new Object[] {2, "Hunter", 7} + + ); + public static final List<Object[]> EQUIPMENT_DATA_AS_OBJECT_ARRAY = Arrays.asList( + new Object[] {1, "fireball", 7, 1}, + new Object[] {2, "rifle", 4, 2}); + + + public static void main(String[] args) throws SQLException { + + + RelDataTypeFactory typeFactory = new JavaTypeFactoryImpl(); + RelDataTypeFactory.Builder playerType = new RelDataTypeFactory.Builder(typeFactory); + + Properties info = new Properties(); + // https://calcite.apache.org/javadocAggregate/org/apache/calcite/config/Lex.html#JAVA + info.setProperty("lex", "MYSQL"); + + Connection connection = DriverManager.getConnection("jdbc:calcite:", info); + CalciteConnection calciteConnection = + connection.unwrap(CalciteConnection.class); + + SchemaPlus rootSchema = calciteConnection.getRootSchema(); + + playerType.add("id", SqlTypeName.INTEGER); + playerType.add("name", SqlTypeName.VARCHAR); + playerType.add("level", SqlTypeName.INTEGER); + + ListTable playerTable = new ListTable(playerType.build(), PLAYER_DATA_AS_OBJECT_ARRAY); + rootSchema.add("player", playerTable); + + RelDataTypeFactory.Builder equipmentType = new RelDataTypeFactory.Builder(typeFactory); + + equipmentType.add("id", SqlTypeName.INTEGER); + equipmentType.add("name", SqlTypeName.VARCHAR); + equipmentType.add("damage", SqlTypeName.INTEGER); + equipmentType.add("player_id", SqlTypeName.INTEGER); + + ListTable equipmentTable = new ListTable(equipmentType.build(), EQUIPMENT_DATA_AS_OBJECT_ARRAY); + rootSchema.add("equipment", equipmentTable); + + + String sql = + "SELECT player.name, equipment.name FROM player INNER JOIN equipment ON player.id = equipment.player_id "; + ResultSet resultSet = connection.createStatement().executeQuery(sql); + StringBuilder b = new StringBuilder(); + while (resultSet.next()) { + b.append(resultSet.getString(1)).append(" attacks with "); + b.append(resultSet.getString(2)).append(" !\n"); + } + System.out.println(b); + + + resultSet.close(); + } + + /** + * A simple table based on a list. + */ + private static class ListTable extends AbstractTable implements ScannableTable { + private final RelDataType rowType; + private final List<Object[]> data; + + ListTable(RelDataType rowType, List<Object[]> data) { + this.rowType = rowType; + this.data = data; + } + + @Override + public Enumerable<Object[]> scan(final DataContext root) { + return Linq4j.asEnumerable(data); + } + + @Override + public RelDataType getRowType(final RelDataTypeFactory typeFactory) { + return rowType; + } + } +} diff --git a/baremaps-core/src/main/java/org/apache/baremaps/collection/algorithm/ExternalMergeSort.java b/baremaps-core/src/main/java/org/apache/baremaps/collection/algorithm/ExternalMergeSort.java index cd8190c1..4a86efb9 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/collection/algorithm/ExternalMergeSort.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/collection/algorithm/ExternalMergeSort.java @@ -125,8 +125,8 @@ public class ExternalMergeSort { } } - for (DataList<T> DataStore : batches) { - DataStore.clear(); + for (DataList<T> batch : batches) { + batch.clear(); } return counter; diff --git a/baremaps-core/src/main/java/org/apache/baremaps/collection/type/BooleanDataType.java b/baremaps-core/src/main/java/org/apache/baremaps/collection/type/BooleanDataType.java new file mode 100644 index 00000000..759ecf68 --- /dev/null +++ b/baremaps-core/src/main/java/org/apache/baremaps/collection/type/BooleanDataType.java @@ -0,0 +1,38 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + */ + +package org.apache.baremaps.collection.type; + + + +import java.nio.ByteBuffer; + +/** A {@link DataType} for reading and writing bytes in {@link ByteBuffer}s. */ +public class BooleanDataType extends MemoryAlignedDataType<Boolean> { + + /** Constructs a {@link ByteDataType}. */ + public BooleanDataType() { + super(Byte.BYTES); + } + + /** {@inheritDoc} */ + @Override + public void write(ByteBuffer buffer, int position, Boolean value) { + buffer.put(position, value ? (byte) 1 : (byte) 0); + } + + /** {@inheritDoc} */ + @Override + public Boolean read(ByteBuffer buffer, int position) { + return buffer.get(position) == 1; + } +} diff --git a/baremaps-core/src/main/java/org/apache/baremaps/collection/type/geometry/GeometryDataType.java b/baremaps-core/src/main/java/org/apache/baremaps/collection/type/geometry/GeometryDataType.java index 376e82de..475ec700 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/collection/type/geometry/GeometryDataType.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/collection/type/geometry/GeometryDataType.java @@ -18,7 +18,7 @@ import org.apache.baremaps.collection.type.DataType; import org.locationtech.jts.geom.*; /** - * A {@code DataType} for {@code MultiPolygon} objects. + * A {@code DataType} for {@link Geometry} objects. */ public class GeometryDataType implements DataType<Geometry> { diff --git a/baremaps-core/src/main/java/org/apache/baremaps/collection/type/geometry/LonLatDataType.java b/baremaps-core/src/main/java/org/apache/baremaps/collection/type/geometry/LonLatDataType.java new file mode 100644 index 00000000..9e2ceb67 --- /dev/null +++ b/baremaps-core/src/main/java/org/apache/baremaps/collection/type/geometry/LonLatDataType.java @@ -0,0 +1,67 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + */ + +package org.apache.baremaps.collection.type.geometry; + + + +import java.nio.ByteBuffer; +import org.apache.baremaps.collection.type.DataType; +import org.apache.baremaps.collection.type.MemoryAlignedDataType; +import org.locationtech.jts.geom.Coordinate; + +/** + * A {@link DataType} for reading and writing longitude/latitude coordinates in {@link ByteBuffer}s. + * An integer is used to compress the coordinates to the detriment of precision (centimeters). + */ +public class LonLatDataType extends MemoryAlignedDataType<Coordinate> { + + private static final double BITS = Math.pow(2, 31); + private static final long SHIFT = 32; + private static final long MASK = (1L << 32) - 1L; + + /** Constructs a {@link LonLatDataType}. */ + public LonLatDataType() { + super(Long.BYTES); + } + + public static long encodeLonLat(double lon, double lat) { + long x = (long) (((lon + 180) / 360) * BITS); + long y = (long) (((lat + 90) / 180) * BITS); + long l = (x << SHIFT); + long r = (y & MASK); + return l | r; + } + + public static double decodeLon(long value) { + double l = (value >>> 32); + return (l / BITS) * 360 - 180; + } + + public static double decodeLat(long value) { + long r = (value & MASK); + return (r / BITS) * 180 - 90; + } + + /** {@inheritDoc} */ + @Override + public void write(ByteBuffer buffer, int position, Coordinate value) { + buffer.putLong(position, encodeLonLat(value.x, value.y)); + } + + /** {@inheritDoc} */ + @Override + public Coordinate read(ByteBuffer buffer, int position) { + var value = buffer.getLong(position); + return new Coordinate(decodeLon(value), decodeLat(value)); + } +} diff --git a/baremaps-core/src/main/java/org/apache/baremaps/collection/type/geometry/WKBDataType.java b/baremaps-core/src/main/java/org/apache/baremaps/collection/type/geometry/WKBDataType.java new file mode 100644 index 00000000..2b4d9a91 --- /dev/null +++ b/baremaps-core/src/main/java/org/apache/baremaps/collection/type/geometry/WKBDataType.java @@ -0,0 +1,54 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + */ + +package org.apache.baremaps.collection.type.geometry; + + + +import java.nio.ByteBuffer; +import org.apache.baremaps.collection.type.DataType; +import org.apache.baremaps.utils.GeometryUtils; +import org.locationtech.jts.geom.Geometry; + +/** A {@link DataType} for reading and writing {@link Geometry} in {@link ByteBuffer}s. */ +public class WKBDataType implements DataType<Geometry> { + + /** {@inheritDoc} */ + @Override + public int size(Geometry value) { + byte[] bytes = GeometryUtils.serialize(value); + return Integer.BYTES + bytes.length; + } + + /** {@inheritDoc} */ + @Override + public int size(ByteBuffer buffer, int position) { + return buffer.getInt(position); + } + + /** {@inheritDoc} */ + @Override + public void write(ByteBuffer buffer, int position, Geometry value) { + byte[] bytes = GeometryUtils.serialize(value); + buffer.putInt(position, Integer.BYTES + bytes.length); + buffer.put(position + Integer.BYTES, bytes); + } + + /** {@inheritDoc} */ + @Override + public Geometry read(ByteBuffer buffer, int position) { + int size = buffer.getInt(position); + byte[] bytes = new byte[Math.max(size - Integer.BYTES, 0)]; + buffer.get(position + Integer.BYTES, bytes); + return GeometryUtils.deserialize(bytes); + } +} diff --git a/baremaps-core/src/main/java/org/apache/baremaps/storage/Row.java b/baremaps-core/src/main/java/org/apache/baremaps/storage/Row.java index 3de90fbf..63bb5582 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/storage/Row.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/storage/Row.java @@ -41,6 +41,14 @@ public interface Row { */ Object get(String column); + /** + * Returns the value of the specified column. + * + * @param index the index of the column + * @return the value of the specified column + */ + Object get(int index); + /** * Sets the value of the specified column. * @@ -49,4 +57,12 @@ public interface Row { */ void set(String column, Object value); + /** + * Sets the value of the specified column. + * + * @param index the index of the column + * @param value the value + */ + void set(int index, Object value); + } diff --git a/baremaps-core/src/main/java/org/apache/baremaps/storage/RowDataType.java b/baremaps-core/src/main/java/org/apache/baremaps/storage/RowDataType.java new file mode 100644 index 00000000..9e13c5b3 --- /dev/null +++ b/baremaps-core/src/main/java/org/apache/baremaps/storage/RowDataType.java @@ -0,0 +1,85 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + */ + +package org.apache.baremaps.storage; + +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Map; +import org.apache.baremaps.collection.type.*; + +public class RowDataType implements DataType<Row> { + + private static final Map<Class, DataType> types = Map.of( + Byte.class, new ByteDataType(), + Boolean.class, new BooleanDataType(), + Short.class, new ShortDataType(), + Integer.class, new IntegerDataType(), + Long.class, new LongDataType(), + Float.class, new FloatDataType(), + Double.class, new DoubleDataType(), + String.class, new StringDataType()); + + private final Schema schema; + + public RowDataType(Schema schema) { + this.schema = schema; + } + + @Override + public int size(Row row) { + var size = Integer.BYTES; + var columns = schema.columns(); + for (int i = 0; i < columns.size(); i++) { + var columnType = columns.get(i).type(); + var dataType = types.get(columnType); + var value = row.get(i); + size += dataType.size(value); + } + return size; + } + + @Override + public int size(ByteBuffer buffer, int position) { + return buffer.getInt(position); + } + + @Override + public void write(final ByteBuffer buffer, final int position, final Row row) { + var p = position + Integer.BYTES; + var columns = schema.columns(); + for (int i = 0; i < columns.size(); i++) { + var column = columns.get(i); + var columnType = column.type(); + var dataType = types.get(columnType); + var value = row.get(i); + dataType.write(buffer, position, value); + p += dataType.size(buffer, position); + } + buffer.putInt(position, p - position); + } + + @Override + public Row read(final ByteBuffer buffer, final int position) { + var p = position + Integer.BYTES; + var columns = schema.columns(); + var values = new ArrayList(); + for (int i = 0; i < columns.size(); i++) { + var column = columns.get(i); + var columnType = column.type(); + var dataType = types.get(columnType); + values.add(dataType.read(buffer, p)); + p += dataType.size(buffer, p); + } + return new RowImpl(schema, values); + } +} diff --git a/baremaps-core/src/main/java/org/apache/baremaps/storage/RowImpl.java b/baremaps-core/src/main/java/org/apache/baremaps/storage/RowImpl.java index fd2bc4ad..f3afcf14 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/storage/RowImpl.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/storage/RowImpl.java @@ -32,6 +32,14 @@ public record RowImpl(Schema schema, List values) implements Row { throw new IllegalArgumentException("Column " + column + " not found."); } + /** + * {@inheritDoc} + */ + @Override + public Object get(int index) { + return values.get(index); + } + /** * {@inheritDoc} */ @@ -46,4 +54,9 @@ public record RowImpl(Schema schema, List values) implements Row { throw new IllegalArgumentException("Column " + column + " not found."); } + @Override + public void set(int index, Object value) { + values.set(index, value); + } + } diff --git a/baremaps-core/src/main/java/org/apache/baremaps/storage/SchemaImpl.java b/baremaps-core/src/main/java/org/apache/baremaps/storage/SchemaImpl.java index 67bba9c8..b3b1bcde 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/storage/SchemaImpl.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/storage/SchemaImpl.java @@ -13,12 +13,39 @@ package org.apache.baremaps.storage; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; /** * A schema defines the structure of a table. */ -public record SchemaImpl(String name, List<Column> columns) implements Schema { +public class SchemaImpl implements Schema { + + private final String name; + + private final List<Column> columns; + + private final Map<String, Integer> index; + + public SchemaImpl(String name, List<Column> columns) { + this.name = name; + this.columns = columns; + this.index = new HashMap<>(); + for (int i = 0; i < columns.size(); i++) { + index.put(columns.get(i).name(), i); + } + } + + @Override + public String name() { + return name; + } + + @Override + public List<Column> columns() { + return columns; + } /** * {@inheritDoc} diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/Task.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/Task.java index 7a5c919f..65676fef 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/Task.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/Task.java @@ -39,8 +39,6 @@ import org.apache.baremaps.workflow.tasks.*; @JsonSubTypes.Type(value = UngzipFile.class, name = "UngzipFile"), @JsonSubTypes.Type(value = UpdateOpenStreetMap.class, name = "UpdateOpenStreetMap"), @JsonSubTypes.Type(value = CreateGeonamesIndex.class, name = "CreateGeonamesIndex"), - @JsonSubTypes.Type(value = CreateEntityCollection.class, name = "CreateEntityCollection"), - @JsonSubTypes.Type(value = TransformEntityCollection.class, name = "TransformEntityCollection"), @JsonSubTypes.Type(value = CreateIplocIndex.class, name = "CreateIplocIndex")}) public interface Task { diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/CreateEntityCollection.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/CreateEntityCollection.java deleted file mode 100644 index e7976bf9..00000000 --- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/CreateEntityCollection.java +++ /dev/null @@ -1,93 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except - * in compliance with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License - * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. See the License for the specific language governing permissions and limitations under - * the License. - */ - -package org.apache.baremaps.workflow.tasks; - - -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.HashMap; -import java.util.stream.Collectors; -import org.apache.baremaps.collection.AppendOnlyBuffer; -import org.apache.baremaps.collection.MemoryAlignedDataList; -import org.apache.baremaps.collection.MonotonicDataMap; -import org.apache.baremaps.collection.memory.MemoryMappedFile; -import org.apache.baremaps.collection.type.*; -import org.apache.baremaps.openstreetmap.model.*; -import org.apache.baremaps.openstreetmap.pbf.PbfEntityReader; -import org.apache.baremaps.utils.FileUtils; -import org.apache.baremaps.utils.ProjectionTransformer; -import org.apache.baremaps.workflow.Task; -import org.apache.baremaps.workflow.WorkflowContext; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - - -public record CreateEntityCollection(Path file, Path collection, - Integer srid) implements Task { - - private static final Logger logger = LoggerFactory.getLogger(ImportOpenStreetMap.class); - - @Override - public void execute(WorkflowContext context) throws Exception { - var path = file.toAbsolutePath(); - var cacheDir = Files.createTempDirectory(Paths.get("."), "cache_"); - - var coordinatesKeysFile = Files.createFile(cacheDir.resolve("coordinates_keys")); - var coordinatesValsFile = Files.createFile(cacheDir.resolve("coordinates_vals")); - var coordinateMap = new MonotonicDataMap<>( - new MemoryAlignedDataList<>(new PairDataType<>(new LongDataType(), new LongDataType()), - new MemoryMappedFile(coordinatesKeysFile)), - new AppendOnlyBuffer<>(new LonLatDataType(), new MemoryMappedFile(coordinatesValsFile))); - - var referencesKeysFile = Files.createFile(cacheDir.resolve("references_keys")); - var referencesValuesFile = Files.createFile(cacheDir.resolve("references_vals")); - var referenceMap = new MonotonicDataMap<>( - new MemoryAlignedDataList<>(new PairDataType<>(new LongDataType(), new LongDataType()), - new MemoryMappedFile(referencesKeysFile)), - new AppendOnlyBuffer<>(new LongListDataType(), new MemoryMappedFile(referencesValuesFile))); - - Files.deleteIfExists(collection); - - var entityCollection = - new AppendOnlyBuffer<>(new EntityDataType(), new MemoryMappedFile(collection)); - - var projectionTransformer = new ProjectionTransformer(4326, srid); - - // Read the PBF file and dispatch the elements to the consumers - new PbfEntityReader() - .geometries(true) - .coordinateMap(coordinateMap) - .referenceMap(referenceMap) - .stream(Files.newInputStream(path)) - .filter(Element.class::isInstance) - .map(Element.class::cast) - .filter(element -> !element.getTags().isEmpty()) - .filter(element -> element.getGeometry() != null) - .map(element -> { - var geometry = element.getGeometry(); - geometry = projectionTransformer.transform(geometry); - var tags = new HashMap<String, String>(); - for (var tag : element.getTags().entrySet()) { - tags.put(tag.getKey(), tag.getValue().toString()); - } - return new Entity(element.id(), tags, geometry); - }) - .collect(Collectors.toCollection(() -> entityCollection)); - - entityCollection.close(); - - FileUtils.deleteRecursively(cacheDir); - } - -} diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/Entity.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/Entity.java deleted file mode 100644 index 05d60f5c..00000000 --- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/Entity.java +++ /dev/null @@ -1,83 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except - * in compliance with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License - * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. See the License for the specific language governing permissions and limitations under - * the License. - */ - -package org.apache.baremaps.workflow.tasks; - -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import org.apache.baremaps.storage.*; -import org.locationtech.jts.geom.Geometry; - -public class Entity implements Row { - - private long id; - - private Map<String, String> tags; - - private Geometry geometry; - - public Entity(long id, Map<String, String> tags, Geometry geometry) { - this.id = id; - this.tags = tags; - this.geometry = geometry; - } - - - @Override - public Schema schema() { - var columns = new ArrayList<Column>(); - columns.add(new ColumnImpl("id", Long.class)); - columns.add(new ColumnImpl("geometry", Geometry.class)); - for (var entry : tags.entrySet()) { - columns.add(new ColumnImpl(entry.getKey(), String.class)); - } - return new SchemaImpl("entity", columns); - } - - @Override - public void set(String column, Object value) { - tags.put(column, value.toString()); - } - - @Override - public Object get(String column) { - if (column.equals("id")) { - return id; - } else if (column.equals("geometry")) { - return geometry; - } else { - return tags.get(column); - } - } - - @Override - public List<Object> values() { - var map = new ArrayList(); - map.add(id); - map.add(geometry); - map.addAll(tags.values()); - return map; - } - - public long getId() { - return id; - } - - public Map<String, String> getTags() { - return tags; - } - - public Geometry getGeometry() { - return geometry; - } -} diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/EntityDataType.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/EntityDataType.java deleted file mode 100644 index 027762d3..00000000 --- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/EntityDataType.java +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except - * in compliance with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License - * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. See the License for the specific language governing permissions and limitations under - * the License. - */ - -package org.apache.baremaps.workflow.tasks; - -import java.nio.ByteBuffer; -import java.util.Map; -import org.apache.baremaps.collection.type.*; -import org.apache.baremaps.collection.type.geometry.GeometryDataType; -import org.locationtech.jts.geom.Geometry; - -public class EntityDataType implements DataType<Entity> { - - private LongDataType idType = new LongDataType(); - - private MapDataType<String, String> tagsType = - new MapDataType<>(new StringDataType(), new StringDataType()); - - private GeometryDataType geometryType = new GeometryDataType(); - - @Override - public int size(Entity value) { - int size = 0; - size += Integer.BYTES; - size += idType.size(value.getId()); - size += tagsType.size(value.getTags()); - size += geometryType.size(value.getGeometry()); - return size; - } - - @Override - public int size(ByteBuffer buffer, int position) { - return buffer.getInt(position); - } - - @Override - public void write(ByteBuffer buffer, int position, Entity value) { - buffer.putInt(position, size(value)); - position += Integer.BYTES; - idType.write(buffer, position, value.getId()); - position += idType.size(); - tagsType.write(buffer, position, value.getTags()); - position += tagsType.size(value.getTags()); - geometryType.write(buffer, position, value.getGeometry()); - } - - @Override - public Entity read(ByteBuffer buffer, int position) { - position += Integer.BYTES; - long id = idType.read(buffer, position); - position += idType.size(id); - Map<String, String> tags = tagsType.read(buffer, position); - position += tagsType.size(tags); - Geometry geometry = geometryType.read(buffer, position); - return new Entity(id, tags, geometry); - } -} diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportOpenStreetMap.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportOpenStreetMap.java index a6a38cc5..54da8b80 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportOpenStreetMap.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportOpenStreetMap.java @@ -19,10 +19,10 @@ import java.nio.file.Paths; import java.util.List; import org.apache.baremaps.collection.*; import org.apache.baremaps.collection.memory.MemoryMappedFile; -import org.apache.baremaps.collection.type.LonLatDataType; import org.apache.baremaps.collection.type.LongDataType; import org.apache.baremaps.collection.type.LongListDataType; import org.apache.baremaps.collection.type.PairDataType; +import org.apache.baremaps.collection.type.geometry.LonLatDataType; import org.apache.baremaps.openstreetmap.model.Node; import org.apache.baremaps.openstreetmap.model.Relation; import org.apache.baremaps.openstreetmap.model.Way; diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/TransformEntityCollection.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/TransformEntityCollection.java deleted file mode 100644 index 4bfcffff..00000000 --- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/TransformEntityCollection.java +++ /dev/null @@ -1,150 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except - * in compliance with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License - * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. See the License for the specific language governing permissions and limitations under - * the License. - */ - -package org.apache.baremaps.workflow.tasks; - - -import java.nio.file.Path; -import java.util.ArrayList; -import java.util.Iterator; -import java.util.List; -import java.util.stream.Collectors; -import java.util.stream.IntStream; -import java.util.stream.Stream; -import org.apache.baremaps.collection.AppendOnlyBuffer; -import org.apache.baremaps.collection.algorithm.UnionStream; -import org.apache.baremaps.collection.memory.MemoryMappedFile; -import org.apache.baremaps.storage.*; -import org.apache.baremaps.storage.postgres.PostgresStore; -import org.apache.baremaps.vectortile.expression.Expressions.Expression; -import org.apache.baremaps.workflow.Task; -import org.apache.baremaps.workflow.WorkflowContext; -import org.locationtech.jts.geom.Geometry; -import org.locationtech.jts.geom.LineString; -import org.locationtech.jts.geom.Polygon; -import org.locationtech.jts.geom.util.GeometryFixer; -import org.locationtech.jts.operation.linemerge.LineMerger; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public record TransformEntityCollection(Path collection, String database, - Recipe recipe) implements Task { - - private static final Logger logger = LoggerFactory.getLogger(TransformEntityCollection.class); - - record Recipe(String name, Expression<Boolean> filter, List<String> groupBy, - Operation operation) { - } - - enum Operation { - union, merge, none - } - - @Override - public void execute(WorkflowContext context) throws Exception { - logger.info("Transform {} with {}", collection, recipe); - - var schema = new SchemaImpl(recipe.name, columns()); - - var groups = new AppendOnlyBuffer<>(new EntityDataType(), new MemoryMappedFile(collection)) - .stream() - .filter(this::filter) - .collect(Collectors.groupingBy(this::propertyValues)); - - var table = new AbstractTable() { - - @Override - public Schema schema() { - return schema; - } - - @Override - public long sizeAsLong() { - return 0; - } - - @Override - public Iterator<Row> iterator() { - return groups.entrySet().stream().flatMap(entry -> { - var tags = IntStream.range(0, recipe.groupBy.size()).boxed() - .collect(Collectors.toMap(i -> recipe.groupBy.get(i), i -> entry.getKey().get(i))); - var geometries = entry.getValue(); - var simplified = simplify(geometries.stream().map(Entity::getGeometry)); - return simplified.map(geometry -> (Row) new Entity(0, tags, geometry)); - }).iterator(); - } - }; - - var dataSource = context.getDataSource(database); - var postgresDatabase = new PostgresStore(dataSource); - postgresDatabase.add(table); - } - - private Stream<Geometry> simplify(Stream<Geometry> geometries) { - return switch (recipe.operation()) { - case union -> union(geometries); - case merge -> merge(geometries); - case none -> geometries; - }; - } - - private List<String> groupBy() { - return recipe.groupBy() == null ? List.of() : recipe.groupBy(); - } - - private List<Column> columns() { - var list = new ArrayList<Column>(); - for (var property : groupBy()) { - list.add(new ColumnImpl(property, String.class)); - } - list.add(new ColumnImpl("geometry", Geometry.class)); - return list; - } - - private List<String> propertyValues(Entity entity) { - var map = new ArrayList<String>(); - for (var property : groupBy()) { - map.add(entity.get(property).toString()); - } - return map; - } - - private Stream<Geometry> union(Stream<Geometry> geometries) { - var filtered = geometries - .filter(Polygon.class::isInstance) - .map(Polygon.class::cast) - .map(GeometryFixer::fix) - .filter(Geometry::isValid) - .collect(Collectors.toCollection(ArrayList::new)); - return new UnionStream(filtered).union(); - } - - private Stream<Geometry> merge(Stream<Geometry> geometries) { - var filtered = geometries - .filter(LineString.class::isInstance) - .map(LineString.class::cast) - .map(GeometryFixer::fix) - .toList(); - - var lineMerger = new LineMerger(); - lineMerger.add(filtered); - - var mergedGeometries = lineMerger.getMergedLineStrings(); - - return mergedGeometries.stream(); - } - - private boolean filter(Entity entity) { - return recipe.filter.evaluate(entity); - } - -} diff --git a/baremaps-core/src/test/java/org/apache/baremaps/collection/type/DataTypeProvider.java b/baremaps-core/src/test/java/org/apache/baremaps/collection/type/DataTypeProvider.java index 0b8d15f8..e6aa1b06 100644 --- a/baremaps-core/src/test/java/org/apache/baremaps/collection/type/DataTypeProvider.java +++ b/baremaps-core/src/test/java/org/apache/baremaps/collection/type/DataTypeProvider.java @@ -17,11 +17,15 @@ import java.util.List; import java.util.Map; import java.util.stream.Stream; import org.apache.baremaps.collection.type.geometry.*; +import org.apache.baremaps.storage.*; import org.junit.jupiter.params.provider.Arguments; import org.locationtech.jts.geom.*; public class DataTypeProvider { + private final Schema schema = new SchemaImpl("row", + List.of(new ColumnImpl("integer", Integer.class), new ColumnImpl("string", String.class))); + private static Stream<Arguments> dataTypes() { return Stream.of( // String @@ -260,6 +264,10 @@ public class DataTypeProvider { new Coordinate(3, 2), new Coordinate(4, 2), new Coordinate(4, 1), new Coordinate(3, 1)})})})), + // Row + // Arguments.of(new RowDataType(sc), ) + + // Geometry Arguments.of(new GeometryDataType(), new GeometryFactory().createEmpty(0)), @@ -267,5 +275,6 @@ public class DataTypeProvider { new GeometryFactory() .createPoint(new Coordinate(1, 1)))); + } } diff --git a/baremaps-core/src/test/java/org/apache/baremaps/tilestore/file/FileTileStoreTest.java b/baremaps-core/src/test/java/org/apache/baremaps/tilestore/file/FileTileStoreTest.java index b89f8833..250397fd 100644 --- a/baremaps-core/src/test/java/org/apache/baremaps/tilestore/file/FileTileStoreTest.java +++ b/baremaps-core/src/test/java/org/apache/baremaps/tilestore/file/FileTileStoreTest.java @@ -18,8 +18,8 @@ import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; -import org.apache.baremaps.tilestore.TileStore; import org.apache.baremaps.tilestore.TileStoreTest; +import org.apache.baremaps.tilestore.TileStore; import org.apache.baremaps.utils.FileUtils; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; diff --git a/baremaps-core/src/test/java/org/apache/baremaps/vectortile/ExpressionsTest.java b/baremaps-core/src/test/java/org/apache/baremaps/vectortile/ExpressionsTest.java index da436be0..61db0112 100644 --- a/baremaps-core/src/test/java/org/apache/baremaps/vectortile/ExpressionsTest.java +++ b/baremaps-core/src/test/java/org/apache/baremaps/vectortile/ExpressionsTest.java @@ -16,7 +16,6 @@ import static org.junit.jupiter.api.Assertions.*; import java.io.IOException; import java.util.List; -import java.util.Map; import org.apache.baremaps.storage.Row; import org.apache.baremaps.storage.Schema; import org.apache.baremaps.vectortile.expression.Expressions; @@ -25,7 +24,10 @@ import org.junit.jupiter.api.Test; class ExpressionsTest { - record RowMock(Map<String, Object> properties) implements Row { + record Property(String name, Object value) { + } + + record RowMock(List<Property> properties) implements Row { @Override public Schema schema() { @@ -34,17 +36,30 @@ class ExpressionsTest { @Override public List<Object> values() { - return properties.values().stream().toList(); + return properties.stream().map(p -> p.value).toList(); } @Override public void set(String column, Object value) { - properties.put(column, value); + properties.add(new Property(column, value)); + } + + @Override + public void set(int index, Object value) { + properties.set(index, new Property(properties.get(index).name, value)); } @Override public Object get(String column) { - return properties.get(column); + return properties.stream() + .filter(p -> p.name.equals(column)) + .findFirst().map(p -> p.value) + .orElse(null); + } + + @Override + public Object get(int index) { + return properties.get(index).value; } } @@ -68,15 +83,15 @@ class ExpressionsTest { @Test public void get() throws IOException { assertEquals("value", - new Get("key").evaluate(new RowMock(Map.of("key", "value")))); - assertEquals(null, new Get("key").evaluate(new RowMock(Map.of()))); + new Get("key").evaluate(new RowMock(List.of(new Property("key", "value"))))); + assertEquals(null, new Get("key").evaluate(new RowMock(List.of()))); } @Test public void has() throws IOException { assertEquals(true, - new Has("key").evaluate(new RowMock(Map.of("key", "value")))); - assertEquals(false, new Has("key").evaluate(new RowMock(Map.of()))); + new Has("key").evaluate(new RowMock(List.of(new Property("key", "value"))))); + assertEquals(false, new Has("key").evaluate(new RowMock(List.of()))); } @Test diff --git a/baremaps-core/src/test/java/org/apache/baremaps/workflow/tasks/CreateAndTransformEntityCollectionTest.java b/baremaps-core/src/test/java/org/apache/baremaps/workflow/tasks/CreateAndTransformEntityCollectionTest.java deleted file mode 100644 index d2ce68e1..00000000 --- a/baremaps-core/src/test/java/org/apache/baremaps/workflow/tasks/CreateAndTransformEntityCollectionTest.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except - * in compliance with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License - * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. See the License for the specific language governing permissions and limitations under - * the License. - */ - -package org.apache.baremaps.workflow.tasks; - - - -import java.nio.file.Files; -import java.nio.file.Paths; -import java.util.List; -import org.apache.baremaps.testing.PostgresContainerTest; -import org.apache.baremaps.testing.TestFiles; -import org.apache.baremaps.vectortile.expression.Expressions.Has; -import org.apache.baremaps.workflow.WorkflowContext; -import org.apache.baremaps.workflow.tasks.TransformEntityCollection.Operation; -import org.apache.baremaps.workflow.tasks.TransformEntityCollection.Recipe; -import org.junit.jupiter.api.Tag; -import org.junit.jupiter.api.Test; - -class CreateAndTransformEntityCollectionTest extends PostgresContainerTest { - - @Test - @Tag("integration") - void execute() throws Exception { - var file = TestFiles.resolve("monaco/monaco.osm.pbf"); - var collection = Paths.get("entity_collection"); - var jdbcUrl = jdbcUrl(); - - var createTask = new CreateEntityCollection(file, collection, 3857); - createTask.execute(new WorkflowContext()); - - var simplifyTask = new TransformEntityCollection(collection, jdbcUrl, - new Recipe("building", new Has("landuse"), List.of("landuse"), - Operation.union)); - simplifyTask.execute(new WorkflowContext()); - - Files.delete(collection); - } -}
