This is an automated email from the ASF dual-hosted git repository.
bchapuis pushed a commit to branch simplify-geometries
in repository https://gitbox.apache.org/repos/asf/incubator-baremaps.git
The following commit(s) were added to refs/heads/simplify-geometries by this
push:
new 4660ce35 Create and simplify collections
4660ce35 is described below
commit 4660ce35658e7b247bbfd8deaa88ae746bf1c86c
Author: Bertil Chapuis <[email protected]>
AuthorDate: Wed Feb 15 00:58:15 2023 +0100
Create and simplify collections
---
.../baremaps/collection/AppendOnlyBuffer.java | 10 +-
.../baremaps/collection/type/ListDataType.java | 3 +-
.../baremaps/collection/type/MapDataType.java | 70 ++++++++++
.../baremaps/mvt/expression/Expressions.java | 32 ++++-
.../baremaps/openstreetmap/CollectionUtils.java | 1 -
.../workflow/tasks/CreateEntityCollection.java | 97 +++++++++++++
.../org/apache/baremaps/workflow/tasks/Entity.java | 74 ++++++++++
.../baremaps/workflow/tasks/EntityDataType.java | 64 +++++++++
.../workflow/tasks/SimplifyEntityCollection.java | 91 ++++++++++++
.../workflow/tasks/SimplifyOpenStreetMap.java | 155 ---------------------
.../baremaps/collection/type/DataTypeProvider.java | 10 +-
.../baremaps/collection/type/DataTypeTest.java | 6 +-
...java => EntityDataTypeGeometryBuilderTest.java} | 2 +-
....java => XmlEntitySpliteratorTestDataType.java} | 2 +-
... => CreateAndSimplifyEntityCollectionTest.java} | 19 ++-
15 files changed, 464 insertions(+), 172 deletions(-)
diff --git
a/baremaps-core/src/main/java/org/apache/baremaps/collection/AppendOnlyBuffer.java
b/baremaps-core/src/main/java/org/apache/baremaps/collection/AppendOnlyBuffer.java
index cb775ae7..19ffb6c4 100644
---
a/baremaps-core/src/main/java/org/apache/baremaps/collection/AppendOnlyBuffer.java
+++
b/baremaps-core/src/main/java/org/apache/baremaps/collection/AppendOnlyBuffer.java
@@ -60,8 +60,8 @@ public class AppendOnlyBuffer<E> extends DataCollection<E> {
this.dataType = dataType;
this.memory = memory;
this.segmentSize = memory.segmentSize();
- this.offset = 0;
- this.size = 0;
+ this.offset = Long.BYTES;
+ this.size = memory.segment(0).getLong(0);
}
/**
@@ -122,6 +122,10 @@ public class AppendOnlyBuffer<E> extends DataCollection<E>
{
return size;
}
+ public void close() {
+ memory.segment(0).putLong(0, size);
+ }
+
/** {@inheritDoc} */
@Override
public void clear() {
@@ -142,7 +146,7 @@ public class AppendOnlyBuffer<E> extends DataCollection<E> {
private long index = 0;
- private long position = 0;
+ private long position = Long.BYTES;
@Override
public boolean hasNext() {
diff --git
a/baremaps-core/src/main/java/org/apache/baremaps/collection/type/ListDataType.java
b/baremaps-core/src/main/java/org/apache/baremaps/collection/type/ListDataType.java
index ed5b6329..63737368 100644
---
a/baremaps-core/src/main/java/org/apache/baremaps/collection/type/ListDataType.java
+++
b/baremaps-core/src/main/java/org/apache/baremaps/collection/type/ListDataType.java
@@ -53,9 +53,10 @@ public class ListDataType<T> implements DataType<List<T>> {
@Override
public void write(ByteBuffer buffer, int position, List<T> values) {
buffer.putInt(position, size(values));
+ position += Integer.BYTES;
for (T value : values) {
- position += dataType.size(value);
dataType.write(buffer, position, value);
+ position += dataType.size(value);
}
}
diff --git
a/baremaps-core/src/main/java/org/apache/baremaps/collection/type/MapDataType.java
b/baremaps-core/src/main/java/org/apache/baremaps/collection/type/MapDataType.java
new file mode 100644
index 00000000..dbf98c4a
--- /dev/null
+++
b/baremaps-core/src/main/java/org/apache/baremaps/collection/type/MapDataType.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
use this file except
+ * in compliance with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express
+ * or implied. See the License for the specific language governing permissions
and limitations under
+ * the License.
+ */
+
+package org.apache.baremaps.collection.type;
+
+import java.nio.ByteBuffer;
+import java.util.HashMap;
+import java.util.Map;
+
+public class MapDataType<K, V> implements DataType<Map<K, V>> {
+
+ private final DataType<K> keyType;
+
+ private final DataType<V> valueType;
+
+ public MapDataType(DataType<K> keyType, DataType<V> valueType) {
+ this.keyType = keyType;
+ this.valueType = valueType;
+ }
+
+ @Override
+ public int size(Map<K, V> value) {
+ int size = Integer.BYTES;
+ for (Map.Entry<K, V> entry : value.entrySet()) {
+ size += keyType.size(entry.getKey());
+ size += valueType.size(entry.getValue());
+ }
+ return size;
+ }
+
+ @Override
+ public int size(ByteBuffer buffer, int position) {
+ return buffer.getInt(position);
+ }
+
+ @Override
+ public void write(ByteBuffer buffer, int position, Map<K, V> value) {
+ buffer.putInt(position, size(value));
+ position += Integer.BYTES;
+ for (Map.Entry<K, V> entry : value.entrySet()) {
+ keyType.write(buffer, position, entry.getKey());
+ position += keyType.size(entry.getKey());
+ valueType.write(buffer, position, entry.getValue());
+ position += valueType.size(entry.getValue());
+ }
+ }
+
+ @Override
+ public Map<K, V> read(ByteBuffer buffer, int position) {
+ int size = buffer.getInt(position);
+ Map<K, V> map = new HashMap<>(size);
+ for (int p = position + Integer.BYTES; p < position + size;) {
+ K key = keyType.read(buffer, p);
+ p += keyType.size(key);
+ V value = valueType.read(buffer, p);
+ p += valueType.size(value);
+ map.put(key, value);
+ }
+ return map;
+ }
+}
diff --git
a/baremaps-core/src/main/java/org/apache/baremaps/mvt/expression/Expressions.java
b/baremaps-core/src/main/java/org/apache/baremaps/mvt/expression/Expressions.java
index 30396b4e..3d48ff3a 100644
---
a/baremaps-core/src/main/java/org/apache/baremaps/mvt/expression/Expressions.java
+++
b/baremaps-core/src/main/java/org/apache/baremaps/mvt/expression/Expressions.java
@@ -32,6 +32,7 @@ import java.util.ArrayList;
import java.util.List;
import java.util.function.Predicate;
import org.apache.baremaps.feature.Feature;
+import org.locationtech.jts.geom.*;
public interface Expressions {
@@ -378,6 +379,36 @@ public interface Expressions {
}
}
+ record GeometryType(Expression expression) implements Expression<String> {
+
+ @Override
+ public String name() {
+ return "geometry-type";
+ }
+
+ @Override
+ public String evaluate(Feature feature) {
+ Object property = feature.getProperty("geom");
+ if (property instanceof Point) {
+ return "Point";
+ } else if (property instanceof LineString) {
+ return "LineString";
+ } else if (property instanceof Polygon) {
+ return "Polygon";
+ } else if (property instanceof MultiPoint) {
+ return "MultiPoint";
+ } else if (property instanceof MultiLineString) {
+ return "MultiLineString";
+ } else if (property instanceof MultiPolygon) {
+ return "MultiPolygon";
+ } else if (property instanceof GeometryCollection) {
+ return "GeometryCollection";
+ } else {
+ return "Unknown";
+ }
+ }
+ }
+
class ExpressionSerializer extends StdSerializer<Expression> {
@@ -407,7 +438,6 @@ public interface Expressions {
}
}
-
class ExpressionDeserializer extends StdDeserializer<Expression> {
protected ExpressionDeserializer() {
diff --git
a/baremaps-core/src/main/java/org/apache/baremaps/openstreetmap/CollectionUtils.java
b/baremaps-core/src/main/java/org/apache/baremaps/openstreetmap/CollectionUtils.java
index a2841fe5..d0f1fc81 100644
---
a/baremaps-core/src/main/java/org/apache/baremaps/openstreetmap/CollectionUtils.java
+++
b/baremaps-core/src/main/java/org/apache/baremaps/openstreetmap/CollectionUtils.java
@@ -30,7 +30,6 @@ public class CollectionUtils {
private CollectionUtils() {}
-
public static DataMap<Coordinate> denseCoordinateMap() throws IOException {
var cacheDir = Files.createTempDirectory(Paths.get("."), "coordinates_");
var coordinatesFile = Files.createFile(cacheDir.resolve("values"));
diff --git
a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/CreateEntityCollection.java
b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/CreateEntityCollection.java
new file mode 100644
index 00000000..d2a14557
--- /dev/null
+++
b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/CreateEntityCollection.java
@@ -0,0 +1,97 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
use this file except
+ * in compliance with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express
+ * or implied. See the License for the specific language governing permissions
and limitations under
+ * the License.
+ */
+
+package org.apache.baremaps.workflow.tasks;
+
+
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.HashMap;
+import org.apache.baremaps.collection.AppendOnlyBuffer;
+import org.apache.baremaps.collection.MemoryAlignedDataList;
+import org.apache.baremaps.collection.MonotonicDataMap;
+import org.apache.baremaps.collection.memory.MemoryMappedFile;
+import org.apache.baremaps.collection.type.*;
+import org.apache.baremaps.collection.utils.FileUtils;
+import org.apache.baremaps.openstreetmap.model.*;
+import org.apache.baremaps.openstreetmap.pbf.PbfEntityReader;
+import org.apache.baremaps.openstreetmap.utils.ProjectionTransformer;
+import org.apache.baremaps.workflow.Task;
+import org.apache.baremaps.workflow.WorkflowContext;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+
+public record CreateEntityCollection(Path file, Path collection,
+ Integer targetSrid) implements Task {
+
+ private static final Logger logger =
LoggerFactory.getLogger(ImportOpenStreetMap.class);
+
+ @Override
+ public void execute(WorkflowContext context) throws Exception {
+ logger.info("Importing {} into {}", file, collection);
+
+ var path = file.toAbsolutePath();
+ var cacheDir = Files.createTempDirectory(Paths.get("."), "cache_");
+
+ var coordinatesKeysFile =
Files.createFile(cacheDir.resolve("coordinates_keys"));
+ var coordinatesValsFile =
Files.createFile(cacheDir.resolve("coordinates_vals"));
+ var coordinateMap = new MonotonicDataMap<>(
+ new MemoryAlignedDataList<>(new PairDataType<>(new LongDataType(), new
LongDataType()),
+ new MemoryMappedFile(coordinatesKeysFile)),
+ new AppendOnlyBuffer<>(new LonLatDataType(), new
MemoryMappedFile(coordinatesValsFile)));
+
+ var referencesKeysFile =
Files.createFile(cacheDir.resolve("references_keys"));
+ var referencesValuesFile =
Files.createFile(cacheDir.resolve("references_vals"));
+ var referenceMap = new MonotonicDataMap<>(
+ new MemoryAlignedDataList<>(new PairDataType<>(new LongDataType(), new
LongDataType()),
+ new MemoryMappedFile(referencesKeysFile)),
+ new AppendOnlyBuffer<>(new LongListDataType(), new
MemoryMappedFile(referencesValuesFile)));
+
+ Files.deleteIfExists(collection);
+
+
+ var entityCollection =
+ new AppendOnlyBuffer<>(new EntityDataType(), new
MemoryMappedFile(collection));
+
+ var projectionTransformer = new ProjectionTransformer(4326, targetSrid);
+
+ // Read the PBF file and dispatch the elements to the consumers
+ new PbfEntityReader()
+ .geometries(true)
+ .coordinateMap(coordinateMap)
+ .referenceMap(referenceMap)
+ .stream(Files.newInputStream(path))
+ .filter(Element.class::isInstance)
+ .map(Element.class::cast)
+ .filter(element -> !element.getTags().isEmpty())
+ .filter(element -> element.getGeometry() != null)
+ .map(element -> {
+ var geometry = element.getGeometry();
+ geometry = projectionTransformer.transform(geometry);
+ var tags = new HashMap<String, String>();
+ for (var tag : element.getTags().entrySet()) {
+ tags.put(tag.getKey(), tag.getValue().toString());
+ }
+ return new Entity(element.id(), tags, geometry);
+ })
+ .forEach(entityCollection::add);
+
+ entityCollection.close();
+
+ FileUtils.deleteRecursively(cacheDir);
+
+ logger.info("Finished importing {} into {}", file, collection);
+ }
+
+}
diff --git
a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/Entity.java
b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/Entity.java
new file mode 100644
index 00000000..ccdfde6e
--- /dev/null
+++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/Entity.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
use this file except
+ * in compliance with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express
+ * or implied. See the License for the specific language governing permissions
and limitations under
+ * the License.
+ */
+
+package org.apache.baremaps.workflow.tasks;
+
+import java.util.HashMap;
+import java.util.Map;
+import org.apache.baremaps.feature.Feature;
+import org.apache.baremaps.feature.FeatureType;
+import org.apache.baremaps.feature.PropertyType;
+import org.locationtech.jts.geom.Geometry;
+
+public class Entity implements Feature {
+
+ private final long id;
+
+ private final Map<String, String> tags;
+
+ private final Geometry geometry;
+
+ public Entity(long id, Map<String, String> tags, Geometry geometry) {
+ this.id = id;
+ this.tags = tags;
+ this.geometry = geometry;
+ }
+
+
+ @Override
+ public FeatureType getType() {
+ var map = new HashMap<String, PropertyType>();
+ map.put("id", new PropertyType<>("id", Long.class));
+ for (var entry : tags.entrySet()) {
+ map.put(entry.getKey(), new PropertyType<>(entry.getKey(),
String.class));
+ }
+ map.put("geometry", new PropertyType<>("geometry", Geometry.class));
+ return new FeatureType("entity", map);
+ }
+
+ @Override
+ public void setProperty(String name, Object value) {
+ tags.put(name, value.toString());
+ }
+
+ @Override
+ public Object getProperty(String name) {
+ return tags.get(name);
+ }
+
+ @Override
+ public Map<String, Object> getProperties() {
+ return Map.copyOf(tags);
+ }
+
+ public long getId() {
+ return id;
+ }
+
+ public Map<String, String> getTags() {
+ return tags;
+ }
+
+ public Geometry getGeometry() {
+ return geometry;
+ }
+}
diff --git
a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/EntityDataType.java
b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/EntityDataType.java
new file mode 100644
index 00000000..f9c9a2de
--- /dev/null
+++
b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/EntityDataType.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
use this file except
+ * in compliance with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express
+ * or implied. See the License for the specific language governing permissions
and limitations under
+ * the License.
+ */
+
+package org.apache.baremaps.workflow.tasks;
+
+import java.nio.ByteBuffer;
+import java.util.Map;
+import org.apache.baremaps.collection.type.*;
+import org.locationtech.jts.geom.Geometry;
+
+public class EntityDataType implements DataType<Entity> {
+
+ private LongDataType idType = new LongDataType();
+
+ private MapDataType<String, String> tagsType =
+ new MapDataType<>(new StringDataType(), new StringDataType());
+
+ private GeometryDataType geometryType = new GeometryDataType();
+
+ @Override
+ public int size(Entity value) {
+ int size = 0;
+ size += idType.size();
+ size += tagsType.size(value.getTags());
+ size += geometryType.size(value.getGeometry());
+ return size;
+ }
+
+ @Override
+ public int size(ByteBuffer buffer, int position) {
+ return buffer.getInt(position);
+ }
+
+ @Override
+ public void write(ByteBuffer buffer, int position, Entity value) {
+ buffer.putInt(position, size(value));
+ position += Integer.BYTES;
+ idType.write(buffer, position, value.getId());
+ position += idType.size();
+ tagsType.write(buffer, position, value.getTags());
+ position += tagsType.size(value.getTags());
+ geometryType.write(buffer, position, value.getGeometry());
+ }
+
+ @Override
+ public Entity read(ByteBuffer buffer, int position) {
+ position += Integer.BYTES;
+ long id = idType.read(buffer, position);
+ position += idType.size(id);
+ Map<String, String> tags = tagsType.read(buffer, position);
+ position += tagsType.size(tags);
+ Geometry geometry = geometryType.read(buffer, position);
+ return new Entity(id, tags, geometry);
+ }
+}
diff --git
a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/SimplifyEntityCollection.java
b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/SimplifyEntityCollection.java
new file mode 100644
index 00000000..2edbf29d
--- /dev/null
+++
b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/SimplifyEntityCollection.java
@@ -0,0 +1,91 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
use this file except
+ * in compliance with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express
+ * or implied. See the License for the specific language governing permissions
and limitations under
+ * the License.
+ */
+
+package org.apache.baremaps.workflow.tasks;
+
+
+import java.nio.file.Path;
+import java.util.Map;
+import java.util.function.*;
+import java.util.stream.IntStream;
+import java.util.stream.Stream;
+import org.apache.baremaps.collection.*;
+import org.apache.baremaps.collection.AppendOnlyBuffer;
+import org.apache.baremaps.collection.memory.MemoryMappedFile;
+import org.apache.baremaps.collection.type.*;
+import org.apache.baremaps.feature.*;
+import org.apache.baremaps.mvt.expression.Expressions.Expression;
+import org.apache.baremaps.storage.postgres.PostgresDatabase;
+import org.apache.baremaps.workflow.Task;
+import org.apache.baremaps.workflow.WorkflowContext;
+import org.locationtech.jts.geom.Geometry;
+import org.locationtech.jts.geom.Polygon;
+import org.locationtech.jts.geom.util.GeometryFixer;
+import org.locationtech.jts.operation.union.CascadedPolygonUnion;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public record SimplifyEntityCollection(Path collection, String database,
+ Recipe recipe) implements Task {
+
+ private static final Logger logger =
LoggerFactory.getLogger(ImportOpenStreetMap.class);
+
+ record Recipe(String name, Expression<Boolean> filter, Operation operation) {
+ }
+
+ enum Operation {
+ union, merge, none
+ }
+
+ @Override
+ public void execute(WorkflowContext context) throws Exception {
+ // read the original collection
+ var entities =
+ new AppendOnlyBuffer<>(new EntityDataType(), new
MemoryMappedFile(collection));
+
+ // filter the entities of the collection
+ var filteredEntities = entities.stream()
+ .filter(entity -> recipe.filter.evaluate(entity))
+ .toList();
+
+ var geometries = filteredEntities.stream()
+ .map(Entity::getGeometry)
+ .filter(Polygon.class::isInstance)
+ .map(Polygon.class::cast)
+ .map(GeometryFixer::fix)
+ .toList();
+
+ // Apply the operation to each recipe and store the result in the
collection
+ var unionedGeometry = new CascadedPolygonUnion(geometries).union();
+
+ var featureType =
+ new FeatureType(recipe.name, Map.of("geom", new PropertyType("geom",
Geometry.class)));
+
+ Stream<Feature> stream = IntStream.range(0,
unionedGeometry.getNumGeometries())
+ .mapToObj(unionedGeometry::getGeometryN)
+ .map(geometry -> new FeatureImpl(featureType, Map.of("geom",
geometry)));
+
+ var dataSource = context.getDataSource(database);
+ var postgresDatabase = new PostgresDatabase(dataSource);
+ postgresDatabase.write(new ReadableFeatureSet() {
+ @Override
+ public FeatureType getType() {
+ return featureType;
+ }
+
+ @Override
+ public Stream<Feature> read() {
+ return stream;
+ }
+ });
+ }
+}
diff --git
a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/SimplifyOpenStreetMap.java
b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/SimplifyOpenStreetMap.java
deleted file mode 100644
index dfd352b9..00000000
---
a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/SimplifyOpenStreetMap.java
+++ /dev/null
@@ -1,155 +0,0 @@
-/*
- * Licensed under the Apache License, Version 2.0 (the "License"); you may not
use this file except
- * in compliance with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
distributed under the License
- * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express
- * or implied. See the License for the specific language governing permissions
and limitations under
- * the License.
- */
-
-package org.apache.baremaps.workflow.tasks;
-
-
-import com.google.common.base.Predicates;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.util.Collection;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.function.*;
-import java.util.stream.Collectors;
-import java.util.stream.IntStream;
-import java.util.stream.Stream;
-import org.apache.baremaps.collection.*;
-import org.apache.baremaps.collection.AppendOnlyBuffer;
-import org.apache.baremaps.collection.MemoryAlignedDataList;
-import org.apache.baremaps.collection.memory.MemoryMappedFile;
-import org.apache.baremaps.collection.memory.OffHeapMemory;
-import org.apache.baremaps.collection.type.*;
-import org.apache.baremaps.collection.utils.FileUtils;
-import org.apache.baremaps.feature.*;
-import org.apache.baremaps.openstreetmap.model.Element;
-import org.apache.baremaps.openstreetmap.pbf.PbfEntityReader;
-import org.apache.baremaps.openstreetmap.utils.ProjectionTransformer;
-import org.apache.baremaps.storage.postgres.PostgresDatabase;
-import org.apache.baremaps.workflow.Task;
-import org.apache.baremaps.workflow.WorkflowContext;
-import org.locationtech.jts.geom.Geometry;
-import org.locationtech.jts.operation.union.CascadedPolygonUnion;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public record SimplifyOpenStreetMap(Path file, String database, Integer
databaseSrid)
- implements
- Task {
-
- private static final Logger logger =
LoggerFactory.getLogger(ImportOpenStreetMap.class);
-
- @Override
- public void execute(WorkflowContext context) throws Exception {
- logger.info("Importing {} into {}", file, database);
-
- var path = file.toAbsolutePath();
-
- var cacheDir = Files.createTempDirectory(Paths.get("."), "cache_");
-
- var coordinatesKeysFile =
Files.createFile(cacheDir.resolve("coordinates_keys"));
- var coordinatesValsFile =
Files.createFile(cacheDir.resolve("coordinates_vals"));
- var coordinateMap = new MonotonicDataMap<>(
- new MemoryAlignedDataList<>(new PairDataType<>(new LongDataType(), new
LongDataType()),
- new MemoryMappedFile(coordinatesKeysFile)),
- new AppendOnlyBuffer<>(new LonLatDataType(), new
MemoryMappedFile(coordinatesValsFile)));
-
- var referencesKeysFile =
Files.createFile(cacheDir.resolve("references_keys"));
- var referencesValuesFile =
Files.createFile(cacheDir.resolve("references_vals"));
- var referenceMap = new MonotonicDataMap<>(
- new MemoryAlignedDataList<>(new PairDataType<>(new LongDataType(), new
LongDataType()),
- new MemoryMappedFile(referencesKeysFile)),
- new AppendOnlyBuffer<>(new LongListDataType(), new
MemoryMappedFile(referencesValuesFile)));
-
- var collection = new AppendOnlyBuffer<>(new GeometryDataType(), new
OffHeapMemory());
- var projectionTransformer = new ProjectionTransformer(4326, databaseSrid);
- new PbfEntityReader()
- .geometries(true)
- .coordinateMap(coordinateMap)
- .referenceMap(referenceMap)
- .stream(Files.newInputStream(path))
- .filter(Element.class::isInstance)
- .map(Element.class::cast)
- .filter(element -> element.getTags().containsKey("building"))
- .map(Element::getGeometry)
- .filter(Predicates.notNull())
- .map(projectionTransformer::transform)
- .forEach(collection::add);
- var unionedGeometry = new CascadedPolygonUnion(collection).union();
-
- var dataSource = context.getDataSource(database);
- var postgresDatabase = new PostgresDatabase(dataSource);
-
- var featureType =
- new FeatureType("buildings", Map.of("geom", new PropertyType("geom",
Geometry.class)));
- Stream<Feature> stream = IntStream.range(0,
unionedGeometry.getNumGeometries())
- .mapToObj(unionedGeometry::getGeometryN)
- .map(geometry -> new FeatureImpl(featureType, Map.of("geom",
geometry)));
-
- postgresDatabase.write(new ReadableFeatureSet() {
- @Override
- public FeatureType getType() {
- return featureType;
- }
-
- @Override
- public Stream<Feature> read() {
- return stream;
- }
- });
-
- FileUtils.deleteRecursively(cacheDir);
-
- logger.info("Finished importing {} into {}", file, database);
- }
-
- public static class PolygonUnionConsumer implements Consumer<Element> {
-
- private final Map<Map<String, Object>, Collection<Element>> groups = new
ConcurrentHashMap<>();
-
- private final Supplier<Collection<Element>> collectionSupplier;
-
- private final Predicate<Element> filter;
-
- private final List<String> groupBy;
-
- public PolygonUnionConsumer(Supplier<Collection<Element>>
collectionSupplier,
- Predicate<Element> filter, List<String> groupBy) {
- this.collectionSupplier = collectionSupplier;
- this.filter = filter;
- this.groupBy = groupBy;
- }
-
- @Override
- public void accept(Element element) {
- if (filter.test(element)) {
- var key =
- groupBy.stream().collect(Collectors.toMap(Function.identity(),
element.getTags()::get));
- var collection = groups.computeIfAbsent(key, k ->
collectionSupplier.get());
- collection.add(element);
- }
- }
-
- public Stream<Geometry> geometries() {
- return groups.entrySet().stream()
- .flatMap(entry -> {
- // var tags = entry.getKey();
- var collection = entry.getValue();
- var geometry = new CascadedPolygonUnion(collection).union();
- return IntStream.range(0, geometry.getNumGeometries())
- .mapToObj(geometry::getGeometryN);
- });
- }
- }
-}
diff --git
a/baremaps-core/src/test/java/org/apache/baremaps/collection/type/DataTypeProvider.java
b/baremaps-core/src/test/java/org/apache/baremaps/collection/type/DataTypeProvider.java
index f6d53d92..12f731e4 100644
---
a/baremaps-core/src/test/java/org/apache/baremaps/collection/type/DataTypeProvider.java
+++
b/baremaps-core/src/test/java/org/apache/baremaps/collection/type/DataTypeProvider.java
@@ -15,6 +15,7 @@ package org.apache.baremaps.collection.type;
import java.util.List;
+import java.util.Map;
import java.util.stream.Stream;
import org.junit.jupiter.params.provider.Arguments;
import org.locationtech.jts.geom.Coordinate;
@@ -23,7 +24,8 @@ import org.locationtech.jts.geom.GeometryFactory;
public class DataTypeProvider {
private static Stream<Arguments> dataTypes() {
- return Stream.of(Arguments.of(new StringDataType(), "Hello, World!"),
+ return Stream.of(
+ Arguments.of(new StringDataType(), "Hello, World!"),
Arguments.of(new ByteDataType(), (byte) 1),
Arguments.of(new ByteListDataType(), List.of((byte) 1, (byte) 2,
(byte) 3)),
Arguments.of(new DoubleDataType(), (double) 1),
@@ -42,6 +44,9 @@ public class DataTypeProvider {
Arguments.of(new ShortDataType(), (short) 1),
Arguments.of(new ShortListDataType(), List.of((short) 1, (short) 2,
(short) 3)),
Arguments.of(new ListDataType<>(new IntegerDataType()), List.of(1, 2,
3)),
+ Arguments.of(new ListDataType<>(new StringDataType()), List.of("aaaa",
"bbbb", "cccc")),
+ Arguments.of(new MapDataType<>(new StringDataType(), new
IntegerDataType()),
+ Map.of("a", 1, "b", 2, "c", 3)),
Arguments.of(new CoordinateDataType(), new Coordinate(-180, -90)),
Arguments.of(new CoordinateDataType(), new Coordinate(180, 90)),
Arguments.of(new LonLatDataType(), new Coordinate(-180, -90)),
@@ -70,7 +75,8 @@ public class DataTypeProvider {
Arguments.of(new SmallLongDataType(7), (long) -Math.pow(2, 55)),
Arguments.of(new SmallLongDataType(8), Long.MAX_VALUE),
Arguments.of(new SmallLongDataType(8), Long.MIN_VALUE),
- Arguments.of(new PairDataType<>(new LongDataType(), new
LongDataType()),
+ Arguments.of(
+ new PairDataType<>(new LongDataType(), new LongDataType()),
new PairDataType.Pair<>(1l, 2l)));
}
}
diff --git
a/baremaps-core/src/test/java/org/apache/baremaps/collection/type/DataTypeTest.java
b/baremaps-core/src/test/java/org/apache/baremaps/collection/type/DataTypeTest.java
index 67afada8..e548d6ff 100644
---
a/baremaps-core/src/test/java/org/apache/baremaps/collection/type/DataTypeTest.java
+++
b/baremaps-core/src/test/java/org/apache/baremaps/collection/type/DataTypeTest.java
@@ -26,7 +26,9 @@ class DataTypeTest {
var size = dataType.size(value);
var buffer = ByteBuffer.allocate(size);
dataType.write(buffer, 0, value);
- assertEquals(value, dataType.read(buffer, 0));
- assertEquals(size, dataType.size(buffer, 0));
+ var recordSize = dataType.size(buffer, 0);
+ var recordValue = dataType.read(buffer, 0);
+ assertEquals(size, recordSize);
+ assertEquals(value, recordValue);
}
}
diff --git
a/baremaps-core/src/test/java/org/apache/baremaps/openstreetmap/geometry/EntityGeometryBuilderTest.java
b/baremaps-core/src/test/java/org/apache/baremaps/openstreetmap/geometry/EntityDataTypeGeometryBuilderTest.java
similarity index 99%
rename from
baremaps-core/src/test/java/org/apache/baremaps/openstreetmap/geometry/EntityGeometryBuilderTest.java
rename to
baremaps-core/src/test/java/org/apache/baremaps/openstreetmap/geometry/EntityDataTypeGeometryBuilderTest.java
index c015f4bb..50e88394 100644
---
a/baremaps-core/src/test/java/org/apache/baremaps/openstreetmap/geometry/EntityGeometryBuilderTest.java
+++
b/baremaps-core/src/test/java/org/apache/baremaps/openstreetmap/geometry/EntityDataTypeGeometryBuilderTest.java
@@ -46,7 +46,7 @@ import org.locationtech.proj4j.CoordinateTransform;
import org.locationtech.proj4j.Proj4jException;
import org.locationtech.proj4j.ProjCoordinate;
-class EntityGeometryBuilderTest {
+class EntityDataTypeGeometryBuilderTest {
static final CRSFactory CRS_FACTORY = new CRSFactory();
diff --git
a/baremaps-core/src/test/java/org/apache/baremaps/openstreetmap/xml/XmlEntitySpliteratorTest.java
b/baremaps-core/src/test/java/org/apache/baremaps/openstreetmap/xml/XmlEntitySpliteratorTestDataType.java
similarity index 97%
rename from
baremaps-core/src/test/java/org/apache/baremaps/openstreetmap/xml/XmlEntitySpliteratorTest.java
rename to
baremaps-core/src/test/java/org/apache/baremaps/openstreetmap/xml/XmlEntitySpliteratorTestDataType.java
index b04edbba..6587cedc 100644
---
a/baremaps-core/src/test/java/org/apache/baremaps/openstreetmap/xml/XmlEntitySpliteratorTest.java
+++
b/baremaps-core/src/test/java/org/apache/baremaps/openstreetmap/xml/XmlEntitySpliteratorTestDataType.java
@@ -26,7 +26,7 @@ import org.apache.baremaps.stream.AccumulatingConsumer;
import org.apache.baremaps.stream.HoldingConsumer;
import org.junit.jupiter.api.Test;
-class XmlEntitySpliteratorTest {
+class XmlEntitySpliteratorTestDataType {
@Test
void tryAdvance() throws IOException {
diff --git
a/baremaps-core/src/test/java/org/apache/baremaps/workflow/tasks/SimplifyOpenStreetMapTest.java
b/baremaps-core/src/test/java/org/apache/baremaps/workflow/tasks/CreateAndSimplifyEntityCollectionTest.java
similarity index 55%
rename from
baremaps-core/src/test/java/org/apache/baremaps/workflow/tasks/SimplifyOpenStreetMapTest.java
rename to
baremaps-core/src/test/java/org/apache/baremaps/workflow/tasks/CreateAndSimplifyEntityCollectionTest.java
index 8943f849..56a42ea4 100644
---
a/baremaps-core/src/test/java/org/apache/baremaps/workflow/tasks/SimplifyOpenStreetMapTest.java
+++
b/baremaps-core/src/test/java/org/apache/baremaps/workflow/tasks/CreateAndSimplifyEntityCollectionTest.java
@@ -14,21 +14,30 @@ package org.apache.baremaps.workflow.tasks;
+import java.nio.file.Paths;
+import org.apache.baremaps.mvt.expression.Expressions.Has;
import org.apache.baremaps.testing.PostgresContainerTest;
import org.apache.baremaps.testing.TestFiles;
import org.apache.baremaps.workflow.WorkflowContext;
+import org.apache.baremaps.workflow.tasks.SimplifyEntityCollection.Operation;
+import org.apache.baremaps.workflow.tasks.SimplifyEntityCollection.Recipe;
import org.junit.jupiter.api.Tag;
import org.junit.jupiter.api.Test;
-class SimplifyOpenStreetMapTest extends PostgresContainerTest {
+class CreateAndSimplifyEntityCollectionTest extends PostgresContainerTest {
@Test
@Tag("integration")
void execute() throws Exception {
- var file = TestFiles.resolve("data.osm.pbf");
+ var file = TestFiles.resolve("monaco/monaco.osm.pbf");
+ var collection = Paths.get("entity_collection");
var jdbcUrl = jdbcUrl();
- var srid = 3857;
- var task = new SimplifyOpenStreetMap(file, jdbcUrl, srid);
- task.execute(new WorkflowContext());
+
+ var createTask = new CreateEntityCollection(file, collection, 3857);
+ createTask.execute(new WorkflowContext());
+
+ var simplifyTask = new SimplifyEntityCollection(collection, jdbcUrl,
+ new Recipe("building", new Has("building"), Operation.union));
+ simplifyTask.execute(new WorkflowContext());
}
}
