This is an automated email from the ASF dual-hosted git repository. bchapuis pushed a commit to branch 681-handle-the-out-of-memory-errors in repository https://gitbox.apache.org/repos/asf/incubator-baremaps.git
commit d6021e7fb025edaf71e3477ffcd05556222d5c2f Author: Bertil Chapuis <[email protected]> AuthorDate: Wed Jul 12 23:41:26 2023 +0200 Estimate the size of the collections --- .../database/collection/AppendOnlyBuffer.java | 1 + .../workflow/tasks/ImportOpenStreetMap.java | 30 ++++++++++++---------- 2 files changed, 18 insertions(+), 13 deletions(-) diff --git a/baremaps-core/src/main/java/org/apache/baremaps/database/collection/AppendOnlyBuffer.java b/baremaps-core/src/main/java/org/apache/baremaps/database/collection/AppendOnlyBuffer.java index bb741110..5e4f2110 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/database/collection/AppendOnlyBuffer.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/database/collection/AppendOnlyBuffer.java @@ -120,6 +120,7 @@ public class AppendOnlyBuffer<E> extends AbstractDataCollection<E> { } /** {@inheritDoc} */ + @Override public long size64() { return size; } diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportOpenStreetMap.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportOpenStreetMap.java index e227dbf6..fb664423 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportOpenStreetMap.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportOpenStreetMap.java @@ -34,6 +34,7 @@ import org.apache.baremaps.openstreetmap.postgres.PostgresWayRepository; import org.apache.baremaps.openstreetmap.repository.*; import org.apache.baremaps.openstreetmap.repository.BlockImporter; import org.apache.baremaps.stream.StreamUtils; +import org.apache.baremaps.utils.FileUtils; import org.apache.baremaps.workflow.Task; import org.apache.baremaps.workflow.WorkflowContext; import org.locationtech.jts.geom.Coordinate; @@ -71,7 +72,9 @@ public record ImportOpenStreetMap(Path file, String database, Integer databaseSr var coordinatesKeysFile = Files.createDirectory(cacheDir.resolve("coordinates_keys")); var coordinatesValsFile = Files.createDirectory(cacheDir.resolve("coordinates_vals")); var coordinateMap = - new Long2ObjectOpenHashDataMap<>(100_000_000L, 0.9f, + new Long2ObjectOpenHashDataMap<>( + estimateNumberOfNodes(path), + 0.99f, () -> new Long2ObjectMemoryAlignedDataMap<>( new LongDataType(), new MemoryMappedDirectory(coordinatesKeysFile)), @@ -85,8 +88,8 @@ public record ImportOpenStreetMap(Path file, String database, Integer databaseSr var referenceMap = new Long2ObjectIndexedDataMap<>( new Long2LongPackedOpenHashDataMap( - 10_000_000L, - 0.9f, + estimateNumberOfWays(path), + 0.99f, () -> new Long2ObjectMemoryAlignedDataMap<>( new PairDataType<>(new LongDataType(), new LongDataType()), new MemoryMappedDirectory(referencesKeysDir))), @@ -94,15 +97,6 @@ public record ImportOpenStreetMap(Path file, String database, Integer databaseSr new LongListDataType(), new MemoryMappedDirectory(referencesValuesDir))); - // var referenceMap = - // new MonotonicDataMap<>( - // new MemoryAlignedDataList<>( - // new PairDataType<>(new LongDataType(), new LongDataType()), - // new MemoryMappedDirectory(referencesKeysDir)), - // new AppendOnlyBuffer<>( - // new LongListDataType(), - // new MemoryMappedDirectory(referencesValuesDir))); - execute( path, coordinateMap, @@ -113,7 +107,17 @@ public record ImportOpenStreetMap(Path file, String database, Integer databaseSr relationRepository, databaseSrid); - // FileUtils.deleteRecursively(cacheDir); + FileUtils.deleteRecursively(cacheDir); + } + + private long estimateNumberOfNodes(Path path) throws IOException { + var size = Files.size(path); + return (long) (size * 0.2); + } + + private long estimateNumberOfWays(Path path) throws IOException { + var size = Files.size(path); + return (long) (size * 0.02); } public static void execute(
