[incubator-baremaps] branch 681-handle-the-out-of-memory-errors updated: Estimate the size of the collections

[bchapuis]

This is an automated email from the ASF dual-hosted git repository.

bchapuis pushed a commit to branch 681-handle-the-out-of-memory-errors
in repository https://gitbox.apache.org/repos/asf/incubator-baremaps.git


The following commit(s) were added to 
refs/heads/681-handle-the-out-of-memory-errors by this push:
     new dce1cf4c Estimate the size of the collections
dce1cf4c is described below

commit dce1cf4cdceb24e3433cc6df5758308beff04b81
Author: Bertil Chapuis <bchap...@gmail.com>
AuthorDate: Wed Jul 12 23:41:26 2023 +0200

    Estimate the size of the collections
---
 .../database/collection/AppendOnlyBuffer.java      |  1 +
 .../workflow/tasks/ImportOpenStreetMap.java        | 28 ++++++++++++----------
 2 files changed, 17 insertions(+), 12 deletions(-)

diff --git 
a/baremaps-core/src/main/java/org/apache/baremaps/database/collection/AppendOnlyBuffer.java
 
b/baremaps-core/src/main/java/org/apache/baremaps/database/collection/AppendOnlyBuffer.java
index bb741110..5e4f2110 100644
--- 
a/baremaps-core/src/main/java/org/apache/baremaps/database/collection/AppendOnlyBuffer.java
+++ 
b/baremaps-core/src/main/java/org/apache/baremaps/database/collection/AppendOnlyBuffer.java
@@ -120,6 +120,7 @@ public class AppendOnlyBuffer<E> extends 
AbstractDataCollection<E> {
   }
 
   /** {@inheritDoc} */
+  @Override
   public long size64() {
     return size;
   }
diff --git 
a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportOpenStreetMap.java
 
b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportOpenStreetMap.java
index e227dbf6..fca55ceb 100644
--- 
a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportOpenStreetMap.java
+++ 
b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportOpenStreetMap.java
@@ -34,6 +34,7 @@ import 
org.apache.baremaps.openstreetmap.postgres.PostgresWayRepository;
 import org.apache.baremaps.openstreetmap.repository.*;
 import org.apache.baremaps.openstreetmap.repository.BlockImporter;
 import org.apache.baremaps.stream.StreamUtils;
+import org.apache.baremaps.utils.FileUtils;
 import org.apache.baremaps.workflow.Task;
 import org.apache.baremaps.workflow.WorkflowContext;
 import org.locationtech.jts.geom.Coordinate;
@@ -71,7 +72,9 @@ public record ImportOpenStreetMap(Path file, String database, 
Integer databaseSr
     var coordinatesKeysFile = 
Files.createDirectory(cacheDir.resolve("coordinates_keys"));
     var coordinatesValsFile = 
Files.createDirectory(cacheDir.resolve("coordinates_vals"));
     var coordinateMap =
-        new Long2ObjectOpenHashDataMap<>(100_000_000L, 0.9f,
+        new Long2ObjectOpenHashDataMap<>(
+            estimateNumberOfNodes(path),
+            0.9f,
             () -> new Long2ObjectMemoryAlignedDataMap<>(
                 new LongDataType(),
                 new MemoryMappedDirectory(coordinatesKeysFile)),
@@ -85,7 +88,7 @@ public record ImportOpenStreetMap(Path file, String database, 
Integer databaseSr
     var referenceMap =
         new Long2ObjectIndexedDataMap<>(
             new Long2LongPackedOpenHashDataMap(
-                10_000_000L,
+                estimateNumberOfWays(path),
                 0.9f,
                 () -> new Long2ObjectMemoryAlignedDataMap<>(
                     new PairDataType<>(new LongDataType(), new LongDataType()),
@@ -94,15 +97,6 @@ public record ImportOpenStreetMap(Path file, String 
database, Integer databaseSr
                 new LongListDataType(),
                 new MemoryMappedDirectory(referencesValuesDir)));
 
-    // var referenceMap =
-    // new MonotonicDataMap<>(
-    // new MemoryAlignedDataList<>(
-    // new PairDataType<>(new LongDataType(), new LongDataType()),
-    // new MemoryMappedDirectory(referencesKeysDir)),
-    // new AppendOnlyBuffer<>(
-    // new LongListDataType(),
-    // new MemoryMappedDirectory(referencesValuesDir)));
-
     execute(
         path,
         coordinateMap,
@@ -113,7 +107,17 @@ public record ImportOpenStreetMap(Path file, String 
database, Integer databaseSr
         relationRepository,
         databaseSrid);
 
-    // FileUtils.deleteRecursively(cacheDir);
+    FileUtils.deleteRecursively(cacheDir);
+  }
+
+  private long estimateNumberOfNodes(Path path) throws IOException {
+    var size = Files.size(path);
+    return (long) (size * 0.2);
+  }
+
+  private long estimateNumberOfWays(Path path) throws IOException {
+    var size = Files.size(path);
+    return (long) (size * 0.02);
   }
 
   public static void execute(