This is an automated email from the ASF dual-hosted git repository.
bchapuis pushed a commit to branch 681-handle-the-out-of-memory-errors
in repository https://gitbox.apache.org/repos/asf/incubator-baremaps.git
The following commit(s) were added to
refs/heads/681-handle-the-out-of-memory-errors by this push:
new 30687638 Use a memory mapped directory instead of the heap
30687638 is described below
commit 306876385fb364fc4dc8981a3547a0514f315d42
Author: Bertil Chapuis <[email protected]>
AuthorDate: Thu Jun 29 11:50:20 2023 +0200
Use a memory mapped directory instead of the heap
---
.../apache/baremaps/collection/LongDataMap.java | 41 -------------------
.../collection/memory/MemoryMappedDirectory.java | 2 +-
.../workflow/tasks/ImportOpenStreetMap.java | 46 +++++++++++-----------
3 files changed, 24 insertions(+), 65 deletions(-)
diff --git
a/baremaps-core/src/main/java/org/apache/baremaps/collection/LongDataMap.java
b/baremaps-core/src/main/java/org/apache/baremaps/collection/LongDataMap.java
deleted file mode 100644
index b3bfee4d..00000000
---
a/baremaps-core/src/main/java/org/apache/baremaps/collection/LongDataMap.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Licensed under the Apache License, Version 2.0 (the "License"); you may not
use this file except
- * in compliance with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
distributed under the License
- * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express
- * or implied. See the License for the specific language governing permissions
and limitations under
- * the License.
- */
-
-package org.apache.baremaps.collection;
-
-
-
-import org.apache.baremaps.collection.memory.Memory;
-import org.apache.baremaps.collection.memory.OffHeapMemory;
-import org.apache.baremaps.collection.type.LongDataType;
-
-/**
- * A list of longs.
- */
-public class LongDataMap extends MemoryAlignedDataMap<Long> {
-
- /**
- * Constructs a list.
- */
- public LongDataMap() {
- this(new OffHeapMemory());
- }
-
- /**
- * Constructs a list.
- *
- * @param memory the memory
- */
- public LongDataMap(Memory memory) {
- super(new LongDataType(), memory);
- }
-}
diff --git
a/baremaps-core/src/main/java/org/apache/baremaps/collection/memory/MemoryMappedDirectory.java
b/baremaps-core/src/main/java/org/apache/baremaps/collection/memory/MemoryMappedDirectory.java
index be02294f..b3b11e7e 100644
---
a/baremaps-core/src/main/java/org/apache/baremaps/collection/memory/MemoryMappedDirectory.java
+++
b/baremaps-core/src/main/java/org/apache/baremaps/collection/memory/MemoryMappedDirectory.java
@@ -35,7 +35,7 @@ public class MemoryMappedDirectory extends
Memory<MappedByteBuffer> {
* @param directory the directory that stores the data
*/
public MemoryMappedDirectory(Path directory) {
- this(directory, 1 << 30);
+ this(directory, 1 << 28);
}
/**
diff --git
a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportOpenStreetMap.java
b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportOpenStreetMap.java
index a6a38cc5..3ebc7aee 100644
---
a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportOpenStreetMap.java
+++
b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportOpenStreetMap.java
@@ -18,7 +18,7 @@ import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.List;
import org.apache.baremaps.collection.*;
-import org.apache.baremaps.collection.memory.MemoryMappedFile;
+import org.apache.baremaps.collection.memory.MemoryMappedDirectory;
import org.apache.baremaps.collection.type.LonLatDataType;
import org.apache.baremaps.collection.type.LongDataType;
import org.apache.baremaps.collection.type.LongListDataType;
@@ -69,35 +69,35 @@ public record ImportOpenStreetMap(Path file, String
database, Integer databaseSr
var cacheDir = Files.createTempDirectory(Paths.get("."), "cache_");
- DataMap<Coordinate> coordinateMap;
- if (Files.size(path) > 1 << 30) {
- var coordinatesFile = Files.createFile(cacheDir.resolve("coordinates"));
- coordinateMap = new MemoryAlignedDataMap<>(
- new LonLatDataType(),
- new MemoryMappedFile(coordinatesFile));
- } else {
- var coordinatesKeysFile =
Files.createFile(cacheDir.resolve("coordinates_keys"));
- var coordinatesValsFile =
Files.createFile(cacheDir.resolve("coordinates_vals"));
- coordinateMap =
- new MonotonicDataMap<>(
- new MemoryAlignedDataList<>(
- new PairDataType<>(new LongDataType(), new LongDataType()),
- new MemoryMappedFile(coordinatesKeysFile)),
- new AppendOnlyBuffer<>(
- new LonLatDataType(),
- new MemoryMappedFile(coordinatesValsFile)));
- }
+ var coordinatesOffsetsDir =
Files.createDirectory(cacheDir.resolve("coordinates_offsets"));
+ var coordinatesKeysDir =
Files.createDirectory(cacheDir.resolve("coordinates_keys"));
+ var coordinatesValsDir =
Files.createDirectory(cacheDir.resolve("coordinates_vals"));
+ var coordinateMap =
+ new MonotonicDataMap<>(
+ new MemoryAlignedDataList<>(
+ new LongDataType(),
+ new MemoryMappedDirectory(coordinatesOffsetsDir)),
+ new MemoryAlignedDataList<>(
+ new PairDataType<>(new LongDataType(), new LongDataType()),
+ new MemoryMappedDirectory(coordinatesKeysDir)),
+ new AppendOnlyBuffer<>(
+ new LonLatDataType(),
+ new MemoryMappedDirectory(coordinatesValsDir)));
- var referencesKeysDir =
Files.createFile(cacheDir.resolve("references_keys"));
- var referencesValuesDir =
Files.createFile(cacheDir.resolve("references_vals"));
+ var referencesOffsetsDir =
Files.createDirectory(cacheDir.resolve("references_offsets"));
+ var referencesKeysDir =
Files.createDirectory(cacheDir.resolve("references_keys"));
+ var referencesValuesDir =
Files.createDirectory(cacheDir.resolve("references_vals"));
var referenceMap =
new MonotonicDataMap<>(
+ new MemoryAlignedDataList<>(
+ new LongDataType(),
+ new MemoryMappedDirectory(referencesOffsetsDir)),
new MemoryAlignedDataList<>(
new PairDataType<>(new LongDataType(), new LongDataType()),
- new MemoryMappedFile(referencesKeysDir)),
+ new MemoryMappedDirectory(referencesKeysDir)),
new AppendOnlyBuffer<>(
new LongListDataType(),
- new MemoryMappedFile(referencesValuesDir)));
+ new MemoryMappedDirectory(referencesValuesDir)));
execute(
path,
