This is an automated email from the ASF dual-hosted git repository. bchapuis pushed a commit to branch 681-handle-the-out-of-memory-errors in repository https://gitbox.apache.org/repos/asf/incubator-baremaps.git
commit ca6c6f8bb5c473143f8a80af9bfc97c83a7816fd Author: Bertil Chapuis <[email protected]> AuthorDate: Tue Jul 4 23:03:36 2023 +0200 Format code --- .../apache/baremaps/collection/JaggedDataMap.java | 216 --------------------- .../database/collection/JaggedDataMap.java | 74 +++++-- 2 files changed, 55 insertions(+), 235 deletions(-) diff --git a/baremaps-core/src/main/java/org/apache/baremaps/collection/JaggedDataMap.java b/baremaps-core/src/main/java/org/apache/baremaps/collection/JaggedDataMap.java deleted file mode 100644 index 9fd3571d..00000000 --- a/baremaps-core/src/main/java/org/apache/baremaps/collection/JaggedDataMap.java +++ /dev/null @@ -1,216 +0,0 @@ -package org.apache.baremaps.collection; - -import org.apache.baremaps.database.collection.AppendOnlyBuffer; -import org.apache.baremaps.database.collection.DataMap; -import org.apache.baremaps.stream.StreamUtils; - -import java.util.Arrays; -import java.util.Iterator; -import java.util.Map; -import java.util.concurrent.atomic.AtomicLong; -import java.util.stream.IntStream; - -public class JaggedDataMap<E> extends DataMap<E> { - - private static final int L_BYTES = 8; - private static final int L_SIZE = 1 << L_BYTES; - private static final int L_MASK = L_SIZE - 1; - private static final int L_SHIFT = 0; - - private static final int K_BYTES = 8; - private static final int K_SIZE = 1 << K_BYTES; - private static final int K_MASK = K_SIZE - 1; - private static final int K_SHIFT = L_SHIFT + L_BYTES; - - private static final int J_BYTES = 12; - private static final int J_SIZE = 1 << J_BYTES; - private static final int J_MASK = J_SIZE - 1; - private static final int J_SHIFT = K_SHIFT + K_BYTES; - - private static final int I_BYTES = 12; - private static final int I_SIZE = 1 << I_BYTES; - private static final int I_MASK = I_SIZE - 1; - private static final int I_SHIFT = J_SHIFT + J_BYTES; - - private static final long CAPACITY = 1L << (I_BYTES + J_BYTES + K_BYTES + L_BYTES); - - private long[][][][] index; - - private final AppendOnlyBuffer<E> values; - - private final AtomicLong size = new AtomicLong(); - - /** - * Constructs a map. - * - * @param values the values - */ - public JaggedDataMap(AppendOnlyBuffer<E> values) { - this.index = new long[I_SIZE][][][]; - this.values = values; - } - - /** - * {@inheritDoc} - */ - @Override - public E put(Long key, E value) { - long v = key; - if (v < 0 || v >= CAPACITY) { - throw new IllegalArgumentException(); - } - int i = (int) (v >>> I_SHIFT) & I_MASK; - int j = (int) (v >>> J_SHIFT) & J_MASK; - int k = (int) (v >>> K_SHIFT) & K_MASK; - int l = (int) (v >>> L_SHIFT) & L_MASK; - if (index[i] == null) { - index[i] = new long[J_SIZE][][]; - } - if (index[i][j] == null) { - index[i][j] = new long[K_SIZE][]; - } - if (index[i][j][k] == null) { - index[i][j][k] = new long[L_SIZE]; - Arrays.fill(index[i][j][k], -1); - } - long position = values.addPositioned(value); - if (index[i][j][k][l] == -1) { - size.incrementAndGet(); - } - index[i][j][k][l] = position; - return value; - } - - /** - * {@inheritDoc} - */ - @Override - public E get(Object key) { - long v = (Long) key; - if (v < 0 || v >= CAPACITY) { - throw new IllegalArgumentException(); - } - int i = (int) (v >>> I_SHIFT) & I_MASK; - int j = (int) (v >>> J_SHIFT) & J_MASK; - int k = (int) (v >>> K_SHIFT) & K_MASK; - int l = (int) (v >>> L_SHIFT) & L_MASK; - if (index[i] == null) { - return null; - } - if (index[i][j] == null) { - return null; - } - if (index[i][j][k] == null) { - return null; - } - long position = index[i][j][k][l]; - if (position == -1) { - return null; - } - return values.read(position); - } - - /** - * {@inheritDoc} - */ - @Override - protected Iterator<Long> keyIterator() { - return IntStream.range(0, index.length) - .filter(i -> index[i] != null) - .mapToObj(i -> IntStream.range(0, index[i].length) - .filter(j -> index[i][j] != null) - .mapToObj(j -> IntStream.range(0, index[i][j].length) - .filter(k -> index[i][j][k] != null) - .mapToObj(k -> IntStream.range(0, index[i][j][k].length) - .filter(l -> index[i][j][k][l] != -1) - .mapToObj(l -> (long) ((i << I_SHIFT) | (j << J_SHIFT) | (k << K_SHIFT) | (l << L_SHIFT)))) - .flatMap(x -> x)) - .flatMap(x -> x)) - .flatMap(x -> x) - .iterator(); - } - - /** - * {@inheritDoc} - */ - @Override - protected Iterator<E> valueIterator() { - return StreamUtils.stream(keyIterator()).map(this::get).iterator(); - } - - /** - * {@inheritDoc} - */ - @Override - protected Iterator<Entry<Long, E>> entryIterator() { - return StreamUtils.stream(keyIterator()).map(key -> Map.entry(key, get(key))).iterator(); - } - - /** - * {@inheritDoc} - */ - @Override - public boolean isEmpty() { - return StreamUtils.stream(keyIterator()).findAny().isEmpty(); - } - - /** - * {@inheritDoc} - */ - @Override - public long sizeAsLong() { - return size.get(); - } - - /** - * {@inheritDoc} - */ - @Override - public boolean containsKey(Object key) { - return get(key) != null; - } - - /** - * {@inheritDoc} - */ - @Override - public boolean containsValue(Object value) { - return StreamUtils.stream(valueIterator()).anyMatch(v -> v.equals(value)); - } - - /** - * {@inheritDoc} - */ - @Override - public E remove(Object key) { - long v = (Long) key; - int i = (int) (v >>> I_SHIFT) & I_MASK; - int j = (int) (v >>> J_SHIFT) & J_MASK; - int k = (int) (v >>> K_SHIFT) & K_MASK; - int l = (int) (v >>> L_SHIFT) & L_MASK; - if (index[i] == null) { - return null; - } - if (index[i][j] == null) { - return null; - } - if (index[i][j][k] == null) { - return null; - } - long position = index[i][j][k][l]; - if (position == -1) { - return null; - } - size.decrementAndGet(); - index[i][j][k][l] = -1; - return values.read(position); - } - - /** - * {@inheritDoc} - */ - @Override - public void clear() { - index = new long[I_SIZE][][][]; - } -} diff --git a/baremaps-core/src/main/java/org/apache/baremaps/database/collection/JaggedDataMap.java b/baremaps-core/src/main/java/org/apache/baremaps/database/collection/JaggedDataMap.java index 6b5fdd23..c6ab5574 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/database/collection/JaggedDataMap.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/database/collection/JaggedDataMap.java @@ -15,22 +15,47 @@ package org.apache.baremaps.database.collection; import java.util.Arrays; import java.util.Iterator; import java.util.Map; +import java.util.concurrent.atomic.AtomicLong; import java.util.stream.IntStream; import org.apache.baremaps.stream.StreamUtils; public class JaggedDataMap<E> extends DataMap<E> { + private static final int L_BYTES = 8; + private static final int L_SIZE = 1 << L_BYTES; + private static final int L_MASK = L_SIZE - 1; + private static final int L_SHIFT = 0; + + private static final int K_BYTES = 8; + private static final int K_SIZE = 1 << K_BYTES; + private static final int K_MASK = K_SIZE - 1; + private static final int K_SHIFT = L_SHIFT + L_BYTES; + + private static final int J_BYTES = 12; + private static final int J_SIZE = 1 << J_BYTES; + private static final int J_MASK = J_SIZE - 1; + private static final int J_SHIFT = K_SHIFT + K_BYTES; + + private static final int I_BYTES = 12; + private static final int I_SIZE = 1 << I_BYTES; + private static final int I_MASK = I_SIZE - 1; + private static final int I_SHIFT = J_SHIFT + J_BYTES; + + private static final long CAPACITY = 1L << (I_BYTES + J_BYTES + K_BYTES + L_BYTES); + private long[][][][] index; private final AppendOnlyBuffer<E> values; + private final AtomicLong size = new AtomicLong(); + /** * Constructs a map. * * @param values the values */ public JaggedDataMap(AppendOnlyBuffer<E> values) { - this.index = new long[1 << 12][][][]; + this.index = new long[I_SIZE][][][]; this.values = values; } @@ -40,21 +65,27 @@ public class JaggedDataMap<E> extends DataMap<E> { @Override public E put(Long key, E value) { long v = key; - int i = (int) (v >>> 28) & 0xFFF; - int j = (int) (v >>> 16) & 0xFFF; - int k = (int) (v >>> 8) & 0xFF; - int l = (int) (v & 0xFF); + if (v < 0 || v >= CAPACITY) { + throw new IllegalArgumentException(); + } + int i = (int) (v >>> I_SHIFT) & I_MASK; + int j = (int) (v >>> J_SHIFT) & J_MASK; + int k = (int) (v >>> K_SHIFT) & K_MASK; + int l = (int) (v >>> L_SHIFT) & L_MASK; if (index[i] == null) { - index[i] = new long[1 << 12][][]; + index[i] = new long[J_SIZE][][]; } if (index[i][j] == null) { - index[i][j] = new long[1 << 12][]; + index[i][j] = new long[K_SIZE][]; } if (index[i][j][k] == null) { - index[i][j][k] = new long[1 << 8]; + index[i][j][k] = new long[L_SIZE]; Arrays.fill(index[i][j][k], -1); } long position = values.addPositioned(value); + if (index[i][j][k][l] == -1) { + size.incrementAndGet(); + } index[i][j][k][l] = position; return value; } @@ -65,10 +96,13 @@ public class JaggedDataMap<E> extends DataMap<E> { @Override public E get(Object key) { long v = (Long) key; - int i = (int) (v >>> 28) & 0xFFF; - int j = (int) (v >>> 16) & 0xFFF; - int k = (int) (v >>> 8) & 0xFF; - int l = (int) (v & 0xFF); + if (v < 0 || v >= CAPACITY) { + throw new IllegalArgumentException(); + } + int i = (int) (v >>> I_SHIFT) & I_MASK; + int j = (int) (v >>> J_SHIFT) & J_MASK; + int k = (int) (v >>> K_SHIFT) & K_MASK; + int l = (int) (v >>> L_SHIFT) & L_MASK; if (index[i] == null) { return null; } @@ -98,7 +132,8 @@ public class JaggedDataMap<E> extends DataMap<E> { .filter(k -> index[i][j][k] != null) .mapToObj(k -> IntStream.range(0, index[i][j][k].length) .filter(l -> index[i][j][k][l] != -1) - .mapToObj(l -> (long) ((i << 28) | (j << 16) | (k << 8) | l))) + .mapToObj(l -> (long) ((i << I_SHIFT) | (j << J_SHIFT) | (k << K_SHIFT) + | (l << L_SHIFT)))) .flatMap(x -> x)) .flatMap(x -> x)) .flatMap(x -> x) @@ -134,7 +169,7 @@ public class JaggedDataMap<E> extends DataMap<E> { */ @Override public long sizeAsLong() { - return StreamUtils.stream(keyIterator()).count(); + return size.get(); } /** @@ -159,10 +194,10 @@ public class JaggedDataMap<E> extends DataMap<E> { @Override public E remove(Object key) { long v = (Long) key; - int i = (int) (v >>> 28) & 0xFFF; - int j = (int) (v >>> 16) & 0xFFF; - int k = (int) (v >>> 8) & 0xFF; - int l = (int) (v & 0xFF); + int i = (int) (v >>> I_SHIFT) & I_MASK; + int j = (int) (v >>> J_SHIFT) & J_MASK; + int k = (int) (v >>> K_SHIFT) & K_MASK; + int l = (int) (v >>> L_SHIFT) & L_MASK; if (index[i] == null) { return null; } @@ -176,6 +211,7 @@ public class JaggedDataMap<E> extends DataMap<E> { if (position == -1) { return null; } + size.decrementAndGet(); index[i][j][k][l] = -1; return values.read(position); } @@ -185,6 +221,6 @@ public class JaggedDataMap<E> extends DataMap<E> { */ @Override public void clear() { - index = new long[1 << 12][][][]; + index = new long[I_SIZE][][][]; } }
