This is an automated email from the ASF dual-hosted git repository. bchapuis pushed a commit to branch 681-handle-the-out-of-memory-errors in repository https://gitbox.apache.org/repos/asf/incubator-baremaps.git
commit 5c23673bdb3deadda1a9a77bc9dfe5a3568a44b0 Author: Bertil Chapuis <[email protected]> AuthorDate: Sat Jul 1 19:43:17 2023 +0200 Remove unused collections --- .../baremaps/benchmarks/DataMapBenchmark.java | 30 +--- .../collection/MonotonicFixedSizeDataMap.java | 157 --------------------- .../collection/MonotonicPairedDataMap.java | 145 ------------------- .../org/apache/baremaps/database/DataMapTest.java | 6 +- 4 files changed, 6 insertions(+), 332 deletions(-) diff --git a/baremaps-benchmark/src/main/java/org/apache/baremaps/benchmarks/DataMapBenchmark.java b/baremaps-benchmark/src/main/java/org/apache/baremaps/benchmarks/DataMapBenchmark.java index 24b11931..1d5e0af0 100644 --- a/baremaps-benchmark/src/main/java/org/apache/baremaps/benchmarks/DataMapBenchmark.java +++ b/baremaps-benchmark/src/main/java/org/apache/baremaps/benchmarks/DataMapBenchmark.java @@ -15,10 +15,13 @@ package org.apache.baremaps.benchmarks; import java.util.concurrent.TimeUnit; -import org.apache.baremaps.database.collection.*; + +import org.apache.baremaps.database.collection.AppendOnlyBuffer; +import org.apache.baremaps.database.collection.DataMap; +import org.apache.baremaps.database.collection.MemoryAlignedDataMap; +import org.apache.baremaps.database.collection.MonotonicDataMap; import org.apache.baremaps.database.memory.OffHeapMemory; import org.apache.baremaps.database.type.LongDataType; -import org.apache.baremaps.database.type.PairDataType; import org.openjdk.jmh.annotations.*; import org.openjdk.jmh.runner.Runner; import org.openjdk.jmh.runner.RunnerException; @@ -59,29 +62,6 @@ public class DataMapBenchmark { benchmark(new MonotonicDataMap<>(new AppendOnlyBuffer<>(new LongDataType())), N); } - @Benchmark - @BenchmarkMode(Mode.SingleShotTime) - @Warmup(iterations = 2) - @Measurement(iterations = 5) - public void monotonicPairedDataMap() { - benchmark(new MonotonicPairedDataMap<>( - new MemoryAlignedDataList<>( - new PairDataType<>(new LongDataType(), new LongDataType()), - new OffHeapMemory())), - N); - } - - @Benchmark - @BenchmarkMode(Mode.SingleShotTime) - @Warmup(iterations = 2) - @Measurement(iterations = 5) - public void monotonicFixedSizeDataMap() { - benchmark(new MonotonicFixedSizeDataMap<>( - new MemoryAlignedDataList<>(new LongDataType()), - new MemoryAlignedDataList<>(new LongDataType()), - new MemoryAlignedDataList<>(new LongDataType())), N); - } - public static void main(String[] args) throws RunnerException { org.openjdk.jmh.runner.options.Options opt = new OptionsBuilder().include(DataMapBenchmark.class.getSimpleName()).forks(1).build(); diff --git a/baremaps-core/src/main/java/org/apache/baremaps/database/collection/MonotonicFixedSizeDataMap.java b/baremaps-core/src/main/java/org/apache/baremaps/database/collection/MonotonicFixedSizeDataMap.java deleted file mode 100644 index 59e7c5f6..00000000 --- a/baremaps-core/src/main/java/org/apache/baremaps/database/collection/MonotonicFixedSizeDataMap.java +++ /dev/null @@ -1,157 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except - * in compliance with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License - * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. See the License for the specific language governing permissions and limitations under - * the License. - */ - -package org.apache.baremaps.database.collection; - - - -import com.google.common.collect.Streams; -import java.util.Iterator; -import java.util.Map; -import org.apache.baremaps.database.type.LongDataType; - -/** - * A map that can hold a large number of fixed-size data elements. The elements must be sorted by - * their key and inserted in a monotonic way. The elements cannot be removed or updated once - * inserted. - * - * <p> - * This code has been adapted from Planetiler (Apache license). - * - * <p> - * Copyright (c) Planetiler. - */ -public class MonotonicFixedSizeDataMap<E> extends DataMap<E> { - - private final DataList<Long> offsets; - private final DataList<Long> keys; - private final DataList<E> values; - private long lastChunk = -1; - - /** - * Constructs a map with default lists for storing offsets and keys. - * - * @param values the list of values - */ - public MonotonicFixedSizeDataMap(DataList<E> values) { - this(new MemoryAlignedDataList<>(new LongDataType()), - new MemoryAlignedDataList<>(new LongDataType()), values); - } - - /** - * Constructs a map. - * - * @param offsets the list of offsets - * @param keys the list of keys - * @param values the list of values - */ - public MonotonicFixedSizeDataMap( - DataList<Long> offsets, - DataList<Long> keys, - DataList<E> values) { - this.offsets = offsets; - this.keys = keys; - this.values = values; - } - - /** {@inheritDoc} */ - public E get(Object keyObject) { - long key = (long) keyObject; - long chunk = key >>> 8; - if (chunk >= offsets.size()) { - return null; - } - long lo = offsets.get(chunk); - long hi = Math.min(keys.sizeAsLong(), - chunk >= offsets.sizeAsLong() - 1 ? keys.sizeAsLong() : offsets.get(chunk + 1)) - 1; - while (lo <= hi) { - long index = (lo + hi) >>> 1; - long value = keys.get(index); - if (value < key) { - lo = index + 1; - } else if (value > key) { - hi = index - 1; - } else { - return values.get(index); - } - } - return null; - } - - /** {@inheritDoc} */ - public E put(Long key, E value) { - long size = keys.sizeAsLong(); - long chunk = key >>> 8; - if (chunk != lastChunk) { - while (offsets.sizeAsLong() <= chunk) { - offsets.add(size); - } - lastChunk = chunk; - } - keys.add(key); - values.add(value); - return null; - } - - /** {@inheritDoc} */ - @Override - public E remove(Object key) { - throw new UnsupportedOperationException(); - } - - /** {@inheritDoc} */ - @Override - public long sizeAsLong() { - return keys.sizeAsLong(); - } - - /** {@inheritDoc} */ - @Override - public boolean containsKey(Object key) { - return keys.contains(key); - } - - /** {@inheritDoc} */ - @Override - public boolean containsValue(Object value) { - return values.contains(value); - } - - /** {@inheritDoc} */ - @Override - protected Iterator<Long> keyIterator() { - return keys.iterator(); - } - - /** {@inheritDoc} */ - @Override - protected Iterator<E> valueIterator() { - return values.iterator(); - } - - /** {@inheritDoc} */ - @Override - protected Iterator<Entry<Long, E>> entryIterator() { - return Streams.zip( - Streams.stream(keyIterator()), - Streams.stream(valueIterator()), - (k, v) -> Map.entry(k, v)).iterator(); - } - - /** {@inheritDoc} */ - @Override - public void clear() { - offsets.clear(); - keys.clear(); - values.clear(); - } -} diff --git a/baremaps-core/src/main/java/org/apache/baremaps/database/collection/MonotonicPairedDataMap.java b/baremaps-core/src/main/java/org/apache/baremaps/database/collection/MonotonicPairedDataMap.java deleted file mode 100644 index 60100f85..00000000 --- a/baremaps-core/src/main/java/org/apache/baremaps/database/collection/MonotonicPairedDataMap.java +++ /dev/null @@ -1,145 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except - * in compliance with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License - * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. See the License for the specific language governing permissions and limitations under - * the License. - */ - -package org.apache.baremaps.database.collection; - - - -import java.util.Iterator; -import java.util.Map; -import org.apache.baremaps.database.type.LongDataType; -import org.apache.baremaps.database.type.PairDataType.Pair; - -/** - * A map that can hold a large number of variable-size data elements. The elements must be sorted by - * their key and inserted in a monotonic way. The elements cannot be removed or updated once - * inserted. - */ -public class MonotonicPairedDataMap<E> extends DataMap<E> { - - private final DataList<Long> offsets; - private final MemoryAlignedDataList<Pair<Long, E>> values; - - private long lastChunk = -1; - - public MonotonicPairedDataMap(MemoryAlignedDataList<Pair<Long, E>> values) { - this(new MemoryAlignedDataList<>(new LongDataType()), values); - } - - /** - * Constructs a map. - * - * @param offsets the list of offsets - * @param values the buffer of values - */ - public MonotonicPairedDataMap(DataList<Long> offsets, - MemoryAlignedDataList<Pair<Long, E>> values) { - this.offsets = offsets; - this.values = values; - } - - /** {@inheritDoc} */ - public E put(Long key, E value) { - long index = values.sizeAsLong(); - long chunk = key >>> 8; - if (chunk != lastChunk) { - while (offsets.sizeAsLong() <= chunk) { - offsets.add(index); - } - lastChunk = chunk; - } - values.add(new Pair<>(key, value)); - return null; - } - - /** {@inheritDoc} */ - public E get(Object keyObject) { - long key = (long) keyObject; - long chunk = key >>> 8; - if (chunk >= offsets.sizeAsLong()) { - return null; - } - long lo = offsets.get(chunk); - long hi = - Math.min( - values.sizeAsLong(), - chunk >= offsets.sizeAsLong() - 1 - ? values.sizeAsLong() - : offsets.get(chunk + 1)) - - 1; - while (lo <= hi) { - long index = (lo + hi) >>> 1; - Pair<Long, E> pair = values.get(index); - long value = pair.left(); - if (value < key) { - lo = index + 1; - } else if (value > key) { - hi = index - 1; - } else { - // found - return pair.right(); - } - } - return null; - } - - /** {@inheritDoc} */ - @Override - protected Iterator<Long> keyIterator() { - return values.stream().map(Pair::left).iterator(); - } - - /** {@inheritDoc} */ - @Override - protected Iterator<E> valueIterator() { - return values.stream().map(Pair::right).iterator(); - } - - /** {@inheritDoc} */ - @Override - protected Iterator<Entry<Long, E>> entryIterator() { - return values.stream() - .map(p -> Map.entry(p.left(), p.right())) - .iterator(); - } - - /** {@inheritDoc} */ - @Override - public long sizeAsLong() { - return values.sizeAsLong(); - } - - /** {@inheritDoc} */ - @Override - public boolean containsKey(Object key) { - return get(key) != null; - } - - /** {@inheritDoc} */ - @Override - public boolean containsValue(Object value) { - return values.stream().anyMatch(p -> p.right().equals(value)); - } - - /** {@inheritDoc} */ - @Override - public E remove(Object key) { - throw new UnsupportedOperationException(); - } - - /** {@inheritDoc} */ - @Override - public void clear() { - offsets.clear(); - values.clear(); - } -} diff --git a/baremaps-core/src/test/java/org/apache/baremaps/database/DataMapTest.java b/baremaps-core/src/test/java/org/apache/baremaps/database/DataMapTest.java index 0ccac5c5..a91e8bb7 100644 --- a/baremaps-core/src/test/java/org/apache/baremaps/database/DataMapTest.java +++ b/baremaps-core/src/test/java/org/apache/baremaps/database/DataMapTest.java @@ -161,14 +161,10 @@ class DataMapTest { Arguments.of( new IndexedDataMap<>( new AppendOnlyBuffer<>(new LongDataType(), new OffHeapMemory()))), - Arguments.of(new MonotonicFixedSizeDataMap<>( - new MemoryAlignedDataList<>(new LongDataType(), new OffHeapMemory()))), Arguments.of(new MonotonicDataMap<>( new MemoryAlignedDataList<>( new PairDataType<>(new LongDataType(), new LongDataType()), new OffHeapMemory()), - new AppendOnlyBuffer<>(new LongDataType(), new OffHeapMemory()))), - Arguments.of(new MonotonicPairedDataMap<>(new MemoryAlignedDataList<>( - new PairDataType<>(new LongDataType(), new LongDataType()))))); + new AppendOnlyBuffer<>(new LongDataType(), new OffHeapMemory())))); } }
