This is an automated email from the ASF dual-hosted git repository. bchapuis pushed a commit to branch collection in repository https://gitbox.apache.org/repos/asf/incubator-baremaps.git
commit e6e4085f2ac67d54015fead66fd7e9747c7f19e4 Author: Bertil Chapuis <[email protected]> AuthorDate: Fri May 3 23:33:04 2024 +0200 Rename buffer into log and implements iterable --- .../{AppendOnlyBuffer.java => AppendOnlyLog.java} | 58 ++++++++++++++-------- .../database/collection/IndexedDataList.java | 10 ++-- .../database/collection/IndexedDataMap.java | 8 +-- .../baremaps/database/collection/LongDataMap.java | 46 ----------------- .../database/collection/MonotonicDataMap.java | 10 ++-- .../postgres/PostgresReferenceMap.java | 1 - .../apache/baremaps/workflow/WorkflowContext.java | 2 +- ...dOnlyBufferTest.java => AppendOnlyLogTest.java} | 16 +++--- .../org/apache/baremaps/database/DataListTest.java | 2 +- .../org/apache/baremaps/database/DataMapTest.java | 4 +- .../baremaps/database/calcite/CalciteTest.java | 6 +-- .../database/sort/ExternalMergeSortTest.java | 4 +- .../workflow/tasks/ImportUpdateSampleTest.java | 6 +-- 13 files changed, 70 insertions(+), 103 deletions(-) diff --git a/baremaps-core/src/main/java/org/apache/baremaps/database/collection/AppendOnlyBuffer.java b/baremaps-core/src/main/java/org/apache/baremaps/database/collection/AppendOnlyLog.java similarity index 82% rename from baremaps-core/src/main/java/org/apache/baremaps/database/collection/AppendOnlyBuffer.java rename to baremaps-core/src/main/java/org/apache/baremaps/database/collection/AppendOnlyLog.java index 0c2a4d3e..e3f41709 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/database/collection/AppendOnlyBuffer.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/database/collection/AppendOnlyLog.java @@ -30,16 +30,16 @@ import org.apache.baremaps.database.memory.OffHeapMemory; import org.apache.baremaps.database.type.DataType; /** - * A buffer of elements backed by a {@link DataType} and a {@link Memory}. Elements are appended to - * the buffer and can be accessed by their position in the {@link Memory}. Appending elements to the + * A log of records backed by a {@link DataType} and a {@link Memory}. Elements are appended to the + * buffer and can be accessed by their position in the {@link Memory}. Appending elements to the * buffer is thread-safe. * * @param <E> The type of the data. */ -public class AppendOnlyBuffer<E> extends AbstractDataCollection<E> { +public class AppendOnlyLog<E> implements Iterable<E> { private final DataType<E> dataType; - private final Memory memory; + private final Memory<?> memory; private final long segmentSize; private long offset; private long size; @@ -51,7 +51,7 @@ public class AppendOnlyBuffer<E> extends AbstractDataCollection<E> { * * @param dataType the data type */ - public AppendOnlyBuffer(DataType<E> dataType) { + public AppendOnlyLog(DataType<E> dataType) { this(dataType, new OffHeapMemory()); } @@ -61,7 +61,7 @@ public class AppendOnlyBuffer<E> extends AbstractDataCollection<E> { * @param dataType the data type * @param memory the memory */ - public AppendOnlyBuffer(DataType<E> dataType, Memory memory) { + public AppendOnlyLog(DataType<E> dataType, Memory<?> memory) { this.dataType = dataType; this.memory = memory; this.segmentSize = memory.segmentSize(); @@ -75,7 +75,7 @@ public class AppendOnlyBuffer<E> extends AbstractDataCollection<E> { * @param value the value * @return the position of the value in the memory. */ - public long addPositioned(E value) { + public long add(E value) { int valueSize = dataType.size(value); if (valueSize > segmentSize) { throw new DataCollectionException("The value is too big to fit in a segment"); @@ -101,15 +101,6 @@ public class AppendOnlyBuffer<E> extends AbstractDataCollection<E> { return position; } - /** - * {@inheritDoc} - */ - @Override - public boolean add(E value) { - addPositioned(value); - return true; - } - /** * Returns a values at the specified position in the memory. * @@ -123,17 +114,25 @@ public class AppendOnlyBuffer<E> extends AbstractDataCollection<E> { return dataType.read(buffer, (int) segmentOffset); } - /** {@inheritDoc} */ - public long sizeAsLong() { + /** + * Returns the size of the log. + * + * @return the size of the log + */ + public long size() { return size; } + /** + * Closes the log. + */ public void close() { memory.segment(0).putLong(0, size); } - /** {@inheritDoc} */ - @Override + /** + * Clears the log. + */ public void clear() { try { memory.clear(); @@ -143,14 +142,29 @@ public class AppendOnlyBuffer<E> extends AbstractDataCollection<E> { } /** - * {@inheritDoc} + * Returns an iterator over the values of the log. + * @return an iterator over the values of the log */ @Override public BufferIterator iterator() { - final long size = sizeAsLong(); + final long size = size(); return new BufferIterator(size); } + /** + * Returns true if the log contains the specified value. + * @param value the value + * @return true if the log contains the specified value + */ + public boolean contains(Object value) { + for (E e : this) { + if (e.equals(value)) { + return true; + } + } + return false; + } + public class BufferIterator implements Iterator<E> { private final long size; diff --git a/baremaps-core/src/main/java/org/apache/baremaps/database/collection/IndexedDataList.java b/baremaps-core/src/main/java/org/apache/baremaps/database/collection/IndexedDataList.java index 49d5abd7..5bb51c63 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/database/collection/IndexedDataList.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/database/collection/IndexedDataList.java @@ -32,7 +32,7 @@ public class IndexedDataList<E> extends DataList<E> { private final DataList<Long> index; - private final AppendOnlyBuffer<E> values; + private final AppendOnlyLog<E> values; /** @@ -40,7 +40,7 @@ public class IndexedDataList<E> extends DataList<E> { * * @param values the values */ - public IndexedDataList(AppendOnlyBuffer<E> values) { + public IndexedDataList(AppendOnlyLog<E> values) { this(new MemoryAlignedDataList<>(new LongDataType()), values); } @@ -50,7 +50,7 @@ public class IndexedDataList<E> extends DataList<E> { * @param index the index * @param values the values */ - public IndexedDataList(DataList<Long> index, AppendOnlyBuffer<E> values) { + public IndexedDataList(DataList<Long> index, AppendOnlyLog<E> values) { this.index = index; this.values = values; } @@ -60,7 +60,7 @@ public class IndexedDataList<E> extends DataList<E> { */ @Override public long addIndexed(E value) { - long position = values.addPositioned(value); + long position = values.add(value); return index.addIndexed(position); } @@ -69,7 +69,7 @@ public class IndexedDataList<E> extends DataList<E> { */ @Override public void set(long index, E value) { - long position = values.addPositioned(value); + long position = values.add(value); this.index.set(index, position); } diff --git a/baremaps-core/src/main/java/org/apache/baremaps/database/collection/IndexedDataMap.java b/baremaps-core/src/main/java/org/apache/baremaps/database/collection/IndexedDataMap.java index b8d10a11..610e67ef 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/database/collection/IndexedDataMap.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/database/collection/IndexedDataMap.java @@ -35,14 +35,14 @@ public class IndexedDataMap<E> extends DataMap<Long, E> { private final Map<Long, Long> index; - private final AppendOnlyBuffer<E> values; + private final AppendOnlyLog<E> values; /** * Constructs a map. * * @param values the values */ - public IndexedDataMap(AppendOnlyBuffer<E> values) { + public IndexedDataMap(AppendOnlyLog<E> values) { this(new Long2LongOpenHashMap(), values); } @@ -52,7 +52,7 @@ public class IndexedDataMap<E> extends DataMap<Long, E> { * @param index the index * @param values the values */ - public IndexedDataMap(Map<Long, Long> index, AppendOnlyBuffer<E> values) { + public IndexedDataMap(Map<Long, Long> index, AppendOnlyLog<E> values) { this.index = index; this.values = values; } @@ -63,7 +63,7 @@ public class IndexedDataMap<E> extends DataMap<Long, E> { @Override public E put(Long key, E value) { var oldIndex = index.get(key); - var position = values.addPositioned(value); + var position = values.add(value); index.put(key, position); return oldIndex == null ? null : values.read(oldIndex); } diff --git a/baremaps-core/src/main/java/org/apache/baremaps/database/collection/LongDataMap.java b/baremaps-core/src/main/java/org/apache/baremaps/database/collection/LongDataMap.java deleted file mode 100644 index e94172b4..00000000 --- a/baremaps-core/src/main/java/org/apache/baremaps/database/collection/LongDataMap.java +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to you under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.baremaps.database.collection; - - - -import org.apache.baremaps.database.memory.Memory; -import org.apache.baremaps.database.memory.OffHeapMemory; -import org.apache.baremaps.database.type.LongDataType; - -/** - * A list of longs. - */ -public class LongDataMap extends MemoryAlignedDataMap<Long> { - - /** - * Constructs a list. - */ - public LongDataMap() { - this(new OffHeapMemory()); - } - - /** - * Constructs a list. - * - * @param memory the memory - */ - public LongDataMap(Memory memory) { - super(new LongDataType(), memory); - } -} diff --git a/baremaps-core/src/main/java/org/apache/baremaps/database/collection/MonotonicDataMap.java b/baremaps-core/src/main/java/org/apache/baremaps/database/collection/MonotonicDataMap.java index 17c8d8c1..01f24842 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/database/collection/MonotonicDataMap.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/database/collection/MonotonicDataMap.java @@ -40,7 +40,7 @@ public class MonotonicDataMap<E> extends DataMap<Long, E> { private final DataList<Long> offsets; private final DataList<Pair<Long, Long>> keys; - private final AppendOnlyBuffer<E> values; + private final AppendOnlyLog<E> values; private long lastChunk = -1; @@ -49,7 +49,7 @@ public class MonotonicDataMap<E> extends DataMap<Long, E> { * * @param values the buffer of values */ - public MonotonicDataMap(AppendOnlyBuffer<E> values) { + public MonotonicDataMap(AppendOnlyLog<E> values) { this( new MemoryAlignedDataList<>(new LongDataType()), new MemoryAlignedDataList<>(new PairDataType<>(new LongDataType(), new LongDataType())), @@ -62,7 +62,7 @@ public class MonotonicDataMap<E> extends DataMap<Long, E> { * @param keys the list of keys * @param values the buffer of values */ - public MonotonicDataMap(DataList<Pair<Long, Long>> keys, AppendOnlyBuffer<E> values) { + public MonotonicDataMap(DataList<Pair<Long, Long>> keys, AppendOnlyLog<E> values) { this( new MemoryAlignedDataList<>(new LongDataType()), keys, @@ -77,7 +77,7 @@ public class MonotonicDataMap<E> extends DataMap<Long, E> { * @param values the buffer of values */ public MonotonicDataMap(DataList<Long> offsets, DataList<Pair<Long, Long>> keys, - AppendOnlyBuffer<E> values) { + AppendOnlyLog<E> values) { this.offsets = offsets; this.keys = keys; this.values = values; @@ -93,7 +93,7 @@ public class MonotonicDataMap<E> extends DataMap<Long, E> { } lastChunk = chunk; } - long position = values.addPositioned(value); + long position = values.add(value); keys.add(new Pair<>(key, position)); return null; } diff --git a/baremaps-core/src/main/java/org/apache/baremaps/openstreetmap/postgres/PostgresReferenceMap.java b/baremaps-core/src/main/java/org/apache/baremaps/openstreetmap/postgres/PostgresReferenceMap.java index 2dd4ab68..7ab8bc66 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/openstreetmap/postgres/PostgresReferenceMap.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/openstreetmap/postgres/PostgresReferenceMap.java @@ -27,7 +27,6 @@ import java.sql.SQLException; import java.util.*; import javax.sql.DataSource; import org.apache.baremaps.database.collection.DataCollectionException; -import org.apache.baremaps.database.collection.DataMap; /** * A read-only {@code LongDataMap} for references baked by OpenStreetMap ways stored in Postgres. diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/WorkflowContext.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/WorkflowContext.java index 172be38d..b16f52f6 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/WorkflowContext.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/WorkflowContext.java @@ -89,7 +89,7 @@ public class WorkflowContext { new MemoryAlignedDataList<>( new PairDataType<>(new LongDataType(), new LongDataType()), new MemoryMappedDirectory(keysDir)), - new AppendOnlyBuffer<>( + new AppendOnlyLog<>( dataType, new MemoryMappedDirectory(valuesDir))); } diff --git a/baremaps-core/src/test/java/org/apache/baremaps/database/AppendOnlyBufferTest.java b/baremaps-core/src/test/java/org/apache/baremaps/database/AppendOnlyLogTest.java similarity index 86% rename from baremaps-core/src/test/java/org/apache/baremaps/database/AppendOnlyBufferTest.java rename to baremaps-core/src/test/java/org/apache/baremaps/database/AppendOnlyLogTest.java index 96132075..70554f4c 100644 --- a/baremaps-core/src/test/java/org/apache/baremaps/database/AppendOnlyBufferTest.java +++ b/baremaps-core/src/test/java/org/apache/baremaps/database/AppendOnlyLogTest.java @@ -22,7 +22,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import java.util.ArrayList; import java.util.Random; -import org.apache.baremaps.database.collection.AppendOnlyBuffer; +import org.apache.baremaps.database.collection.AppendOnlyLog; import org.apache.baremaps.database.memory.OffHeapMemory; import org.apache.baremaps.database.type.DataType; import org.apache.baremaps.database.type.IntegerDataType; @@ -31,13 +31,13 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.MethodSource; -class AppendOnlyBufferTest { +class AppendOnlyLogTest { @Test void addFixedSizeData() { - var collection = new AppendOnlyBuffer<>(new IntegerDataType(), new OffHeapMemory(1 << 10)); + var collection = new AppendOnlyLog<>(new IntegerDataType(), new OffHeapMemory(1 << 10)); for (int i = 0; i < 1 << 20; i++) { - assertEquals(Long.BYTES + (i << 2), collection.addPositioned(i)); + assertEquals(Long.BYTES + (i << 2), collection.add(i)); } for (int i = 0; i < 1 << 20; i++) { assertEquals(i, collection.read(Long.BYTES + (i << 2))); @@ -46,7 +46,7 @@ class AppendOnlyBufferTest { @Test void addVariableSizeValues() { - var collection = new AppendOnlyBuffer<>(new IntegerListDataType(), new OffHeapMemory(1 << 10)); + var collection = new AppendOnlyLog<>(new IntegerListDataType(), new OffHeapMemory(1 << 10)); var random = new Random(0); var positions = new ArrayList<Long>(); var values = new ArrayList<ArrayList<Integer>>(); @@ -56,7 +56,7 @@ class AppendOnlyBufferTest { for (int j = 0; j < size; j++) { value.add(random.nextInt(1 << 20)); } - positions.add(collection.addPositioned(value)); + positions.add(collection.add(value)); values.add(value); } for (int i = 0; i < positions.size(); i++) { @@ -69,11 +69,11 @@ class AppendOnlyBufferTest { @MethodSource("org.apache.baremaps.database.type.DataTypeProvider#dataTypes") void testAllDataTypes(DataType dataType, Object value) { var num = 1000; - var collection = new AppendOnlyBuffer<>(dataType, new OffHeapMemory(1 << 22)); + var collection = new AppendOnlyLog<>(dataType, new OffHeapMemory(1 << 22)); // write values for (int i = 0; i < num; i++) { - collection.addPositioned(value); + collection.add(value); } collection.close(); diff --git a/baremaps-core/src/test/java/org/apache/baremaps/database/DataListTest.java b/baremaps-core/src/test/java/org/apache/baremaps/database/DataListTest.java index d6607671..45c8339d 100644 --- a/baremaps-core/src/test/java/org/apache/baremaps/database/DataListTest.java +++ b/baremaps-core/src/test/java/org/apache/baremaps/database/DataListTest.java @@ -77,7 +77,7 @@ class DataListTest { return Stream.of( Arguments.of(new FixedSizeDataList<>(new LongDataType())), Arguments.of(new IndexedDataList<>(new MemoryAlignedDataList<>(new LongDataType()), - new AppendOnlyBuffer<>(new LongDataType()))), + new AppendOnlyLog<>(new LongDataType()))), Arguments.of(new MemoryAlignedDataList<>(new LongDataType()))); } } diff --git a/baremaps-core/src/test/java/org/apache/baremaps/database/DataMapTest.java b/baremaps-core/src/test/java/org/apache/baremaps/database/DataMapTest.java index 531f85e4..c09203dc 100644 --- a/baremaps-core/src/test/java/org/apache/baremaps/database/DataMapTest.java +++ b/baremaps-core/src/test/java/org/apache/baremaps/database/DataMapTest.java @@ -165,14 +165,14 @@ class DataMapTest { .of( Arguments.of( new IndexedDataMap<>( - new AppendOnlyBuffer<>(new LongDataType(), new OffHeapMemory()))), + new AppendOnlyLog<>(new LongDataType(), new OffHeapMemory()))), Arguments.of(new MonotonicFixedSizeDataMap<>( new MemoryAlignedDataList<>(new LongDataType(), new OffHeapMemory()))), Arguments.of(new MonotonicDataMap<>( new MemoryAlignedDataList<>( new PairDataType<>(new LongDataType(), new LongDataType()), new OffHeapMemory()), - new AppendOnlyBuffer<>(new LongDataType(), new OffHeapMemory()))), + new AppendOnlyLog<>(new LongDataType(), new OffHeapMemory()))), Arguments.of(new MonotonicPairedDataMap<>(new MemoryAlignedDataList<>( new PairDataType<>(new LongDataType(), new LongDataType()))))); } diff --git a/baremaps-core/src/test/java/org/apache/baremaps/database/calcite/CalciteTest.java b/baremaps-core/src/test/java/org/apache/baremaps/database/calcite/CalciteTest.java index 1a541b70..1a681d8b 100644 --- a/baremaps-core/src/test/java/org/apache/baremaps/database/calcite/CalciteTest.java +++ b/baremaps-core/src/test/java/org/apache/baremaps/database/calcite/CalciteTest.java @@ -24,7 +24,7 @@ import com.google.common.collect.ImmutableList; import java.sql.*; import java.util.List; import java.util.Properties; -import org.apache.baremaps.database.collection.AppendOnlyBuffer; +import org.apache.baremaps.database.collection.AppendOnlyLog; import org.apache.baremaps.database.collection.IndexedDataList; import org.apache.baremaps.database.schema.*; import org.apache.baremaps.database.schema.DataColumn.Type; @@ -81,7 +81,7 @@ public class CalciteTest { new DataColumnImpl("geometry", Type.GEOMETRY))); DataTable cityDataTable = new DataTableImpl( cityRowType, - new IndexedDataList<>(new AppendOnlyBuffer<>(new RowDataType(cityRowType)))); + new IndexedDataList<>(new AppendOnlyLog<>(new RowDataType(cityRowType)))); cityDataTable.add(new DataRowImpl(cityDataTable.rowType(), List.of(1, "Paris", geometryFactory.createPoint(new Coordinate(2.3522, 48.8566))))); cityDataTable.add(new DataRowImpl(cityDataTable.rowType(), @@ -95,7 +95,7 @@ public class CalciteTest { new DataColumnImpl("population", Type.INTEGER))); DataTable populationDataTable = new DataTableImpl( populationRowType, - new IndexedDataList<>(new AppendOnlyBuffer<>(new RowDataType(populationRowType)))); + new IndexedDataList<>(new AppendOnlyLog<>(new RowDataType(populationRowType)))); populationDataTable .add(new DataRowImpl(populationDataTable.rowType(), List.of(1, 2_161_000))); populationDataTable diff --git a/baremaps-core/src/test/java/org/apache/baremaps/database/sort/ExternalMergeSortTest.java b/baremaps-core/src/test/java/org/apache/baremaps/database/sort/ExternalMergeSortTest.java index 059e97a7..f5a4208a 100644 --- a/baremaps-core/src/test/java/org/apache/baremaps/database/sort/ExternalMergeSortTest.java +++ b/baremaps-core/src/test/java/org/apache/baremaps/database/sort/ExternalMergeSortTest.java @@ -27,7 +27,7 @@ import java.util.List; import java.util.Random; import java.util.function.Supplier; import org.apache.baremaps.database.algorithm.ExternalMergeSort; -import org.apache.baremaps.database.collection.AppendOnlyBuffer; +import org.apache.baremaps.database.collection.AppendOnlyLog; import org.apache.baremaps.database.collection.DataList; import org.apache.baremaps.database.collection.IndexedDataList; import org.apache.baremaps.database.collection.MemoryAlignedDataList; @@ -51,7 +51,7 @@ class ExternalMergeSortTest { void before() { supplier = () -> new IndexedDataList<>( new MemoryAlignedDataList<>(new LongDataType(), new OnHeapMemory()), - new AppendOnlyBuffer<>(new StringDataType(), new OnHeapMemory())); + new AppendOnlyLog<>(new StringDataType(), new OnHeapMemory())); input = supplier.get(); output = supplier.get(); for (var string : strings) { diff --git a/baremaps-core/src/test/java/org/apache/baremaps/workflow/tasks/ImportUpdateSampleTest.java b/baremaps-core/src/test/java/org/apache/baremaps/workflow/tasks/ImportUpdateSampleTest.java index 38df3c91..125b24f0 100644 --- a/baremaps-core/src/test/java/org/apache/baremaps/workflow/tasks/ImportUpdateSampleTest.java +++ b/baremaps-core/src/test/java/org/apache/baremaps/workflow/tasks/ImportUpdateSampleTest.java @@ -25,7 +25,7 @@ import static org.junit.jupiter.api.Assertions.assertNull; import java.nio.file.Files; import java.util.List; import java.util.Map; -import org.apache.baremaps.database.collection.AppendOnlyBuffer; +import org.apache.baremaps.database.collection.AppendOnlyLog; import org.apache.baremaps.database.collection.IndexedDataMap; import org.apache.baremaps.database.memory.OnHeapMemory; import org.apache.baremaps.database.type.LongListDataType; @@ -59,9 +59,9 @@ class ImportUpdateSampleTest extends PostgresRepositoryTest { // Initialize the data maps Map<Long, Coordinate> coordinateMap = - new IndexedDataMap<>(new AppendOnlyBuffer<>(new CoordinateDataType(), new OnHeapMemory())); + new IndexedDataMap<>(new AppendOnlyLog<>(new CoordinateDataType(), new OnHeapMemory())); Map<Long, List<Long>> referenceMap = - new IndexedDataMap<>(new AppendOnlyBuffer<>(new LongListDataType(), new OnHeapMemory())); + new IndexedDataMap<>(new AppendOnlyLog<>(new LongListDataType(), new OnHeapMemory())); // Import the sample data ImportOsmPbf.execute(OsmSample.SAMPLE_OSM_PBF, coordinateMap, referenceMap, headerRepository,
