This is an automated email from the ASF dual-hosted git repository.
jackie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 32a02bc1a3 Fix some resource leak in tests (#12794)
32a02bc1a3 is described below
commit 32a02bc1a31de0ff02851e0b983df69e15bdd271
Author: Xiaotian (Jackie) Jiang <[email protected]>
AuthorDate: Sun Apr 7 18:21:24 2024 -0700
Fix some resource leak in tests (#12794)
---
...centileSmartTDigestAggregationFunctionTest.java | 5 -
...MutableSegmentImplIngestionAggregationTest.java | 2 +
.../impl/VarByteChunkSVForwardIndexWriterTest.java | 10 +-
.../local/segment/index/ColumnMetadataTest.java | 18 +--
.../MultiValueVarByteRawIndexCreatorTest.java | 36 ++---
.../SegmentGenerationWithBytesTypeTest.java | 155 ++++++++-------------
.../segment/spi/memory/unsafe/DirectMemory.java | 14 +-
.../segment/spi/memory/unsafe/MmapMemory.java | 13 +-
8 files changed, 99 insertions(+), 154 deletions(-)
diff --git
a/pinot-core/src/test/java/org/apache/pinot/core/query/aggregation/function/PercentileSmartTDigestAggregationFunctionTest.java
b/pinot-core/src/test/java/org/apache/pinot/core/query/aggregation/function/PercentileSmartTDigestAggregationFunctionTest.java
index b1eb471c70..68a180ea88 100644
---
a/pinot-core/src/test/java/org/apache/pinot/core/query/aggregation/function/PercentileSmartTDigestAggregationFunctionTest.java
+++
b/pinot-core/src/test/java/org/apache/pinot/core/query/aggregation/function/PercentileSmartTDigestAggregationFunctionTest.java
@@ -78,10 +78,5 @@ public class PercentileSmartTDigestAggregationFunctionTest {
String expectedAggrWithoutNull90(Scenario scenario) {
return "7.100000000000001";
}
-
- @Override
- String expectedAggrWithoutNull100(Scenario scenario) {
- return super.expectedAggrWithoutNull100(scenario);
- }
}
}
diff --git
a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/indexsegment/mutable/MutableSegmentImplIngestionAggregationTest.java
b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/indexsegment/mutable/MutableSegmentImplIngestionAggregationTest.java
index 5e048520c0..711634ea17 100644
---
a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/indexsegment/mutable/MutableSegmentImplIngestionAggregationTest.java
+++
b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/indexsegment/mutable/MutableSegmentImplIngestionAggregationTest.java
@@ -480,6 +480,8 @@ public class MutableSegmentImplIngestionAggregationTest {
Assert.assertThrows(IllegalArgumentException.class, () -> {
mutableSegmentImpl.index(row, defaultMetadata);
});
+
+ mutableSegmentImpl.destroy();
}
private BigDecimal generateRandomBigDecimal(Random random, int maxPrecision,
int scale) {
diff --git
a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/io/writer/impl/VarByteChunkSVForwardIndexWriterTest.java
b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/io/writer/impl/VarByteChunkSVForwardIndexWriterTest.java
index 5a46c9d3f3..a9b8c34bb5 100644
---
a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/io/writer/impl/VarByteChunkSVForwardIndexWriterTest.java
+++
b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/io/writer/impl/VarByteChunkSVForwardIndexWriterTest.java
@@ -35,7 +35,7 @@ import
org.apache.pinot.segment.local.segment.index.readers.forward.VarByteChunk
import org.apache.pinot.segment.spi.V1Constants;
import org.apache.pinot.segment.spi.compression.ChunkCompressionType;
import org.apache.pinot.segment.spi.memory.PinotDataBuffer;
-import org.apache.pinot.spi.data.FieldSpec;
+import org.apache.pinot.spi.data.FieldSpec.DataType;
import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.DataProvider;
@@ -90,8 +90,8 @@ public class VarByteChunkSVForwardIndexWriterTest {
writer.putStringMV(array);
}
}
- try (VarByteChunkSVForwardIndexReader reader = new
VarByteChunkSVForwardIndexReader(
- PinotDataBuffer.loadBigEndianFile(file), FieldSpec.DataType.STRING);
+ try (PinotDataBuffer dataBuffer = PinotDataBuffer.loadBigEndianFile(file);
+ VarByteChunkSVForwardIndexReader reader = new
VarByteChunkSVForwardIndexReader(dataBuffer, DataType.STRING);
ChunkReaderContext context = reader.createContext()) {
for (int i = 0; i < arrays.size(); i++) {
String[] array = arrays.get(i);
@@ -125,8 +125,8 @@ public class VarByteChunkSVForwardIndexWriterTest {
writer.putBytesMV(Arrays.stream(array).map(str ->
str.getBytes(UTF_8)).toArray(byte[][]::new));
}
}
- try (VarByteChunkSVForwardIndexReader reader = new
VarByteChunkSVForwardIndexReader(
- PinotDataBuffer.loadBigEndianFile(file), FieldSpec.DataType.BYTES);
+ try (PinotDataBuffer dataBuffer = PinotDataBuffer.loadBigEndianFile(file);
+ VarByteChunkSVForwardIndexReader reader = new
VarByteChunkSVForwardIndexReader(dataBuffer, DataType.BYTES);
ChunkReaderContext context = reader.createContext()) {
for (int i = 0; i < arrays.size(); i++) {
String[] array = arrays.get(i);
diff --git
a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/ColumnMetadataTest.java
b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/ColumnMetadataTest.java
index b8c1f58519..dbee83216c 100644
---
a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/ColumnMetadataTest.java
+++
b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/ColumnMetadataTest.java
@@ -29,22 +29,20 @@ import java.util.stream.Stream;
import org.apache.commons.configuration2.PropertiesConfiguration;
import org.apache.commons.configuration2.ex.ConfigurationException;
import org.apache.commons.io.FileUtils;
-import
org.apache.pinot.segment.local.indexsegment.immutable.ImmutableSegmentLoader;
import org.apache.pinot.segment.local.segment.creator.SegmentTestUtils;
import
org.apache.pinot.segment.local.segment.creator.impl.SegmentCreationDriverFactory;
import org.apache.pinot.segment.spi.ColumnMetadata;
-import org.apache.pinot.segment.spi.IndexSegment;
import org.apache.pinot.segment.spi.SegmentMetadata;
import org.apache.pinot.segment.spi.creator.SegmentGeneratorConfig;
import org.apache.pinot.segment.spi.creator.SegmentIndexCreationDriver;
import org.apache.pinot.segment.spi.index.metadata.ColumnMetadataImpl;
+import org.apache.pinot.segment.spi.index.metadata.SegmentMetadataImpl;
import
org.apache.pinot.segment.spi.partition.BoundedColumnValuePartitionFunction;
import org.apache.pinot.spi.config.table.ColumnPartitionConfig;
import org.apache.pinot.spi.config.table.SegmentPartitionConfig;
import org.apache.pinot.spi.data.DimensionFieldSpec;
import org.apache.pinot.spi.data.FieldSpec.DataType;
import org.apache.pinot.spi.env.CommonsConfigurationUtils;
-import org.apache.pinot.spi.utils.ReadMode;
import org.apache.pinot.util.TestUtils;
import org.testng.Assert;
import org.testng.annotations.AfterMethod;
@@ -140,8 +138,7 @@ public class ColumnMetadataTest {
driver.build();
// Load segment metadata.
- IndexSegment segment =
ImmutableSegmentLoader.load(INDEX_DIR.listFiles()[0], ReadMode.mmap);
- SegmentMetadata segmentMetadata = segment.getSegmentMetadata();
+ SegmentMetadata segmentMetadata = new
SegmentMetadataImpl(INDEX_DIR.listFiles()[0]);
verifySegmentAfterLoading(segmentMetadata);
// Make sure we got the creator name as well.
@@ -159,8 +156,7 @@ public class ColumnMetadataTest {
driver.build();
// Load segment metadata.
- IndexSegment segment =
ImmutableSegmentLoader.load(INDEX_DIR.listFiles()[0], ReadMode.mmap);
- SegmentMetadata segmentMetadata = segment.getSegmentMetadata();
+ SegmentMetadata segmentMetadata = new
SegmentMetadataImpl(INDEX_DIR.listFiles()[0]);
verifySegmentAfterLoading(segmentMetadata);
// Make sure we get null for creator name.
@@ -177,9 +173,8 @@ public class ColumnMetadataTest {
driver.build();
// Load segment metadata.
- IndexSegment segment =
ImmutableSegmentLoader.load(INDEX_DIR.listFiles()[0], ReadMode.mmap);
- SegmentMetadata metadata = segment.getSegmentMetadata();
- verifySegmentAfterLoading(metadata);
+ SegmentMetadata segmentMetadata = new
SegmentMetadataImpl(INDEX_DIR.listFiles()[0]);
+ verifySegmentAfterLoading(segmentMetadata);
}
@Test
@@ -198,8 +193,7 @@ public class ColumnMetadataTest {
driver.build();
// Load segment metadata.
- IndexSegment segment =
ImmutableSegmentLoader.load(INDEX_DIR.listFiles()[0], ReadMode.mmap);
- SegmentMetadata segmentMetadata = segment.getSegmentMetadata();
+ SegmentMetadata segmentMetadata = new
SegmentMetadataImpl(INDEX_DIR.listFiles()[0]);
verifySegmentAfterLoading(segmentMetadata);
// Make sure we get null for creator name.
Assert.assertNull(segmentMetadata.getCreatorName());
diff --git
a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/creator/MultiValueVarByteRawIndexCreatorTest.java
b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/creator/MultiValueVarByteRawIndexCreatorTest.java
index 32f4ff1a2c..7b382b4dc5 100644
---
a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/creator/MultiValueVarByteRawIndexCreatorTest.java
+++
b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/creator/MultiValueVarByteRawIndexCreatorTest.java
@@ -126,14 +126,15 @@ public class MultiValueVarByteRawIndexCreatorTest {
}
//read
- final PinotDataBuffer buffer = PinotDataBuffer.mapFile(file, true, 0,
file.length(), ByteOrder.BIG_ENDIAN, "");
- ForwardIndexReader reader =
ForwardIndexReaderFactory.createRawIndexReader(buffer, DataType.STRING, false);
- final ForwardIndexReaderContext context = reader.createContext();
- String[] values = new String[maxElements];
- for (int i = 0; i < numDocs; i++) {
- int length = reader.getStringMV(i, values, context);
- String[] readValue = Arrays.copyOf(values, length);
- Assert.assertEquals(inputs.get(i), readValue);
+ try (PinotDataBuffer buffer = PinotDataBuffer.mapFile(file, true, 0,
file.length(), ByteOrder.BIG_ENDIAN, "");
+ ForwardIndexReader reader =
ForwardIndexReaderFactory.createRawIndexReader(buffer, DataType.STRING, false);
+ ForwardIndexReaderContext context = reader.createContext()) {
+ String[] values = new String[maxElements];
+ for (int i = 0; i < numDocs; i++) {
+ int length = reader.getStringMV(i, values, context);
+ String[] readValue = Arrays.copyOf(values, length);
+ Assert.assertEquals(inputs.get(i), readValue);
+ }
}
}
@@ -177,15 +178,16 @@ public class MultiValueVarByteRawIndexCreatorTest {
}
//read
- final PinotDataBuffer buffer = PinotDataBuffer.mapFile(file, true, 0,
file.length(), ByteOrder.BIG_ENDIAN, "");
- ForwardIndexReader reader =
ForwardIndexReaderFactory.createRawIndexReader(buffer, DataType.BYTES, false);
- final ForwardIndexReaderContext context = reader.createContext();
- byte[][] values = new byte[maxElements][];
- for (int i = 0; i < numDocs; i++) {
- int length = reader.getBytesMV(i, values, context);
- byte[][] readValue = Arrays.copyOf(values, length);
- for (int j = 0; j < length; j++) {
- Assert.assertTrue(Arrays.equals(inputs.get(i)[j], readValue[j]));
+ try (PinotDataBuffer buffer = PinotDataBuffer.mapFile(file, true, 0,
file.length(), ByteOrder.BIG_ENDIAN, "");
+ ForwardIndexReader reader =
ForwardIndexReaderFactory.createRawIndexReader(buffer, DataType.BYTES, false);
+ ForwardIndexReaderContext context = reader.createContext()) {
+ byte[][] values = new byte[maxElements][];
+ for (int i = 0; i < numDocs; i++) {
+ int length = reader.getBytesMV(i, values, context);
+ byte[][] readValue = Arrays.copyOf(values, length);
+ for (int j = 0; j < length; j++) {
+ Assert.assertTrue(Arrays.equals(inputs.get(i)[j], readValue[j]));
+ }
}
}
}
diff --git
a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/creator/SegmentGenerationWithBytesTypeTest.java
b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/creator/SegmentGenerationWithBytesTypeTest.java
index 5c96f4b396..7e33ad173e 100644
---
a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/creator/SegmentGenerationWithBytesTypeTest.java
+++
b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/creator/SegmentGenerationWithBytesTypeTest.java
@@ -27,7 +27,6 @@ import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
-import java.util.Map;
import java.util.Random;
import org.apache.avro.file.DataFileWriter;
import org.apache.avro.generic.GenericData;
@@ -37,7 +36,6 @@ import org.apache.pinot.plugin.inputformat.avro.AvroUtils;
import
org.apache.pinot.segment.local.aggregator.PercentileTDigestValueAggregator;
import
org.apache.pinot.segment.local.indexsegment.immutable.ImmutableSegmentLoader;
import
org.apache.pinot.segment.local.segment.creator.impl.SegmentIndexCreationDriverImpl;
-import org.apache.pinot.segment.local.segment.index.loader.IndexLoadingConfig;
import
org.apache.pinot.segment.local.segment.index.readers.BaseImmutableDictionary;
import org.apache.pinot.segment.local.segment.readers.GenericRowRecordReader;
import org.apache.pinot.segment.local.segment.readers.PinotSegmentRecordReader;
@@ -45,26 +43,25 @@ import org.apache.pinot.segment.spi.ImmutableSegment;
import org.apache.pinot.segment.spi.IndexSegment;
import org.apache.pinot.segment.spi.SegmentMetadata;
import org.apache.pinot.segment.spi.creator.SegmentGeneratorConfig;
-import org.apache.pinot.segment.spi.loader.SegmentDirectoryLoaderContext;
-import org.apache.pinot.segment.spi.loader.SegmentDirectoryLoaderRegistry;
import org.apache.pinot.spi.config.table.TableConfig;
import org.apache.pinot.spi.config.table.TableType;
-import org.apache.pinot.spi.data.DimensionFieldSpec;
-import org.apache.pinot.spi.data.FieldSpec;
+import org.apache.pinot.spi.data.FieldSpec.DataType;
import org.apache.pinot.spi.data.MetricFieldSpec;
import org.apache.pinot.spi.data.Schema;
import org.apache.pinot.spi.data.readers.GenericRow;
import org.apache.pinot.spi.data.readers.RecordReader;
-import org.apache.pinot.spi.env.PinotConfiguration;
import org.apache.pinot.spi.utils.ByteArray;
import org.apache.pinot.spi.utils.BytesUtils;
import org.apache.pinot.spi.utils.ReadMode;
import org.apache.pinot.spi.utils.builder.TableConfigBuilder;
-import org.testng.Assert;
import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test;
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertFalse;
+import static org.testng.Assert.assertTrue;
+
/**
* Class for testing segment generation with byte[] data type.
@@ -89,92 +86,70 @@ public class SegmentGenerationWithBytesTypeTest {
private static final String VARIABLE_BYTES_COLUMN = "variableBytes";
private Random _random;
- private RecordReader _recordReader;
- private Schema _schema;
+ private List<GenericRow> _rows;
private TableConfig _tableConfig;
+ private Schema _schema;
private ImmutableSegment _segment;
/**
* Setup to build a segment with raw indexes (no-dictionary) of various data
types.
- *
- * @throws Exception
*/
@BeforeClass
- public void setup()
+ public void setUp()
throws Exception {
-
- _schema = new Schema();
- _schema.addField(new DimensionFieldSpec(FIXED_BYTE_SORTED_COLUMN,
FieldSpec.DataType.BYTES, true));
- _schema.addField(new DimensionFieldSpec(FIXED_BYTES_UNSORTED_COLUMN,
FieldSpec.DataType.BYTES, true));
- _schema.addField(new DimensionFieldSpec(FIXED_BYTES_NO_DICT_COLUMN,
FieldSpec.DataType.BYTES, true));
- _schema.addField(new DimensionFieldSpec(VARIABLE_BYTES_COLUMN,
FieldSpec.DataType.BYTES, true));
-
- _tableConfig = new
TableConfigBuilder(TableType.OFFLINE).setTableName("test").build();
-
_random = new Random(System.nanoTime());
- _recordReader = buildIndex(_schema);
+ _rows = generateRows();
+ _tableConfig = new
TableConfigBuilder(TableType.OFFLINE).setTableName("test").build();
+ _schema = new Schema.SchemaBuilder().setSchemaName("test")
+ .addSingleValueDimension(FIXED_BYTE_SORTED_COLUMN, DataType.BYTES)
+ .addSingleValueDimension(FIXED_BYTES_UNSORTED_COLUMN, DataType.BYTES)
+ .addSingleValueDimension(FIXED_BYTES_NO_DICT_COLUMN, DataType.BYTES)
+ .addSingleValueDimension(VARIABLE_BYTES_COLUMN,
DataType.BYTES).build();
+ buildSegment();
_segment = ImmutableSegmentLoader.load(new File(SEGMENT_DIR_NAME,
SEGMENT_NAME), ReadMode.heap);
}
- /**
- * Clean up after test
- */
@AfterClass
- public void cleanup()
+ public void tearDown()
throws IOException {
- _recordReader.close();
_segment.destroy();
FileUtils.deleteQuietly(new File(SEGMENT_DIR_NAME));
FileUtils.deleteQuietly(new File(AVRO_DIR_NAME));
}
@Test
- public void test()
+ public void testRecords()
throws Exception {
- PinotSegmentRecordReader pinotReader = new PinotSegmentRecordReader(new
File(SEGMENT_DIR_NAME, SEGMENT_NAME));
-
- _recordReader.rewind();
- while (pinotReader.hasNext()) {
- GenericRow expectedRow = _recordReader.next();
- GenericRow actualRow = pinotReader.next();
-
- for (String column : _schema.getColumnNames()) {
- byte[] actual = (byte[]) actualRow.getValue(column);
- byte[] expected = (byte[]) expectedRow.getValue(column);
-
- if (ByteArray.compare(actual, expected) != 0) {
- Assert.assertEquals(actualRow.getValue(column),
expectedRow.getValue(column));
- }
+ try (PinotSegmentRecordReader recordReader = new
PinotSegmentRecordReader()) {
+ recordReader.init(_segment);
+ for (int i = 0; i < NUM_ROWS; i++) {
+ assertEquals(recordReader.next(), _rows.get(i));
}
+ assertFalse(recordReader.hasNext());
}
-
- // Ensure both record readers are exhausted, ie same number of rows.
- Assert.assertFalse(_recordReader.hasNext());
- pinotReader.close();
}
@Test
public void testMetadata() {
-
Assert.assertTrue(_segment.getDataSource(FIXED_BYTE_SORTED_COLUMN).getDataSourceMetadata().isSorted());
-
Assert.assertFalse(_segment.getSegmentMetadata().getColumnMetadataFor(FIXED_BYTES_NO_DICT_COLUMN).hasDictionary());
+
assertTrue(_segment.getDataSource(FIXED_BYTE_SORTED_COLUMN).getDataSourceMetadata().isSorted());
+
assertFalse(_segment.getSegmentMetadata().getColumnMetadataFor(FIXED_BYTES_NO_DICT_COLUMN).hasDictionary());
}
@Test
public void testDictionary() {
BaseImmutableDictionary dictionary = (BaseImmutableDictionary)
_segment.getDictionary(FIXED_BYTE_SORTED_COLUMN);
- Assert.assertEquals(dictionary.length(), NUM_SORTED_VALUES);
+ assertEquals(dictionary.length(), NUM_SORTED_VALUES);
// Test dictionary indexing.
for (int i = 0; i < NUM_ROWS; i++) {
int value = (i * NUM_SORTED_VALUES) / NUM_ROWS;
// For sorted columns, values are written as 0, 0, 0.., 1, 1, 1...n, n, n
-
Assert.assertEquals(dictionary.indexOf(BytesUtils.toHexString(Ints.toByteArray(value))),
- value % NUM_SORTED_VALUES);
+
assertEquals(dictionary.indexOf(BytesUtils.toHexString(Ints.toByteArray(value))),
value % NUM_SORTED_VALUES);
}
// Test value not in dictionary.
-
Assert.assertEquals(dictionary.indexOf(BytesUtils.toHexString(Ints.toByteArray(NUM_SORTED_VALUES
+ 1))), -1);
-
Assert.assertEquals(dictionary.insertionIndexOf(BytesUtils.toHexString(Ints.toByteArray(NUM_SORTED_VALUES
+ 1))),
+
assertEquals(dictionary.indexOf(BytesUtils.toHexString(Ints.toByteArray(NUM_SORTED_VALUES
+ 1))), -1);
+
assertEquals(dictionary.insertionIndexOf(BytesUtils.toHexString(Ints.toByteArray(NUM_SORTED_VALUES
+ 1))),
-(NUM_SORTED_VALUES + 1));
int[] dictIds = new int[NUM_SORTED_VALUES];
@@ -186,7 +161,7 @@ public class SegmentGenerationWithBytesTypeTest {
dictionary.readBytesValues(dictIds, NUM_SORTED_VALUES, values);
for (int expected = 0; expected < NUM_SORTED_VALUES; expected++) {
int actual = ByteBuffer.wrap(values[expected]).asIntBuffer().get();
- Assert.assertEquals(actual, expected);
+ assertEquals(actual, expected);
}
}
@@ -197,8 +172,8 @@ public class SegmentGenerationWithBytesTypeTest {
public void testTDigestAvro()
throws Exception {
Schema schema = new Schema();
- schema.addField(new MetricFieldSpec(FIXED_BYTES_UNSORTED_COLUMN,
FieldSpec.DataType.BYTES));
- schema.addField(new MetricFieldSpec(VARIABLE_BYTES_COLUMN,
FieldSpec.DataType.BYTES));
+ schema.addField(new MetricFieldSpec(FIXED_BYTES_UNSORTED_COLUMN,
DataType.BYTES));
+ schema.addField(new MetricFieldSpec(VARIABLE_BYTES_COLUMN,
DataType.BYTES));
List<byte[]> fixedExpected = new ArrayList<>(NUM_ROWS);
List<byte[]> varExpected = new ArrayList<>(NUM_ROWS);
@@ -207,38 +182,24 @@ public class SegmentGenerationWithBytesTypeTest {
IndexSegment segment = buildSegmentFromAvro(schema, AVRO_DIR_NAME,
AVRO_NAME, SEGMENT_NAME);
SegmentMetadata metadata = segment.getSegmentMetadata();
-
-
Assert.assertTrue(metadata.getColumnMetadataFor(FIXED_BYTES_UNSORTED_COLUMN).hasDictionary());
-
Assert.assertTrue(metadata.getColumnMetadataFor(VARIABLE_BYTES_COLUMN).hasDictionary());
-
- PinotSegmentRecordReader reader = new PinotSegmentRecordReader(new
File(AVRO_DIR_NAME, SEGMENT_NAME));
- GenericRow row = new GenericRow();
-
- int i = 0;
- while (reader.hasNext()) {
- row = reader.next(row);
- Assert.assertEquals(ByteArray.compare((byte[])
row.getValue(FIXED_BYTES_UNSORTED_COLUMN), fixedExpected.get(i)),
- 0);
- Assert.assertEquals(ByteArray.compare((byte[])
row.getValue(VARIABLE_BYTES_COLUMN), varExpected.get(i++)), 0);
+
assertTrue(metadata.getColumnMetadataFor(FIXED_BYTES_UNSORTED_COLUMN).hasDictionary());
+
assertTrue(metadata.getColumnMetadataFor(VARIABLE_BYTES_COLUMN).hasDictionary());
+
+ try (PinotSegmentRecordReader reader = new PinotSegmentRecordReader()) {
+ reader.init(segment);
+ GenericRow row = new GenericRow();
+ int i = 0;
+ while (reader.hasNext()) {
+ row = reader.next(row);
+ assertEquals(ByteArray.compare((byte[])
row.getValue(FIXED_BYTES_UNSORTED_COLUMN), fixedExpected.get(i)), 0);
+ assertEquals(ByteArray.compare((byte[])
row.getValue(VARIABLE_BYTES_COLUMN), varExpected.get(i++)), 0);
+ }
}
+
segment.destroy();
}
- /**
- * Helper method to build a segment containing a single valued string column
with RAW (no-dictionary) index.
- *
- * @return Array of string values for the rows in the generated index.
- * @throws Exception
- */
-
- private RecordReader buildIndex(Schema schema)
- throws Exception {
- SegmentGeneratorConfig config = new SegmentGeneratorConfig(_tableConfig,
schema);
-
- config.setOutDir(SEGMENT_DIR_NAME);
- config.setSegmentName(SEGMENT_NAME);
-
config.setRawIndexCreationColumns(Collections.singletonList(FIXED_BYTES_NO_DICT_COLUMN));
-
+ private List<GenericRow> generateRows() {
List<GenericRow> rows = new ArrayList<>(NUM_ROWS);
for (int i = 0; i < NUM_ROWS; i++) {
HashMap<String, Object> map = new HashMap<>();
@@ -264,19 +225,21 @@ public class SegmentGenerationWithBytesTypeTest {
genericRow.init(map);
rows.add(genericRow);
}
+ return rows;
+ }
- RecordReader recordReader = new GenericRowRecordReader(rows);
- SegmentIndexCreationDriverImpl driver = new
SegmentIndexCreationDriverImpl();
- driver.init(config, recordReader);
- driver.build();
+ private void buildSegment()
+ throws Exception {
+ SegmentGeneratorConfig config = new SegmentGeneratorConfig(_tableConfig,
_schema);
+ config.setOutDir(SEGMENT_DIR_NAME);
+ config.setSegmentName(SEGMENT_NAME);
+
config.setRawIndexCreationColumns(Collections.singletonList(FIXED_BYTES_NO_DICT_COLUMN));
- Map<String, Object> props = new HashMap<>();
- props.put(IndexLoadingConfig.READ_MODE_KEY, ReadMode.mmap.toString());
-
SegmentDirectoryLoaderRegistry.getDefaultSegmentDirectoryLoader().load(driver.getOutputDirectory().toURI(),
- new
SegmentDirectoryLoaderContext.Builder().setTableConfig(_tableConfig)
- .setSegmentDirectoryConfigs(new
PinotConfiguration(props)).build());
- recordReader.rewind();
- return recordReader;
+ try (RecordReader recordReader = new GenericRowRecordReader(_rows)) {
+ SegmentIndexCreationDriverImpl driver = new
SegmentIndexCreationDriverImpl();
+ driver.init(config, recordReader);
+ driver.build();
+ }
}
/**
diff --git
a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/memory/unsafe/DirectMemory.java
b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/memory/unsafe/DirectMemory.java
index aef92af1df..04e13a2572 100644
---
a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/memory/unsafe/DirectMemory.java
+++
b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/memory/unsafe/DirectMemory.java
@@ -18,7 +18,6 @@
*/
package org.apache.pinot.segment.spi.memory.unsafe;
-import java.io.IOException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -55,15 +54,10 @@ public class DirectMemory implements Memory {
}
@Override
- public void close()
- throws IOException {
+ public synchronized void close() {
if (!_closed) {
- synchronized (this) {
- if (!_closed) {
- Unsafer.UNSAFE.freeMemory(_address);
- _closed = true;
- }
- }
+ Unsafer.UNSAFE.freeMemory(_address);
+ _closed = true;
}
}
@@ -71,7 +65,7 @@ public class DirectMemory implements Memory {
protected void finalize()
throws Throwable {
if (!_closed) {
- LOGGER.warn("Mmap section of " + _size + " wasn't explicitly closed");
+ LOGGER.warn("Direct memory of size: {} wasn't explicitly closed", _size);
close();
}
super.finalize();
diff --git
a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/memory/unsafe/MmapMemory.java
b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/memory/unsafe/MmapMemory.java
index 521d7f4cdf..ce91c255fa 100644
---
a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/memory/unsafe/MmapMemory.java
+++
b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/memory/unsafe/MmapMemory.java
@@ -98,16 +98,11 @@ public class MmapMemory implements Memory {
}
@Override
- public void close()
- throws IOException {
+ public synchronized void close() {
try {
if (!_closed) {
- synchronized (this) {
- if (!_closed) {
- _section._unmapFun.unmap();
- _closed = true;
- }
- }
+ _section._unmapFun.unmap();
+ _closed = true;
}
} catch (InvocationTargetException | IllegalAccessException e) {
throw new RuntimeException("Error while calling unmap", e);
@@ -118,7 +113,7 @@ public class MmapMemory implements Memory {
protected void finalize()
throws Throwable {
if (!_closed) {
- LOGGER.warn("Mmap section of " + _size + " wasn't explicitly closed");
+ LOGGER.warn("Mmap section of size: {} wasn't explicitly closed", _size);
close();
}
super.finalize();
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
