This is an automated email from the ASF dual-hosted git repository.
xbli pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 4f113a891c test the checking logic of whether segment preloading is
enabled for upsert table (#14085)
4f113a891c is described below
commit 4f113a891cb3df9dea8a3215ea41de0f83199718
Author: Xiaobing <[email protected]>
AuthorDate: Thu Sep 26 10:32:25 2024 -0700
test the checking logic of whether segment preloading is enabled for upsert
table (#14085)
---
.../TableUpsertMetadataManagerFactoryTest.java | 65 ++++++++++++++++++----
1 file changed, 53 insertions(+), 12 deletions(-)
diff --git
a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/upsert/TableUpsertMetadataManagerFactoryTest.java
b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/upsert/TableUpsertMetadataManagerFactoryTest.java
index 08f9796ff6..20a96b2e17 100644
---
a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/upsert/TableUpsertMetadataManagerFactoryTest.java
+++
b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/upsert/TableUpsertMetadataManagerFactoryTest.java
@@ -20,6 +20,7 @@ package org.apache.pinot.segment.local.upsert;
import com.google.common.collect.Lists;
import java.io.File;
+import java.util.concurrent.ExecutorService;
import org.apache.pinot.segment.local.data.manager.TableDataManager;
import org.apache.pinot.spi.config.table.HashFunction;
import org.apache.pinot.spi.config.table.TableConfig;
@@ -32,6 +33,8 @@ import org.testng.annotations.Test;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertFalse;
import static org.testng.Assert.assertNotNull;
import static org.testng.Assert.assertTrue;
@@ -44,10 +47,9 @@ public class TableUpsertMetadataManagerFactoryTest {
public void testCreateForDefaultManagerClass() {
UpsertConfig upsertConfig = new UpsertConfig(UpsertConfig.Mode.FULL);
upsertConfig.setHashFunction(HashFunction.NONE);
- Schema schema =
- new Schema.SchemaBuilder().setSchemaName(RAW_TABLE_NAME)
- .addSingleValueDimension("myCol", FieldSpec.DataType.STRING)
- .setPrimaryKeyColumns(Lists.newArrayList("myCol")).build();
+ Schema schema = new Schema.SchemaBuilder().setSchemaName(RAW_TABLE_NAME)
+ .addSingleValueDimension("myCol",
FieldSpec.DataType.STRING).setPrimaryKeyColumns(Lists.newArrayList("myCol"))
+ .build();
TableDataManager tableDataManager = mock(TableDataManager.class);
when(tableDataManager.getTableDataDir()).thenReturn(new
File(RAW_TABLE_NAME));
_tableConfig =
@@ -57,8 +59,8 @@ public class TableUpsertMetadataManagerFactoryTest {
assertNotNull(tableUpsertMetadataManager);
assertTrue(tableUpsertMetadataManager instanceof
ConcurrentMapTableUpsertMetadataManager);
tableUpsertMetadataManager.init(_tableConfig, schema, tableDataManager);
- assertTrue(tableUpsertMetadataManager.getOrCreatePartitionManager(0)
- instanceof ConcurrentMapPartitionUpsertMetadataManager);
+ assertTrue(tableUpsertMetadataManager.getOrCreatePartitionManager(
+ 0) instanceof ConcurrentMapPartitionUpsertMetadataManager);
}
@Test
@@ -66,10 +68,9 @@ public class TableUpsertMetadataManagerFactoryTest {
UpsertConfig upsertConfig = new UpsertConfig(UpsertConfig.Mode.FULL);
upsertConfig.setHashFunction(HashFunction.NONE);
upsertConfig.setEnableDeletedKeysCompactionConsistency(true);
- Schema schema =
- new Schema.SchemaBuilder().setSchemaName(RAW_TABLE_NAME)
- .addSingleValueDimension("myCol", FieldSpec.DataType.STRING)
- .setPrimaryKeyColumns(Lists.newArrayList("myCol")).build();
+ Schema schema = new Schema.SchemaBuilder().setSchemaName(RAW_TABLE_NAME)
+ .addSingleValueDimension("myCol",
FieldSpec.DataType.STRING).setPrimaryKeyColumns(Lists.newArrayList("myCol"))
+ .build();
TableDataManager tableDataManager = mock(TableDataManager.class);
when(tableDataManager.getTableDataDir()).thenReturn(new
File(RAW_TABLE_NAME));
_tableConfig =
@@ -79,7 +80,47 @@ public class TableUpsertMetadataManagerFactoryTest {
assertNotNull(tableUpsertMetadataManager);
assertTrue(tableUpsertMetadataManager instanceof
ConcurrentMapTableUpsertMetadataManager);
tableUpsertMetadataManager.init(_tableConfig, schema, tableDataManager);
- assertTrue(tableUpsertMetadataManager.getOrCreatePartitionManager(0)
- instanceof
ConcurrentMapPartitionUpsertMetadataManagerForConsistentDeletes);
+ assertTrue(tableUpsertMetadataManager.getOrCreatePartitionManager(
+ 0) instanceof
ConcurrentMapPartitionUpsertMetadataManagerForConsistentDeletes);
+ }
+
+ @Test
+ public void testEnablePreload() {
+ UpsertConfig upsertConfig = new UpsertConfig(UpsertConfig.Mode.FULL);
+ upsertConfig.setHashFunction(HashFunction.NONE);
+ upsertConfig.setEnablePreload(true);
+ upsertConfig.setEnableSnapshot(true);
+ Schema schema = new Schema.SchemaBuilder().setSchemaName(RAW_TABLE_NAME)
+ .addSingleValueDimension("myCol",
FieldSpec.DataType.STRING).setPrimaryKeyColumns(Lists.newArrayList("myCol"))
+ .build();
+ _tableConfig =
+ new
TableConfigBuilder(TableType.REALTIME).setTableName(RAW_TABLE_NAME).setUpsertConfig(upsertConfig).build();
+ TableUpsertMetadataManager tableUpsertMetadataManager =
+ TableUpsertMetadataManagerFactory.create(_tableConfig, null);
+ assertNotNull(tableUpsertMetadataManager);
+
+ // Preloading is not enabled even if enablePreload and enableSnapshot
flags are true, as no threads for preloading.
+ TableDataManager tableDataManager = mock(TableDataManager.class);
+ when(tableDataManager.getTableDataDir()).thenReturn(new
File(RAW_TABLE_NAME));
+ when(tableDataManager.getSegmentPreloadExecutor()).thenReturn(null);
+ tableUpsertMetadataManager.init(_tableConfig, schema, tableDataManager);
+ assertFalse(tableUpsertMetadataManager.isEnablePreload());
+
+ // Preloading is enabled if there are threads for preloading and
enablePreload and enableSnapshot flags are true.
+ tableDataManager = mock(TableDataManager.class);
+ when(tableDataManager.getTableDataDir()).thenReturn(new
File(RAW_TABLE_NAME));
+
when(tableDataManager.getSegmentPreloadExecutor()).thenReturn(mock(ExecutorService.class));
+ for (boolean[] flags : new boolean[][]{
+ {true, false}, {false, true}, {false, false}, {true, true}
+ }) {
+ upsertConfig.setEnableSnapshot(flags[0]);
+ upsertConfig.setEnablePreload(flags[1]);
+ _tableConfig =
+ new
TableConfigBuilder(TableType.REALTIME).setTableName(RAW_TABLE_NAME).setUpsertConfig(upsertConfig).build();
+ tableUpsertMetadataManager =
TableUpsertMetadataManagerFactory.create(_tableConfig, null);
+ tableUpsertMetadataManager.init(_tableConfig, schema, tableDataManager);
+ assertEquals(tableUpsertMetadataManager.isEnablePreload(), flags[0] &&
flags[1],
+ String.format("enableSnapshot: %b, enablePreload: %b", flags[0],
flags[1]));
+ }
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
