This is an automated email from the ASF dual-hosted git repository.
Jackie-Jiang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new dd28703d459 Replace segmentDirectoryConfigs map with ReadMode in
SegmentDirectoryLoaderContext (#18806)
dd28703d459 is described below
commit dd28703d4591e291ded06507633f312b96052839
Author: Xiaotian (Jackie) Jiang <[email protected]>
AuthorDate: Thu Jun 18 15:16:34 2026 -0700
Replace segmentDirectoryConfigs map with ReadMode in
SegmentDirectoryLoaderContext (#18806)
---
.../core/data/manager/BaseTableDataManager.java | 25 ++++++------
.../pinot/core/minion/SegmentPurgerTest.java | 9 +----
.../pinot/perf/BenchmarkOfflineIndexReader.java | 11 +----
.../refreshsegment/RefreshSegmentTaskExecutor.java | 18 ++++-----
.../immutable/ImmutableSegmentLoader.java | 47 ++++++++++------------
.../loader/DefaultSegmentDirectoryLoader.java | 7 +---
.../loader/TierBasedSegmentDirectoryLoader.java | 5 +--
.../segment/creator/impl/BaseSegmentCreator.java | 34 +++++-----------
.../converter/SegmentV1V2ToV3FormatConverter.java | 16 ++------
.../segment/index/loader/IndexLoadingConfig.java | 7 ----
.../segment/store/SegmentLocalFSDirectory.java | 42 +++++++------------
.../text/LuceneTextIndexConfigReloadTest.java | 15 +++----
.../spi/loader/SegmentDirectoryLoaderContext.java | 37 +++++++++--------
.../server/predownload/PredownloadSegmentInfo.java | 18 +++++----
14 files changed, 109 insertions(+), 182 deletions(-)
diff --git
a/pinot-core/src/main/java/org/apache/pinot/core/data/manager/BaseTableDataManager.java
b/pinot-core/src/main/java/org/apache/pinot/core/data/manager/BaseTableDataManager.java
index 5667b5d938d..a095d8e2cf1 100644
---
a/pinot-core/src/main/java/org/apache/pinot/core/data/manager/BaseTableDataManager.java
+++
b/pinot-core/src/main/java/org/apache/pinot/core/data/manager/BaseTableDataManager.java
@@ -1931,19 +1931,18 @@ public abstract class BaseTableDataManager implements
TableDataManager {
protected SegmentDirectory initSegmentDirectory(String segmentName, String
segmentCrc,
IndexLoadingConfig indexLoadingConfig, @Nullable SegmentZKMetadata
zkMetadata)
throws Exception {
- Map<String, String> segmentCustomConfigs = zkMetadata != null ?
zkMetadata.getCustomMap() : new HashMap<>();
- SegmentDirectoryLoaderContext loaderContext =
- new
SegmentDirectoryLoaderContext.Builder().setTableConfig(indexLoadingConfig.getTableConfig())
- .setSchema(indexLoadingConfig.getSchema())
- .setInstanceId(indexLoadingConfig.getInstanceId())
- .setTableDataDir(indexLoadingConfig.getTableDataDir())
- .setSegmentName(segmentName)
- .setSegmentCrc(segmentCrc)
- .setSegmentTier(indexLoadingConfig.getSegmentTier())
-
.setInstanceTierConfigs(indexLoadingConfig.getInstanceTierConfigs())
-
.setSegmentDirectoryConfigs(indexLoadingConfig.getSegmentDirectoryConfigs())
- .setSegmentCustomConfigs(segmentCustomConfigs)
- .build();
+ SegmentDirectoryLoaderContext loaderContext = new
SegmentDirectoryLoaderContext.Builder()
+ .setReadMode(indexLoadingConfig.getReadMode())
+ .setTableConfig(indexLoadingConfig.getTableConfig())
+ .setSchema(indexLoadingConfig.getSchema())
+ .setInstanceId(indexLoadingConfig.getInstanceId())
+ .setTableDataDir(indexLoadingConfig.getTableDataDir())
+ .setSegmentName(segmentName)
+ .setSegmentCrc(segmentCrc)
+ .setSegmentTier(indexLoadingConfig.getSegmentTier())
+ .setInstanceTierConfigs(indexLoadingConfig.getInstanceTierConfigs())
+ .setSegmentCustomConfigs(zkMetadata != null ?
zkMetadata.getCustomMap() : Map.of())
+ .build();
SegmentDirectoryLoader segmentDirectoryLoader =
SegmentDirectoryLoaderRegistry.getSegmentDirectoryLoader(indexLoadingConfig.getSegmentDirectoryLoader());
File indexDir =
diff --git
a/pinot-core/src/test/java/org/apache/pinot/core/minion/SegmentPurgerTest.java
b/pinot-core/src/test/java/org/apache/pinot/core/minion/SegmentPurgerTest.java
index 36d38e828ca..c16bccf0ecf 100644
---
a/pinot-core/src/test/java/org/apache/pinot/core/minion/SegmentPurgerTest.java
+++
b/pinot-core/src/test/java/org/apache/pinot/core/minion/SegmentPurgerTest.java
@@ -20,13 +20,10 @@ package org.apache.pinot.core.minion;
import java.io.File;
import java.util.ArrayList;
-import java.util.HashMap;
import java.util.List;
-import java.util.Map;
import java.util.Random;
import org.apache.commons.io.FileUtils;
import
org.apache.pinot.segment.local.segment.creator.impl.SegmentIndexCreationDriverImpl;
-import org.apache.pinot.segment.local.segment.index.loader.IndexLoadingConfig;
import org.apache.pinot.segment.local.segment.readers.GenericRowRecordReader;
import org.apache.pinot.segment.local.segment.readers.PinotSegmentRecordReader;
import org.apache.pinot.segment.spi.creator.SegmentGeneratorConfig;
@@ -42,8 +39,6 @@ import org.apache.pinot.spi.config.table.TableType;
import org.apache.pinot.spi.data.FieldSpec;
import org.apache.pinot.spi.data.Schema;
import org.apache.pinot.spi.data.readers.GenericRow;
-import org.apache.pinot.spi.env.PinotConfiguration;
-import org.apache.pinot.spi.utils.ReadMode;
import org.apache.pinot.spi.utils.builder.TableConfigBuilder;
import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
@@ -166,11 +161,9 @@ public class SegmentPurgerTest {
}
// Check inverted index
- Map<String, Object> props = new HashMap<>();
- props.put(IndexLoadingConfig.READ_MODE_KEY, ReadMode.mmap.toString());
try (SegmentDirectory segmentDirectory =
SegmentDirectoryLoaderRegistry.getDefaultSegmentDirectoryLoader()
.load(purgedIndexDir.toURI(), new
SegmentDirectoryLoaderContext.Builder().setTableConfig(_tableConfig)
-
.setSegmentName(purgedSegmentMetadata.getName()).setSegmentDirectoryConfigs(new
PinotConfiguration(props))
+ .setSegmentName(purgedSegmentMetadata.getName())
.build()); SegmentDirectory.Reader reader =
segmentDirectory.createReader()) {
assertTrue(reader.hasIndexFor(D1, StandardIndexes.inverted()));
assertFalse(reader.hasIndexFor(D2, StandardIndexes.inverted()));
diff --git
a/pinot-perf/src/main/java/org/apache/pinot/perf/BenchmarkOfflineIndexReader.java
b/pinot-perf/src/main/java/org/apache/pinot/perf/BenchmarkOfflineIndexReader.java
index e4550408763..e683f5142ef 100644
---
a/pinot-perf/src/main/java/org/apache/pinot/perf/BenchmarkOfflineIndexReader.java
+++
b/pinot-perf/src/main/java/org/apache/pinot/perf/BenchmarkOfflineIndexReader.java
@@ -22,8 +22,6 @@ import com.google.common.base.Preconditions;
import java.io.File;
import java.io.IOException;
import java.net.URL;
-import java.util.HashMap;
-import java.util.Map;
import java.util.Random;
import java.util.concurrent.TimeUnit;
import org.apache.commons.io.FileUtils;
@@ -31,7 +29,6 @@ import org.apache.pinot.common.utils.TarCompressionUtils;
import org.apache.pinot.integration.tests.ClusterTest;
import org.apache.pinot.segment.local.segment.creator.SegmentTestUtils;
import
org.apache.pinot.segment.local.segment.creator.impl.SegmentIndexCreationDriverImpl;
-import org.apache.pinot.segment.local.segment.index.loader.IndexLoadingConfig;
import org.apache.pinot.segment.local.segment.index.readers.DoubleDictionary;
import org.apache.pinot.segment.local.segment.index.readers.FloatDictionary;
import org.apache.pinot.segment.local.segment.index.readers.IntDictionary;
@@ -48,8 +45,6 @@ import
org.apache.pinot.segment.spi.index.reader.ForwardIndexReaderContext;
import org.apache.pinot.segment.spi.loader.SegmentDirectoryLoaderContext;
import org.apache.pinot.segment.spi.loader.SegmentDirectoryLoaderRegistry;
import org.apache.pinot.segment.spi.store.SegmentDirectory;
-import org.apache.pinot.spi.env.PinotConfiguration;
-import org.apache.pinot.spi.utils.ReadMode;
import org.apache.pinot.util.TestUtils;
import org.openjdk.jmh.annotations.Benchmark;
import org.openjdk.jmh.annotations.BenchmarkMode;
@@ -119,12 +114,10 @@ public class BenchmarkOfflineIndexReader {
File indexDir = new File(dataDir, TABLE_NAME);
SegmentMetadataImpl segmentMetadata = new SegmentMetadataImpl(indexDir);
- Map<String, Object> props = new HashMap<>();
- props.put(IndexLoadingConfig.READ_MODE_KEY, ReadMode.mmap.toString());
SegmentDirectory segmentDirectory =
SegmentDirectoryLoaderRegistry.getDefaultSegmentDirectoryLoader()
- .load(indexDir.toURI(), new
SegmentDirectoryLoaderContext.Builder().setSegmentName(segmentMetadata.getName())
- .setSegmentDirectoryConfigs(new
PinotConfiguration(props)).build());
+ .load(indexDir.toURI(),
+ new
SegmentDirectoryLoaderContext.Builder().setSegmentName(segmentMetadata.getName()).build());
SegmentDirectory.Reader segmentReader = segmentDirectory.createReader();
// Forward index
diff --git
a/pinot-plugins/pinot-minion-tasks/pinot-minion-builtin-tasks/src/main/java/org/apache/pinot/plugin/minion/tasks/refreshsegment/RefreshSegmentTaskExecutor.java
b/pinot-plugins/pinot-minion-tasks/pinot-minion-builtin-tasks/src/main/java/org/apache/pinot/plugin/minion/tasks/refreshsegment/RefreshSegmentTaskExecutor.java
index 524a63fa143..5cfc7dee942 100644
---
a/pinot-plugins/pinot-minion-tasks/pinot-minion-builtin-tasks/src/main/java/org/apache/pinot/plugin/minion/tasks/refreshsegment/RefreshSegmentTaskExecutor.java
+++
b/pinot-plugins/pinot-minion-tasks/pinot-minion-builtin-tasks/src/main/java/org/apache/pinot/plugin/minion/tasks/refreshsegment/RefreshSegmentTaskExecutor.java
@@ -43,7 +43,6 @@ import org.apache.pinot.spi.config.instance.InstanceType;
import org.apache.pinot.spi.config.table.TableConfig;
import org.apache.pinot.spi.data.FieldSpec;
import org.apache.pinot.spi.data.Schema;
-import org.apache.pinot.spi.env.PinotConfiguration;
import org.apache.pinot.spi.utils.Obfuscator;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -83,15 +82,14 @@ public class RefreshSegmentTaskExecutor extends
BaseSingleSegmentConversionExecu
IndexLoadingConfig indexLoadingConfig = new
IndexLoadingConfig(tableConfig, schema);
SegmentMetadataImpl segmentMetadata = new SegmentMetadataImpl(indexDir);
- PinotConfiguration segmentDirectoryConfigs =
indexLoadingConfig.getSegmentDirectoryConfigs();
- SegmentDirectoryLoaderContext segmentLoaderContext =
- new
SegmentDirectoryLoaderContext.Builder().setTableConfig(indexLoadingConfig.getTableConfig())
- .setSchema(schema)
- .setInstanceId(indexLoadingConfig.getInstanceId())
- .setSegmentName(segmentMetadata.getName())
- .setSegmentCrc(segmentMetadata.getCrc())
- .setSegmentDirectoryConfigs(segmentDirectoryConfigs)
- .build();
+ SegmentDirectoryLoaderContext segmentLoaderContext = new
SegmentDirectoryLoaderContext.Builder()
+ .setReadMode(indexLoadingConfig.getReadMode())
+ .setTableConfig(indexLoadingConfig.getTableConfig())
+ .setSchema(schema)
+ .setInstanceId(indexLoadingConfig.getInstanceId())
+ .setSegmentName(segmentMetadata.getName())
+ .setSegmentCrc(segmentMetadata.getCrc())
+ .build();
SegmentDirectory segmentDirectory =
SegmentDirectoryLoaderRegistry.getDefaultSegmentDirectoryLoader().load(indexDir.toURI(),
segmentLoaderContext);
diff --git
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/indexsegment/immutable/ImmutableSegmentLoader.java
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/indexsegment/immutable/ImmutableSegmentLoader.java
index 20c16ad962d..0e16f2f4925 100644
---
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/indexsegment/immutable/ImmutableSegmentLoader.java
+++
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/indexsegment/immutable/ImmutableSegmentLoader.java
@@ -21,7 +21,6 @@ package org.apache.pinot.segment.local.indexsegment.immutable;
import com.google.common.base.Preconditions;
import it.unimi.dsi.fastutil.objects.Object2ObjectOpenHashMap;
import java.io.File;
-import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
@@ -50,7 +49,6 @@ import org.apache.pinot.segment.spi.store.SegmentDirectory;
import org.apache.pinot.segment.spi.store.SegmentDirectoryPaths;
import org.apache.pinot.spi.data.FieldSpec;
import org.apache.pinot.spi.data.Schema;
-import org.apache.pinot.spi.env.PinotConfiguration;
import org.apache.pinot.spi.utils.ReadMode;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -143,19 +141,18 @@ public class ImmutableSegmentLoader {
preprocess(indexDir, indexLoadingConfig, segmentOperationsThrottlerSet,
zkMetadata);
}
String segmentName = segmentMetadata.getName();
- Map<String, String> segmentCustomConfigs = zkMetadata != null ?
zkMetadata.getCustomMap() : new HashMap<>();
- SegmentDirectoryLoaderContext segmentLoaderContext =
- new
SegmentDirectoryLoaderContext.Builder().setTableConfig(indexLoadingConfig.getTableConfig())
- .setSchema(indexLoadingConfig.getSchema())
- .setInstanceId(indexLoadingConfig.getInstanceId())
- .setTableDataDir(indexLoadingConfig.getTableDataDir())
- .setSegmentName(segmentName)
- .setSegmentCrc(segmentMetadata.getCrc())
- .setSegmentTier(indexLoadingConfig.getSegmentTier())
-
.setInstanceTierConfigs(indexLoadingConfig.getInstanceTierConfigs())
-
.setSegmentDirectoryConfigs(indexLoadingConfig.getSegmentDirectoryConfigs())
- .setSegmentCustomConfigs(segmentCustomConfigs)
- .build();
+ SegmentDirectoryLoaderContext segmentLoaderContext = new
SegmentDirectoryLoaderContext.Builder()
+ .setReadMode(indexLoadingConfig.getReadMode())
+ .setTableConfig(indexLoadingConfig.getTableConfig())
+ .setSchema(indexLoadingConfig.getSchema())
+ .setInstanceId(indexLoadingConfig.getInstanceId())
+ .setTableDataDir(indexLoadingConfig.getTableDataDir())
+ .setSegmentName(segmentName)
+ .setSegmentCrc(segmentMetadata.getCrc())
+ .setSegmentTier(indexLoadingConfig.getSegmentTier())
+ .setInstanceTierConfigs(indexLoadingConfig.getInstanceTierConfigs())
+ .setSegmentCustomConfigs(zkMetadata != null ?
zkMetadata.getCustomMap() : Map.of())
+ .build();
SegmentDirectoryLoader segmentLoader =
SegmentDirectoryLoaderRegistry.getSegmentDirectoryLoader(indexLoadingConfig.getSegmentDirectoryLoader());
SegmentDirectory segmentDirectory = segmentLoader.load(indexDir.toURI(),
segmentLoaderContext);
@@ -325,17 +322,15 @@ public class ImmutableSegmentLoader {
IndexLoadingConfig indexLoadingConfig, @Nullable
SegmentOperationsThrottlerSet segmentOperationsThrottlerSet,
SegmentZKMetadata zkMetadata)
throws Exception {
- PinotConfiguration segmentDirectoryConfigs =
indexLoadingConfig.getSegmentDirectoryConfigs();
- Map<String, String> segmentCustomConfigs = zkMetadata != null ?
zkMetadata.getCustomMap() : new HashMap<>();
- SegmentDirectoryLoaderContext segmentLoaderContext =
- new
SegmentDirectoryLoaderContext.Builder().setTableConfig(indexLoadingConfig.getTableConfig())
- .setSchema(indexLoadingConfig.getSchema())
- .setInstanceId(indexLoadingConfig.getInstanceId())
- .setSegmentName(segmentName)
- .setSegmentCrc(segmentCrc)
- .setSegmentDirectoryConfigs(segmentDirectoryConfigs)
- .setSegmentCustomConfigs(segmentCustomConfigs)
- .build();
+ SegmentDirectoryLoaderContext segmentLoaderContext = new
SegmentDirectoryLoaderContext.Builder()
+ .setReadMode(indexLoadingConfig.getReadMode())
+ .setTableConfig(indexLoadingConfig.getTableConfig())
+ .setSchema(indexLoadingConfig.getSchema())
+ .setInstanceId(indexLoadingConfig.getInstanceId())
+ .setSegmentName(segmentName)
+ .setSegmentCrc(segmentCrc)
+ .setSegmentCustomConfigs(zkMetadata != null ?
zkMetadata.getCustomMap() : Map.of())
+ .build();
SegmentDirectory segmentDirectory =
SegmentDirectoryLoaderRegistry.getDefaultSegmentDirectoryLoader().load(indexDir.toURI(),
segmentLoaderContext);
try (SegmentPreProcessor preProcessor = new
SegmentPreProcessor(segmentDirectory, indexLoadingConfig)) {
diff --git
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/loader/DefaultSegmentDirectoryLoader.java
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/loader/DefaultSegmentDirectoryLoader.java
index 17f8cb6d92d..9998163b96e 100644
---
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/loader/DefaultSegmentDirectoryLoader.java
+++
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/loader/DefaultSegmentDirectoryLoader.java
@@ -21,14 +21,11 @@ package org.apache.pinot.segment.local.loader;
import java.io.File;
import java.net.URI;
import org.apache.commons.io.FileUtils;
-import org.apache.pinot.segment.local.segment.index.loader.IndexLoadingConfig;
import org.apache.pinot.segment.local.segment.store.SegmentLocalFSDirectory;
import org.apache.pinot.segment.spi.loader.SegmentDirectoryLoader;
import org.apache.pinot.segment.spi.loader.SegmentDirectoryLoaderContext;
import org.apache.pinot.segment.spi.loader.SegmentLoader;
import org.apache.pinot.segment.spi.store.SegmentDirectory;
-import org.apache.pinot.spi.env.PinotConfiguration;
-import org.apache.pinot.spi.utils.ReadMode;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -50,13 +47,11 @@ public class DefaultSegmentDirectoryLoader implements
SegmentDirectoryLoader {
@Override
public SegmentDirectory load(URI indexDir, SegmentDirectoryLoaderContext
segmentLoaderContext)
throws Exception {
- PinotConfiguration segmentDirectoryConfigs =
segmentLoaderContext.getSegmentDirectoryConfigs();
File directory = new File(indexDir);
if (!directory.exists()) {
return new SegmentLocalFSDirectory(directory);
}
- return new SegmentLocalFSDirectory(directory, segmentLoaderContext,
-
ReadMode.valueOf(segmentDirectoryConfigs.getProperty(IndexLoadingConfig.READ_MODE_KEY)));
+ return new SegmentLocalFSDirectory(directory,
segmentLoaderContext.getReadMode());
}
@Override
diff --git
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/loader/TierBasedSegmentDirectoryLoader.java
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/loader/TierBasedSegmentDirectoryLoader.java
index 397aa79c567..d37a9501eca 100644
---
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/loader/TierBasedSegmentDirectoryLoader.java
+++
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/loader/TierBasedSegmentDirectoryLoader.java
@@ -28,14 +28,12 @@ import java.nio.charset.StandardCharsets;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.pinot.common.utils.config.TierConfigUtils;
-import org.apache.pinot.segment.local.segment.index.loader.IndexLoadingConfig;
import org.apache.pinot.segment.local.segment.store.SegmentLocalFSDirectory;
import org.apache.pinot.segment.spi.loader.SegmentDirectoryLoader;
import org.apache.pinot.segment.spi.loader.SegmentDirectoryLoaderContext;
import org.apache.pinot.segment.spi.loader.SegmentLoader;
import org.apache.pinot.segment.spi.store.SegmentDirectory;
import org.apache.pinot.spi.config.table.TableConfig;
-import org.apache.pinot.spi.utils.ReadMode;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -103,8 +101,7 @@ public class TierBasedSegmentDirectoryLoader implements
SegmentDirectoryLoader {
if (!destDir.exists()) {
segmentDirectory = new SegmentLocalFSDirectory(destDir);
} else {
- segmentDirectory = new SegmentLocalFSDirectory(destDir, ReadMode.valueOf(
-
segmentLoaderContext.getSegmentDirectoryConfigs().getProperty(IndexLoadingConfig.READ_MODE_KEY)));
+ segmentDirectory = new SegmentLocalFSDirectory(destDir,
segmentLoaderContext.getReadMode());
}
LOGGER.info("Created segmentDirectory object for segment: {} with dataDir:
{} on targetTier: {}", segmentName,
destDir, targetTierName);
diff --git
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/BaseSegmentCreator.java
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/BaseSegmentCreator.java
index 3938a6437b3..a978221ef69 100644
---
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/BaseSegmentCreator.java
+++
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/BaseSegmentCreator.java
@@ -27,7 +27,6 @@ import java.io.FileOutputStream;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
-import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -89,8 +88,6 @@ import org.apache.pinot.spi.data.FieldSpec.FieldType;
import org.apache.pinot.spi.data.FieldSpec.MaxLengthExceedStrategy;
import org.apache.pinot.spi.data.Schema;
import org.apache.pinot.spi.env.CommonsConfigurationUtils;
-import org.apache.pinot.spi.env.PinotConfiguration;
-import org.apache.pinot.spi.utils.ReadMode;
import org.apache.pinot.spi.utils.TimeUtils;
import org.joda.time.DateTimeZone;
import org.joda.time.Interval;
@@ -901,18 +898,13 @@ public abstract class BaseSegmentCreator implements
SegmentCreator {
private void buildMultiColumnTextIndex(File segmentOutputDir)
throws Exception {
if (_config.getMultiColumnTextIndexConfig() != null) {
- PinotConfiguration segmentDirectoryConfigs =
- new PinotConfiguration(Map.of(IndexLoadingConfig.READ_MODE_KEY,
ReadMode.mmap));
-
TableConfig tableConfig = _config.getTableConfig();
Schema schema = _config.getSchema();
- SegmentDirectoryLoaderContext segmentLoaderContext =
- new SegmentDirectoryLoaderContext.Builder()
- .setTableConfig(tableConfig)
- .setSchema(schema)
- .setSegmentName(_segmentName)
- .setSegmentDirectoryConfigs(segmentDirectoryConfigs)
- .build();
+ SegmentDirectoryLoaderContext segmentLoaderContext = new
SegmentDirectoryLoaderContext.Builder()
+ .setTableConfig(tableConfig)
+ .setSchema(schema)
+ .setSegmentName(_segmentName)
+ .build();
IndexLoadingConfig indexLoadingConfig = new IndexLoadingConfig(null,
tableConfig, schema);
@@ -938,19 +930,13 @@ public abstract class BaseSegmentCreator implements
SegmentCreator {
if (!postSegCreationIndexes.isEmpty()) {
// Build other indexes
- Map<String, Object> props = new HashMap<>();
- props.put(IndexLoadingConfig.READ_MODE_KEY, ReadMode.mmap);
- PinotConfiguration segmentDirectoryConfigs = new
PinotConfiguration(props);
-
TableConfig tableConfig = _config.getTableConfig();
Schema schema = _config.getSchema();
- SegmentDirectoryLoaderContext segmentLoaderContext =
- new SegmentDirectoryLoaderContext.Builder()
- .setTableConfig(tableConfig)
- .setSchema(schema)
- .setSegmentName(_segmentName)
- .setSegmentDirectoryConfigs(segmentDirectoryConfigs)
- .build();
+ SegmentDirectoryLoaderContext segmentLoaderContext = new
SegmentDirectoryLoaderContext.Builder()
+ .setTableConfig(tableConfig)
+ .setSchema(schema)
+ .setSegmentName(_segmentName)
+ .build();
IndexLoadingConfig indexLoadingConfig = new IndexLoadingConfig(null,
tableConfig, schema);
diff --git
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/converter/SegmentV1V2ToV3FormatConverter.java
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/converter/SegmentV1V2ToV3FormatConverter.java
index c23b996c778..597f7d947e0 100644
---
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/converter/SegmentV1V2ToV3FormatConverter.java
+++
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/converter/SegmentV1V2ToV3FormatConverter.java
@@ -26,12 +26,9 @@ import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.attribute.PosixFilePermission;
import java.util.EnumSet;
-import java.util.HashMap;
-import java.util.Map;
import org.apache.commons.configuration2.PropertiesConfiguration;
import org.apache.commons.configuration2.ex.ConfigurationException;
import org.apache.commons.io.FileUtils;
-import org.apache.pinot.segment.local.segment.index.loader.IndexLoadingConfig;
import org.apache.pinot.segment.local.segment.store.TextIndexUtils;
import org.apache.pinot.segment.spi.V1Constants;
import org.apache.pinot.segment.spi.converter.SegmentFormatConverter;
@@ -47,8 +44,6 @@ import org.apache.pinot.segment.spi.memory.PinotDataBuffer;
import org.apache.pinot.segment.spi.store.SegmentDirectory;
import org.apache.pinot.segment.spi.store.SegmentDirectoryPaths;
import org.apache.pinot.spi.env.CommonsConfigurationUtils;
-import org.apache.pinot.spi.env.PinotConfiguration;
-import org.apache.pinot.spi.utils.ReadMode;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -142,15 +137,12 @@ public class SegmentV1V2ToV3FormatConverter implements
SegmentFormatConverter {
private void copyIndexData(File v2Directory, SegmentMetadataImpl v2Metadata,
File v3Directory)
throws Exception {
- Map<String, Object> props = new HashMap<>();
- props.put(IndexLoadingConfig.READ_MODE_KEY, ReadMode.mmap.toString());
- PinotConfiguration configuration = new PinotConfiguration(props);
try (SegmentDirectory v2Segment =
SegmentDirectoryLoaderRegistry.getDefaultSegmentDirectoryLoader()
- .load(v2Directory.toURI(), new
SegmentDirectoryLoaderContext.Builder().setSegmentName(v2Metadata.getName())
- .setSegmentDirectoryConfigs(configuration).build());
+ .load(v2Directory.toURI(),
+ new
SegmentDirectoryLoaderContext.Builder().setSegmentName(v2Metadata.getName()).build());
SegmentDirectory v3Segment =
SegmentDirectoryLoaderRegistry.getDefaultSegmentDirectoryLoader()
- .load(v3Directory.toURI(), new
SegmentDirectoryLoaderContext.Builder().setSegmentName(v2Metadata.getName())
- .setSegmentDirectoryConfigs(configuration).build())) {
+ .load(v3Directory.toURI(),
+ new
SegmentDirectoryLoaderContext.Builder().setSegmentName(v2Metadata.getName()).build()))
{
try (SegmentDirectory.Reader v2DataReader = v2Segment.createReader();
SegmentDirectory.Writer v3DataWriter = v3Segment.createWriter()) {
for (String column : v2Metadata.getAllColumns()) {
diff --git
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/IndexLoadingConfig.java
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/IndexLoadingConfig.java
index a2565f09ddc..1a1a64f75d9 100644
---
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/IndexLoadingConfig.java
+++
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/IndexLoadingConfig.java
@@ -42,7 +42,6 @@ import org.apache.pinot.spi.config.table.TableConfig;
import org.apache.pinot.spi.data.DimensionFieldSpec;
import org.apache.pinot.spi.data.FieldSpec;
import org.apache.pinot.spi.data.Schema;
-import org.apache.pinot.spi.env.PinotConfiguration;
import org.apache.pinot.spi.utils.ReadMode;
import org.apache.pinot.spi.utils.TimestampIndexUtils;
@@ -315,12 +314,6 @@ public class IndexLoadingConfig {
: SegmentDirectoryLoaderRegistry.DEFAULT_SEGMENT_DIRECTORY_LOADER_NAME;
}
- public PinotConfiguration getSegmentDirectoryConfigs() {
- Map<String, Object> props = new HashMap<>();
- props.put(READ_MODE_KEY, _readMode);
- return new PinotConfiguration(props);
- }
-
public String getInstanceId() {
return _instanceId;
}
diff --git
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/store/SegmentLocalFSDirectory.java
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/store/SegmentLocalFSDirectory.java
index a68271e34c9..54f7207ce9e 100644
---
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/store/SegmentLocalFSDirectory.java
+++
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/store/SegmentLocalFSDirectory.java
@@ -61,9 +61,9 @@ public class SegmentLocalFSDirectory extends SegmentDirectory
{
private final File _indexDir;
private final File _segmentDirectory;
- private final SegmentLock _segmentLock;
private final ReadMode _readMode;
private final SegmentDirectoryLoaderContext _segmentDirectoryLoaderContext;
+ private final SegmentLock _segmentLock = new SegmentLock();
private SegmentMetadataImpl _segmentMetadata;
private ColumnIndexDirectory _columnIndexDirectory;
private StarTreeIndexReader _starTreeIndexReader;
@@ -72,45 +72,33 @@ public class SegmentLocalFSDirectory extends
SegmentDirectory {
// Create an empty SegmentLocalFSDirectory object mainly used to
// prepare env for subsequent processing on the segment.
- public SegmentLocalFSDirectory(File directory) {
- _indexDir = directory;
+ public SegmentLocalFSDirectory(File indexDir) {
+ _indexDir = indexDir;
_segmentDirectory = null;
- _segmentLock = new SegmentLock();
_readMode = null;
_segmentDirectoryLoaderContext = null;
}
- public SegmentLocalFSDirectory(File directory, ReadMode readMode)
+ public SegmentLocalFSDirectory(File indexDir, ReadMode readMode)
throws IOException, ConfigurationException {
- this(directory, new SegmentMetadataImpl(directory), null, readMode);
+ this(indexDir, new SegmentMetadataImpl(indexDir), readMode, null);
}
- public SegmentLocalFSDirectory(File directory, @Nullable
SegmentDirectoryLoaderContext segmentDirectoryLoaderContext,
- ReadMode readMode)
- throws IOException, ConfigurationException {
- this(directory, new SegmentMetadataImpl(directory),
segmentDirectoryLoaderContext, readMode);
- }
-
- public SegmentLocalFSDirectory(File directory, SegmentMetadataImpl metadata,
ReadMode readMode)
+ @VisibleForTesting
+ public SegmentLocalFSDirectory(File indexDir, SegmentMetadataImpl metadata,
ReadMode readMode)
throws IOException, ConfigurationException {
- this(directory, metadata, null, readMode);
+ this(indexDir, metadata, readMode, null);
}
- @VisibleForTesting
- public SegmentLocalFSDirectory(File directoryFile, SegmentMetadataImpl
metadata,
- @Nullable SegmentDirectoryLoaderContext segmentDirectoryLoaderContext,
ReadMode readMode) {
- _segmentDirectoryLoaderContext = segmentDirectoryLoaderContext;
-
- Preconditions.checkNotNull(directoryFile);
- Preconditions.checkNotNull(metadata);
-
- _indexDir = directoryFile;
- _segmentDirectory = getSegmentPath(directoryFile, metadata.getVersion());
- Preconditions.checkState(_segmentDirectory.exists(), "Segment directory: "
+ directoryFile + " must exist");
-
- _segmentLock = new SegmentLock();
+ public SegmentLocalFSDirectory(File indexDir, SegmentMetadataImpl metadata,
ReadMode readMode,
+ @Nullable SegmentDirectoryLoaderContext segmentDirectoryLoaderContext) {
+ _indexDir = indexDir;
+ _segmentDirectory = getSegmentPath(indexDir, metadata.getVersion());
+ Preconditions.checkState(_segmentDirectory.exists(), "Segment directory: "
+ indexDir + " must exist");
_segmentMetadata = metadata;
_readMode = readMode;
+ _segmentDirectoryLoaderContext = segmentDirectoryLoaderContext;
+
try {
load();
} catch (IOException | ConfigurationException e) {
diff --git
a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/text/LuceneTextIndexConfigReloadTest.java
b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/text/LuceneTextIndexConfigReloadTest.java
index 547d5d4a885..2ce227094f6 100644
---
a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/text/LuceneTextIndexConfigReloadTest.java
+++
b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/text/LuceneTextIndexConfigReloadTest.java
@@ -23,7 +23,6 @@ import java.io.IOException;
import java.nio.file.Files;
import java.util.ArrayList;
import java.util.Arrays;
-import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.apache.commons.io.FileUtils;
@@ -48,8 +47,6 @@ import org.apache.pinot.spi.data.FieldSpec;
import org.apache.pinot.spi.data.Schema;
import org.apache.pinot.spi.data.readers.GenericRow;
import org.apache.pinot.spi.data.readers.RecordReader;
-import org.apache.pinot.spi.env.PinotConfiguration;
-import org.apache.pinot.spi.utils.ReadMode;
import org.apache.pinot.spi.utils.builder.TableConfigBuilder;
import org.roaringbitmap.buffer.MutableRoaringBitmap;
import org.slf4j.Logger;
@@ -215,13 +212,11 @@ public class LuceneTextIndexConfigReloadTest {
*/
private boolean checkNeedUpdateIndices(File segmentFile, TableConfig
tableConfig, Schema schema)
throws Exception {
- Map<String, Object> props = new HashMap<>();
- props.put(IndexLoadingConfig.READ_MODE_KEY, ReadMode.mmap);
- PinotConfiguration segmentDirectoryConfigs = new PinotConfiguration(props);
-
- SegmentDirectoryLoaderContext segmentLoaderContext =
- new
SegmentDirectoryLoaderContext.Builder().setTableConfig(tableConfig).setSchema(schema)
-
.setSegmentName(segmentFile.getName()).setSegmentDirectoryConfigs(segmentDirectoryConfigs).build();
+ SegmentDirectoryLoaderContext segmentLoaderContext = new
SegmentDirectoryLoaderContext.Builder()
+ .setTableConfig(tableConfig)
+ .setSchema(schema)
+ .setSegmentName(segmentFile.getName())
+ .build();
SegmentDirectory segmentDirectory =
SegmentDirectoryLoaderRegistry.getDefaultSegmentDirectoryLoader()
.load(segmentFile.toURI(), segmentLoaderContext);
diff --git
a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/loader/SegmentDirectoryLoaderContext.java
b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/loader/SegmentDirectoryLoaderContext.java
index 5fc6db87132..2fc8eed295e 100644
---
a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/loader/SegmentDirectoryLoaderContext.java
+++
b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/loader/SegmentDirectoryLoaderContext.java
@@ -21,14 +21,14 @@ package org.apache.pinot.segment.spi.loader;
import java.util.Map;
import org.apache.pinot.spi.config.table.TableConfig;
import org.apache.pinot.spi.data.Schema;
-import org.apache.pinot.spi.env.PinotConfiguration;
+import org.apache.pinot.spi.utils.ReadMode;
/**
* Context for {@link SegmentDirectoryLoader}
*/
public class SegmentDirectoryLoaderContext {
-
+ private final ReadMode _readMode;
private final TableConfig _tableConfig;
private final Schema _schema;
private final String _instanceId;
@@ -37,12 +37,12 @@ public class SegmentDirectoryLoaderContext {
private final String _segmentCrc;
private final String _segmentTier;
private final Map<String, Map<String, String>> _instanceTierConfigs;
- private final PinotConfiguration _segmentDirectoryConfigs;
private final Map<String, String> _segmentCustomConfigs;
- private SegmentDirectoryLoaderContext(TableConfig tableConfig, Schema
schema, String instanceId, String tableDataDir,
- String segmentName, String segmentCrc, String segmentTier, Map<String,
Map<String, String>> instanceTierConfigs,
- PinotConfiguration segmentDirectoryConfigs, Map<String, String>
segmentCustomConfigs) {
+ private SegmentDirectoryLoaderContext(ReadMode readMode, TableConfig
tableConfig, Schema schema, String instanceId,
+ String tableDataDir, String segmentName, String segmentCrc, String
segmentTier,
+ Map<String, Map<String, String>> instanceTierConfigs, Map<String,
String> segmentCustomConfigs) {
+ _readMode = readMode;
_tableConfig = tableConfig;
_schema = schema;
_instanceId = instanceId;
@@ -51,10 +51,13 @@ public class SegmentDirectoryLoaderContext {
_segmentCrc = segmentCrc;
_segmentTier = segmentTier;
_instanceTierConfigs = instanceTierConfigs;
- _segmentDirectoryConfigs = segmentDirectoryConfigs;
_segmentCustomConfigs = segmentCustomConfigs;
}
+ public ReadMode getReadMode() {
+ return _readMode;
+ }
+
public TableConfig getTableConfig() {
return _tableConfig;
}
@@ -83,10 +86,6 @@ public class SegmentDirectoryLoaderContext {
return _segmentTier;
}
- public PinotConfiguration getSegmentDirectoryConfigs() {
- return _segmentDirectoryConfigs;
- }
-
public Map<String, Map<String, String>> getInstanceTierConfigs() {
return _instanceTierConfigs;
}
@@ -96,6 +95,7 @@ public class SegmentDirectoryLoaderContext {
}
public static class Builder {
+ private ReadMode _readMode = ReadMode.DEFAULT_MODE;
private TableConfig _tableConfig;
private Schema _schema;
private String _instanceId;
@@ -104,9 +104,13 @@ public class SegmentDirectoryLoaderContext {
private String _segmentCrc;
private String _segmentTier;
private Map<String, Map<String, String>> _instanceTierConfigs;
- private PinotConfiguration _segmentDirectoryConfigs;
private Map<String, String> _segmentCustomConfigs;
+ public Builder setReadMode(ReadMode readMode) {
+ _readMode = readMode;
+ return this;
+ }
+
public Builder setTableConfig(TableConfig tableConfig) {
_tableConfig = tableConfig;
return this;
@@ -147,19 +151,14 @@ public class SegmentDirectoryLoaderContext {
return this;
}
- public Builder setSegmentDirectoryConfigs(PinotConfiguration
segmentDirectoryConfigs) {
- _segmentDirectoryConfigs = segmentDirectoryConfigs;
- return this;
- }
-
public Builder setSegmentCustomConfigs(Map<String, String>
segmentCustomConfigs) {
_segmentCustomConfigs = segmentCustomConfigs;
return this;
}
public SegmentDirectoryLoaderContext build() {
- return new SegmentDirectoryLoaderContext(_tableConfig, _schema,
_instanceId, _tableDataDir, _segmentName,
- _segmentCrc, _segmentTier, _instanceTierConfigs,
_segmentDirectoryConfigs, _segmentCustomConfigs);
+ return new SegmentDirectoryLoaderContext(_readMode, _tableConfig,
_schema, _instanceId, _tableDataDir,
+ _segmentName, _segmentCrc, _segmentTier, _instanceTierConfigs,
_segmentCustomConfigs);
}
}
}
diff --git
a/pinot-server/src/main/java/org/apache/pinot/server/predownload/PredownloadSegmentInfo.java
b/pinot-server/src/main/java/org/apache/pinot/server/predownload/PredownloadSegmentInfo.java
index d876192ef5f..cb4810c72d1 100644
---
a/pinot-server/src/main/java/org/apache/pinot/server/predownload/PredownloadSegmentInfo.java
+++
b/pinot-server/src/main/java/org/apache/pinot/server/predownload/PredownloadSegmentInfo.java
@@ -120,13 +120,17 @@ public class PredownloadSegmentInfo {
public SegmentDirectory initSegmentDirectory(IndexLoadingConfig
indexLoadingConfig,
PredownloadTableInfo predownloadTableInfo) {
try {
- SegmentDirectoryLoaderContext loaderContext =
- new
SegmentDirectoryLoaderContext.Builder().setTableConfig(indexLoadingConfig.getTableConfig())
-
.setSchema(indexLoadingConfig.getSchema()).setInstanceId(indexLoadingConfig.getInstanceId())
-
.setTableDataDir(indexLoadingConfig.getTableDataDir()).setSegmentName(_segmentName)
-
.setSegmentCrc(String.valueOf(_crc)).setSegmentTier(indexLoadingConfig.getSegmentTier())
-
.setInstanceTierConfigs(indexLoadingConfig.getInstanceTierConfigs())
-
.setSegmentDirectoryConfigs(indexLoadingConfig.getSegmentDirectoryConfigs()).build();
+ SegmentDirectoryLoaderContext loaderContext = new
SegmentDirectoryLoaderContext.Builder()
+ .setReadMode(indexLoadingConfig.getReadMode())
+ .setTableConfig(indexLoadingConfig.getTableConfig())
+ .setSchema(indexLoadingConfig.getSchema())
+ .setInstanceId(indexLoadingConfig.getInstanceId())
+ .setTableDataDir(indexLoadingConfig.getTableDataDir())
+ .setSegmentName(_segmentName)
+ .setSegmentCrc(String.valueOf(_crc))
+ .setSegmentTier(indexLoadingConfig.getSegmentTier())
+ .setInstanceTierConfigs(indexLoadingConfig.getInstanceTierConfigs())
+ .build();
SegmentDirectoryLoader segmentDirectoryLoader =
SegmentDirectoryLoaderRegistry.getSegmentDirectoryLoader(indexLoadingConfig.getSegmentDirectoryLoader());
File indexDir = getSegmentDataDir(predownloadTableInfo, true);
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
