This is an automated email from the ASF dual-hosted git repository.
Jackie-Jiang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 2c313412f1c Gate preprocess() on needPreprocess() in cold-download and
consuming segment callers (#19394)
2c313412f1c is described below
commit 2c313412f1c2bd6403548b86d56cdbe8be7edabe
Author: Chaitanya Deepthi <[email protected]>
AuthorDate: Thu Sep 3 00:10:33 2026 -0700
Gate preprocess() on needPreprocess() in cold-download and consuming
segment callers (#19394)
---
.../immutable/ImmutableSegmentLoader.java | 21 ++++++++++++++++-----
1 file changed, 16 insertions(+), 5 deletions(-)
diff --git
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/indexsegment/immutable/ImmutableSegmentLoader.java
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/indexsegment/immutable/ImmutableSegmentLoader.java
index 53405a68944..724bd04ef69 100644
---
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/indexsegment/immutable/ImmutableSegmentLoader.java
+++
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/indexsegment/immutable/ImmutableSegmentLoader.java
@@ -112,8 +112,11 @@ public class ImmutableSegmentLoader {
return load(indexDir, indexLoadingConfig, needPreprocess, null, null);
}
- /// Loads the segment with specified schema and IndexLoadingConfig, and
allows to control whether to
- /// modify the segment like to convert segment format, add or remove indices.
+ /// Loads the segment with specified schema and IndexLoadingConfig.
+ ///
+ /// `needPreprocess` is the caller's opt-in signal: `false` skips preprocess
unconditionally; `true` asks the
+ /// loader to decide by calling [#needPreprocess(SegmentDirectory,
IndexLoadingConfig)]. Preprocess is only
+ /// invoked when both the caller opts in and the loader determines work is
actually pending.
public static ImmutableSegment load(File indexDir, IndexLoadingConfig
indexLoadingConfig, boolean needPreprocess,
@Nullable SegmentOperationsThrottlerSet segmentOperationsThrottlerSet,
@Nullable SegmentZKMetadata zkMetadata)
throws Exception {
@@ -124,9 +127,6 @@ public class ImmutableSegmentLoader {
if (segmentMetadata.getTotalDocs() == 0) {
return new EmptyIndexSegment(segmentMetadata);
}
- if (needPreprocess) {
- preprocess(indexDir, indexLoadingConfig, segmentOperationsThrottlerSet,
zkMetadata);
- }
String segmentName = segmentMetadata.getName();
SegmentDirectoryLoaderContext segmentLoaderContext = new
SegmentDirectoryLoaderContext.Builder()
.setReadMode(indexLoadingConfig.getReadMode())
@@ -140,6 +140,17 @@ public class ImmutableSegmentLoader {
.setInstanceTierConfigs(indexLoadingConfig.getInstanceTierConfigs())
.setSegmentCustomConfigs(zkMetadata != null ?
zkMetadata.getCustomMap() : Map.of())
.build();
+ if (needPreprocess) {
+ // Probe with the default (non-tier-aware) loader so this check never
physically moves the segment across
+ // tiers; the tier-aware loader is only used for the final open below.
+ try (SegmentDirectory checkDirectory =
+
SegmentDirectoryLoaderRegistry.getDefaultSegmentDirectoryLoader().load(indexDir.toURI(),
+ segmentLoaderContext)) {
+ if (needPreprocess(checkDirectory, indexLoadingConfig)) {
+ preprocess(indexDir, indexLoadingConfig,
segmentOperationsThrottlerSet, zkMetadata);
+ }
+ }
+ }
SegmentDirectoryLoader segmentLoader =
SegmentDirectoryLoaderRegistry.getSegmentDirectoryLoader(indexLoadingConfig.getSegmentDirectoryLoader());
SegmentDirectory segmentDirectory = segmentLoader.load(indexDir.toURI(),
segmentLoaderContext);
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
