This is an automated email from the ASF dual-hosted git repository.
Jackie-Jiang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new d8d2ff2d43b docs(comments): align PathMatcher include/exclude comments
with glob+regex behavior (#8192) (#19025)
d8d2ff2d43b is described below
commit d8d2ff2d43b73000750637344834014abbbe9b3a
Author: deepinsight coder <[email protected]>
AuthorDate: Tue Aug 11 17:12:00 2026 -0700
docs(comments): align PathMatcher include/exclude comments with glob+regex
behavior (#8192) (#19025)
---
.../segment/generation/SegmentGenerationUtils.java | 6 ++---
.../SegmentGenerationAndPushTaskGenerator.java | 2 ++
.../batch/spec/SegmentGenerationJobSpec.java | 29 ++++++++--------------
.../resources/ingestion_job_spec_template.yaml | 13 ++++++----
4 files changed, 24 insertions(+), 26 deletions(-)
diff --git
a/pinot-common/src/main/java/org/apache/pinot/common/segment/generation/SegmentGenerationUtils.java
b/pinot-common/src/main/java/org/apache/pinot/common/segment/generation/SegmentGenerationUtils.java
index 8d6f1d7be68..e764127ed33 100644
---
a/pinot-common/src/main/java/org/apache/pinot/common/segment/generation/SegmentGenerationUtils.java
+++
b/pinot-common/src/main/java/org/apache/pinot/common/segment/generation/SegmentGenerationUtils.java
@@ -274,9 +274,9 @@ public class SegmentGenerationUtils {
/// @param pinotFs root directory fs
/// @param fileUri root directory uri
- /// @param includePattern optional glob patterns for files to include
- /// @param excludePattern optional glob patterns for files to exclude
- /// @param searchRecursively if ture, search files recursively from
directory specified in fileUri
+ /// @param includePattern optional Java NIO PathMatcher glob or regex
pattern for files to include
+ /// @param excludePattern optional Java NIO PathMatcher glob or regex
pattern for files to exclude
+ /// @param searchRecursively if true, search files recursively from
directory specified in fileUri
/// @return list of matching files.
/// @throws IOException on IO failure for list files in root directory.
/// @throws URISyntaxException for matching file URIs
diff --git
a/pinot-plugins/pinot-minion-tasks/pinot-minion-builtin-tasks/src/main/java/org/apache/pinot/plugin/minion/tasks/segmentgenerationandpush/SegmentGenerationAndPushTaskGenerator.java
b/pinot-plugins/pinot-minion-tasks/pinot-minion-builtin-tasks/src/main/java/org/apache/pinot/plugin/minion/tasks/segmentgenerationandpush/SegmentGenerationAndPushTaskGenerator.java
index 5044c963cda..b3d9b1f2050 100644
---
a/pinot-plugins/pinot-minion-tasks/pinot-minion-builtin-tasks/src/main/java/org/apache/pinot/plugin/minion/tasks/segmentgenerationandpush/SegmentGenerationAndPushTaskGenerator.java
+++
b/pinot-plugins/pinot-minion-tasks/pinot-minion-builtin-tasks/src/main/java/org/apache/pinot/plugin/minion/tasks/segmentgenerationandpush/SegmentGenerationAndPushTaskGenerator.java
@@ -61,6 +61,8 @@ import org.slf4j.LoggerFactory;
/// input.fs.className - Optional, the class name of filesystem to read
input data. Default to be inferred from
/// inputDirURI if not specified.
/// input.fs.prop.<keys> - Optional, defines the configs to initialize input
filesystem.
+/// includeFileNamePattern - Optional, Java NIO PathMatcher glob or regex
pattern for files to include.
+/// excludeFileNamePattern - Optional, Java NIO PathMatcher glob or regex
pattern for files to exclude.
///
/// outputDirURI - Optional, the location of output segments. Use local temp
dir with push mode TAR, If not specified.
/// output.fs.className - Optional, the class name of filesystem to write
output segments. Default to be inferred
diff --git
a/pinot-spi/src/main/java/org/apache/pinot/spi/ingestion/batch/spec/SegmentGenerationJobSpec.java
b/pinot-spi/src/main/java/org/apache/pinot/spi/ingestion/batch/spec/SegmentGenerationJobSpec.java
index 898d1cc2fed..3992c2e4202 100644
---
a/pinot-spi/src/main/java/org/apache/pinot/spi/ingestion/batch/spec/SegmentGenerationJobSpec.java
+++
b/pinot-spi/src/main/java/org/apache/pinot/spi/ingestion/batch/spec/SegmentGenerationJobSpec.java
@@ -40,16 +40,15 @@ public class SegmentGenerationJobSpec implements
Serializable {
// TODO: set the default value to false after all clients are aware of this.
private boolean _searchRecursively = true;
- /// include file name pattern, supported glob pattern.
- /// Sample usage:
- /// 'glob:\*.avro' will include all avro files just under the
inputDirURI, not sub directories;
- /// 'glob:\*\*\/\*.avro' will include all the avro files under
inputDirURI recursively.
+ /// Full path pattern for files to include. Passed to
`FileSystems.getDefault().getPathMatcher`.
+ /// Requires a `glob:` or `regex:` prefix (Java regex, not PCRE/JS). Matches
the whole
+ /// normalized path (URI schemes such as `s3://` normalize to a single
slash, e.g. `s3:/...`).
+ /// Prefer `regex:.*[.]avro` over patterns with `$` or backslashes when
job-spec templating
+ /// (Groovy SimpleTemplateEngine) is used.
private String _includeFileNamePattern;
- /// exclude file name pattern, supported glob pattern.
- /// Sample usage:
- /// 'glob:\*.avro' will exclude all avro files just under the
inputDirURI, not sub directories;
- /// 'glob:\*\*\/\*.avro' will exclude all the avro files under
inputDirURI recursively.
+ /// Full path pattern for files to exclude. Same PathMatcher rules as the
include pattern:
+ /// `glob:` or `regex:` prefix, Java regex, whole normalized path.
private String _excludeFileNamePattern;
/// Root directory of output segments, expected to have scheme configured in
PinotFS. Note that this
@@ -139,12 +138,9 @@ public class SegmentGenerationJobSpec implements
Serializable {
return _includeFileNamePattern;
}
- /// include file name pattern, supported glob pattern.
- /// Sample usage:
- /// 'glob:\*.avro' will include all avro files just under the
inputDirURI, not sub directories;
- /// 'glob:\*\*\/\*.avro' will include all the avro files under
inputDirURI recursively.
+ /// Sets the full path pattern for files to include (Java NIO PathMatcher
`glob:` or `regex:`).
///
- /// @param includeFileNamePattern
+ /// @param includeFileNamePattern pattern with a `glob:` or `regex:` prefix
public void setIncludeFileNamePattern(String includeFileNamePattern) {
_includeFileNamePattern = includeFileNamePattern;
}
@@ -153,12 +149,9 @@ public class SegmentGenerationJobSpec implements
Serializable {
return _excludeFileNamePattern;
}
- /// exclude file name pattern, supported glob pattern.
- /// Sample usage:
- /// 'glob:\*.avro' will exclude all avro files just under the
inputDirURI, not sub directories;
- /// 'glob:\*\*\/\*.avro' will exclude all the avro files under
inputDirURI recursively.
+ /// Sets the full path pattern for files to exclude (Java NIO PathMatcher
`glob:` or `regex:`).
///
- /// @param excludeFileNamePattern
+ /// @param excludeFileNamePattern pattern with a `glob:` or `regex:` prefix
public void setExcludeFileNamePattern(String excludeFileNamePattern) {
_excludeFileNamePattern = excludeFileNamePattern;
}
diff --git a/pinot-spi/src/test/resources/ingestion_job_spec_template.yaml
b/pinot-spi/src/test/resources/ingestion_job_spec_template.yaml
index 665b0be218c..1a2ecbb7b60 100644
--- a/pinot-spi/src/test/resources/ingestion_job_spec_template.yaml
+++ b/pinot-spi/src/test/resources/ingestion_job_spec_template.yaml
@@ -27,12 +27,15 @@ jobType: SegmentCreationAndTarPush
inputDirURI: 'file:///path/to/input/${ year }/${ month }/${ day }'
searchRecursively: false
-# includeFileNamePattern: include file name pattern, supported glob pattern.
+# includeFileNamePattern and excludeFileNamePattern: Java NIO PathMatcher
(glob: or regex: prefix).
+# regex: is Java regex via FileSystems.getDefault().getPathMatcher (not
PCRE/JS).
+# Patterns match the whole normalized path (s3://bucket/... becomes
s3:/bucket/...).
+# Groovy job-spec templating (SimpleTemplateEngine) treats the dollar sign
specially, so prefer
+# a bracketed dot over an escaped dot and avoid raw dollar signs in patterns.
# Sample usage:
-# 'glob:*.avro' will include all avro files just under the inputDirURI, not
sub directories;
-# 'glob:**\/*.avro' will include all the avro files under inputDirURI
recursively.
+# 'glob:**/*.avro' or 'regex:.*[.]avro' matches Avro file paths.
includeFileNamePattern: 'glob:**/*.parquet'
-excludeFileNamePattern: 'glob:**/*.avro' # 'glob:**\/*.avro' will exclude all
the avro files.
+excludeFileNamePattern: 'glob:**/*.avro'
outputDirURI: 'file:///path/to/output/${year}/${month}/${day}'
overwriteOutput: true
segmentCreationJobParallelism: 100
@@ -50,4 +53,4 @@ pinotClusterSpecs:
- controllerURI: 'localhost:9000'
pushJobSpec:
pushAttempts: 2
- pushRetryIntervalMillis: 1000
\ No newline at end of file
+ pushRetryIntervalMillis: 1000
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
