This is an automated email from the ASF dual-hosted git repository.
Jackie-Jiang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 79b4354abcb Add -searchRecursively flag to CreateSegment CLI command
(#19609)
79b4354abcb is described below
commit 79b4354abcb1c57c842be3fe7b79e99f78c5f5fe
Author: Kazuki Kanda <[email protected]>
AuthorDate: Sat Sep 26 09:36:46 2026 +0900
Add -searchRecursively flag to CreateSegment CLI command (#19609)
---
.../tools/admin/command/CreateSegmentCommand.java | 23 ++++-
.../admin/command/CreateSegmentCommandTest.java | 107 +++++++++++++++++++++
2 files changed, 125 insertions(+), 5 deletions(-)
diff --git
a/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/CreateSegmentCommand.java
b/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/CreateSegmentCommand.java
index 0cc74796abf..2cd93628996 100644
---
a/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/CreateSegmentCommand.java
+++
b/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/CreateSegmentCommand.java
@@ -88,6 +88,11 @@ public class CreateSegmentCommand extends
AbstractBaseAdminCommand implements Co
@CommandLine.Option(names = {"-numThreads"}, description = "Parallelism
while generating segments, default is 1.")
private int _numThreads = 1;
+ @CommandLine.Option(names = {"-searchRecursively"}, arity = "1",
+ description = "Whether to search for data files recursively under
'dataDir', default is true. "
+ + "Pass '-searchRecursively false' to only look at the top level of
'dataDir'.")
+ private boolean _searchRecursively = true;
+
public CreateSegmentCommand setDataDir(String dataDir) {
_dataDir = dataDir;
return this;
@@ -143,13 +148,19 @@ public class CreateSegmentCommand extends
AbstractBaseAdminCommand implements Co
return this;
}
+ public CreateSegmentCommand setSearchRecursively(boolean searchRecursively) {
+ _searchRecursively = searchRecursively;
+ return this;
+ }
+
@Override
public String toString() {
return String.format(
"CreateSegment -dataDir %s -format %s -outDir %s -overwrite %s
-tableConfigFile %s -schemaFile %s "
- + "-readerConfigFile %s -retry %d -failOnEmptySegment %s
-postCreationVerification %s -numThreads %d",
+ + "-readerConfigFile %s -retry %d -failOnEmptySegment %s
-postCreationVerification %s -numThreads %d "
+ + "-searchRecursively %s",
_dataDir, _format, _outDir, _overwrite, _tableConfigFile, _schemaFile,
_readerConfigFile, _retry,
- _failOnEmptySegment, _postCreationVerification, _numThreads);
+ _failOnEmptySegment, _postCreationVerification, _numThreads,
_searchRecursively);
}
@Override
@@ -272,7 +283,7 @@ public class CreateSegmentCommand extends
AbstractBaseAdminCommand implements Co
return true;
}
- private List<String> getDataFiles(File dataDir) {
+ List<String> getDataFiles(File dataDir) {
List<String> dataFiles = new ArrayList<>();
//noinspection ConstantConditions
getDataFilesHelper(dataDir.listFiles(), dataFiles);
@@ -282,8 +293,10 @@ public class CreateSegmentCommand extends
AbstractBaseAdminCommand implements Co
private void getDataFilesHelper(File[] files, List<String> dataFiles) {
for (File file : files) {
if (file.isDirectory()) {
- //noinspection ConstantConditions
- getDataFilesHelper(file.listFiles(), dataFiles);
+ if (_searchRecursively) {
+ //noinspection ConstantConditions
+ getDataFilesHelper(file.listFiles(), dataFiles);
+ }
} else {
if (isDataFile(file.getName())) {
dataFiles.add(file.getPath());
diff --git
a/pinot-tools/src/test/java/org/apache/pinot/tools/admin/command/CreateSegmentCommandTest.java
b/pinot-tools/src/test/java/org/apache/pinot/tools/admin/command/CreateSegmentCommandTest.java
new file mode 100644
index 00000000000..533f855b845
--- /dev/null
+++
b/pinot-tools/src/test/java/org/apache/pinot/tools/admin/command/CreateSegmentCommandTest.java
@@ -0,0 +1,107 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.tools.admin.command;
+
+import java.io.File;
+import java.util.List;
+import org.apache.commons.io.FileUtils;
+import org.apache.pinot.spi.data.readers.FileFormat;
+import org.testng.annotations.AfterMethod;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+import picocli.CommandLine;
+
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertTrue;
+
+
+public class CreateSegmentCommandTest {
+ private File _tempDir;
+
+ @BeforeMethod
+ public void setUp()
+ throws Exception {
+ _tempDir = new File(FileUtils.getTempDirectory(),
"CreateSegmentCommandTest_" + System.nanoTime());
+ File nestedDir = new File(_tempDir, "nested");
+ FileUtils.forceMkdir(nestedDir);
+ FileUtils.write(new File(_tempDir, "top.csv"), "a,b\n1,2\n", "UTF-8");
+ FileUtils.write(new File(nestedDir, "nested.csv"), "a,b\n3,4\n", "UTF-8");
+ }
+
+ @AfterMethod
+ public void tearDown()
+ throws Exception {
+ FileUtils.deleteDirectory(_tempDir);
+ }
+
+ @Test
+ public void testSearchRecursivelyDefaultTrueFindsNestedFiles() {
+ CreateSegmentCommand command = new
CreateSegmentCommand().setFormat(FileFormat.CSV);
+ List<String> dataFiles = command.getDataFiles(_tempDir);
+ assertEquals(dataFiles.size(), 2, "Default behavior should search
recursively and find both files");
+ assertTrue(dataFiles.stream().anyMatch(f -> f.endsWith("top.csv")));
+ assertTrue(dataFiles.stream().anyMatch(f -> f.endsWith("nested.csv")));
+ }
+
+ @Test
+ public void testSearchRecursivelyFalseSkipsNestedFiles() {
+ CreateSegmentCommand command =
+ new
CreateSegmentCommand().setFormat(FileFormat.CSV).setSearchRecursively(false);
+ List<String> dataFiles = command.getDataFiles(_tempDir);
+ assertEquals(dataFiles.size(), 1, "Non-recursive search should only find
the top-level file");
+ assertTrue(dataFiles.get(0).endsWith("top.csv"));
+ }
+
+ @Test
+ public void testSearchRecursivelyTrueExplicitFindsNestedFiles() {
+ CreateSegmentCommand command =
+ new
CreateSegmentCommand().setFormat(FileFormat.CSV).setSearchRecursively(true);
+ List<String> dataFiles = command.getDataFiles(_tempDir);
+ assertEquals(dataFiles.size(), 2, "Explicit true should still search
recursively");
+ }
+
+ @Test
+ public void testCliParsingDefaultIsRecursive() {
+ CreateSegmentCommand command = new CreateSegmentCommand();
+ new CommandLine(command).parseArgs("-dataDir", _tempDir.getPath(),
"-format", "CSV");
+ List<String> dataFiles = command.getDataFiles(_tempDir);
+ assertEquals(dataFiles.size(), 2, "CLI without -searchRecursively should
default to recursive search");
+ assertTrue(dataFiles.stream().anyMatch(f -> f.endsWith("top.csv")));
+ assertTrue(dataFiles.stream().anyMatch(f -> f.endsWith("nested.csv")));
+ }
+
+ @Test
+ public void testCliParsingSearchRecursivelyFalse() {
+ CreateSegmentCommand command = new CreateSegmentCommand();
+ new CommandLine(command).parseArgs("-dataDir", _tempDir.getPath(),
"-format", "CSV",
+ "-searchRecursively", "false");
+ List<String> dataFiles = command.getDataFiles(_tempDir);
+ assertEquals(dataFiles.size(), 1, "CLI '-searchRecursively false' must
actually disable recursion");
+ assertTrue(dataFiles.get(0).endsWith("top.csv"));
+ }
+
+ @Test
+ public void testCliParsingSearchRecursivelyTrueExplicit() {
+ CreateSegmentCommand command = new CreateSegmentCommand();
+ new CommandLine(command).parseArgs("-dataDir", _tempDir.getPath(),
"-format", "CSV",
+ "-searchRecursively", "true");
+ List<String> dataFiles = command.getDataFiles(_tempDir);
+ assertEquals(dataFiles.size(), 2, "CLI '-searchRecursively true' should
keep recursive search");
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
