This is an automated email from the ASF dual-hosted git repository.
xiangfu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 8013a2d Use minion data directory as tmp directory for
SegmentGenerationAndPushTask (#6560)
8013a2d is described below
commit 8013a2d97ead3bd3a5f75cc71b018cb85a796e55
Author: Xiang Fu <[email protected]>
AuthorDate: Tue Feb 9 21:57:47 2021 -0800
Use minion data directory as tmp directory for SegmentGenerationAndPushTask
(#6560)
* Use minion data directory as tmp directory for
SegmentGenerationAndPushTask
* Update MinionStarter.java
---
.../src/main/java/org/apache/pinot/minion/MinionStarter.java | 6 ++++++
.../pinot/minion/executor/SegmentGenerationAndPushTaskExecutor.java | 6 ++++--
2 files changed, 10 insertions(+), 2 deletions(-)
diff --git
a/pinot-minion/src/main/java/org/apache/pinot/minion/MinionStarter.java
b/pinot-minion/src/main/java/org/apache/pinot/minion/MinionStarter.java
index 0d3d40f..015b8e2 100644
--- a/pinot-minion/src/main/java/org/apache/pinot/minion/MinionStarter.java
+++ b/pinot-minion/src/main/java/org/apache/pinot/minion/MinionStarter.java
@@ -226,6 +226,12 @@ public class MinionStarter implements ServiceStartable {
_helixManager.disconnect();
LOGGER.info("Deregistering service status handler");
ServiceStatus.removeServiceStatusCallback(_instanceId);
+ LOGGER.info("Clean up Minion data directory");
+ try {
+ FileUtils.cleanDirectory(MinionContext.getInstance().getDataDir());
+ } catch (IOException e) {
+ LOGGER.warn("Failed to clean up Minion data directory: {}",
MinionContext.getInstance().getDataDir(), e);
+ }
LOGGER.info("Pinot minion stopped");
}
diff --git
a/pinot-minion/src/main/java/org/apache/pinot/minion/executor/SegmentGenerationAndPushTaskExecutor.java
b/pinot-minion/src/main/java/org/apache/pinot/minion/executor/SegmentGenerationAndPushTaskExecutor.java
index 525d395..e490de2 100644
---
a/pinot-minion/src/main/java/org/apache/pinot/minion/executor/SegmentGenerationAndPushTaskExecutor.java
+++
b/pinot-minion/src/main/java/org/apache/pinot/minion/executor/SegmentGenerationAndPushTaskExecutor.java
@@ -27,10 +27,11 @@ import java.util.List;
import java.util.Map;
import java.util.UUID;
import org.apache.commons.io.FileUtils;
+import org.apache.pinot.common.segment.generation.SegmentGenerationUtils;
import org.apache.pinot.common.utils.TarGzCompressionUtils;
import org.apache.pinot.core.minion.PinotTaskConfig;
+import org.apache.pinot.minion.MinionContext;
import
org.apache.pinot.plugin.ingestion.batch.common.SegmentGenerationTaskRunner;
-import org.apache.pinot.common.segment.generation.SegmentGenerationUtils;
import org.apache.pinot.plugin.ingestion.batch.common.SegmentPushUtils;
import org.apache.pinot.spi.config.table.TableConfig;
import org.apache.pinot.spi.data.Schema;
@@ -102,7 +103,8 @@ public class SegmentGenerationAndPushTaskExecutor extends
BaseTaskExecutor {
LOGGER.info("Executing SegmentGenerationAndPushTask with task config: {}",
pinotTaskConfig);
Map<String, String> taskConfigs = pinotTaskConfig.getConfigs();
SegmentGenerationAndPushResult.Builder resultBuilder = new
SegmentGenerationAndPushResult.Builder();
- File localTempDir = new File(FileUtils.getTempDirectory(), "pinot-" +
UUID.randomUUID());
+ File localTempDir = new File(new
File(MinionContext.getInstance().getDataDir(),
"SegmentGenerationAndPushResult"),
+ "tmp-" + UUID.randomUUID());
try {
// Generate Pinot Segment
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
