This is an automated email from the ASF dual-hosted git repository.
snlee pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
The following commit(s) were added to refs/heads/master by this push:
new c752a4f Improve the logging for segment merge command (#3573)
c752a4f is described below
commit c752a4fd2a1e91175c3d01935abb7a41a5b82fa0
Author: Seunghyun Lee <[email protected]>
AuthorDate: Fri Nov 30 17:49:30 2018 -0800
Improve the logging for segment merge command (#3573)
1. add min start / max end time for input segments and merged segment
2. added the total number of documents for input and merged segment
---
.../segment/converter/SegmentMergeCommand.java | 47 +++++++++++++---------
1 file changed, 27 insertions(+), 20 deletions(-)
diff --git
a/pinot-tools/src/main/java/com/linkedin/pinot/tools/segment/converter/SegmentMergeCommand.java
b/pinot-tools/src/main/java/com/linkedin/pinot/tools/segment/converter/SegmentMergeCommand.java
index 1c0da29..205231e 100644
---
a/pinot-tools/src/main/java/com/linkedin/pinot/tools/segment/converter/SegmentMergeCommand.java
+++
b/pinot-tools/src/main/java/com/linkedin/pinot/tools/segment/converter/SegmentMergeCommand.java
@@ -151,9 +151,27 @@ public class SegmentMergeCommand extends
AbstractBaseAdminCommand implements Com
LOGGER.info("Table config: {}", tableConfig);
LOGGER.info("Schema : {}", schema);
+ // Compute mix/max time from segment metadata
+ long minStartTime = Long.MAX_VALUE;
+ long maxEndTime = Long.MIN_VALUE;
+ long totalNumDocsBeforeMerge = 0L;
+ for (File indexDir : inputIndexDirs) {
+ SegmentMetadata segmentMetadata = new SegmentMetadataImpl(indexDir);
+ long currentStartTime = segmentMetadata.getStartTime();
+ if (currentStartTime < minStartTime) {
+ minStartTime = currentStartTime;
+ }
+
+ long currentEndTime = segmentMetadata.getEndTime();
+ if (currentEndTime > maxEndTime) {
+ maxEndTime = currentEndTime;
+ }
+ totalNumDocsBeforeMerge += segmentMetadata.getTotalDocs();
+ }
+
// Compute segment name if it is not specified
if (_outputSegmentName == null) {
- _outputSegmentName = getDefaultSegmentName(tableConfig, schema,
inputIndexDirs);
+ _outputSegmentName = getDefaultSegmentName(tableConfig, schema,
inputIndexDirs, minStartTime, maxEndTime);
}
LOGGER.info("Output segment name: {}", _outputSegmentName);
@@ -195,6 +213,11 @@ public class SegmentMergeCommand extends
AbstractBaseAdminCommand implements Com
FileUtils.moveDirectory(outputSegment, finalOutputPath);
}
LOGGER.info("Segment has been merged correctly. Output file is located
at {}", finalOutputPath);
+ LOGGER.info("Min start time / max end time for input segments : " +
minStartTime + " / " + maxEndTime);
+ LOGGER.info("Min start time / max end time for merged segment: " +
outputSegmentMetadata.getStartTime() + " / "
+ + outputSegmentMetadata.getEndTime());
+ LOGGER.info("Total number of documents for input segments: " +
totalNumDocsBeforeMerge);
+ LOGGER.info("Total number of documents for merged segment: " +
outputSegmentMetadata.getTotalDocs());
} finally {
// Clean up working directory
FileUtils.deleteQuietly(workingDir);
@@ -207,25 +230,9 @@ public class SegmentMergeCommand extends
AbstractBaseAdminCommand implements Com
return "Create the merged segment using concatenation";
}
- private String getDefaultSegmentName(TableConfig tableConfig, Schema schema,
List<File> inputIndexDirs)
- throws Exception {
- String tableName =
TableNameBuilder.extractRawTableName(tableConfig.getTableName());
-
- // Compute mix/max time from segment metadata
- long minStartTime = Long.MAX_VALUE;
- long maxEndTime = Long.MIN_VALUE;
- for (File indexDir : inputIndexDirs) {
- SegmentMetadata segmentMetadata = new SegmentMetadataImpl(indexDir);
- long currentStartTime = segmentMetadata.getStartTime();
- if (currentStartTime < minStartTime) {
- minStartTime = currentStartTime;
- }
-
- long currentEndTime = segmentMetadata.getEndTime();
- if (currentEndTime > maxEndTime) {
- maxEndTime = currentEndTime;
- }
- }
+ private String getDefaultSegmentName(TableConfig tableConfig, Schema schema,
List<File> inputIndexDirs,
+ long minStartTime, long maxEndTime) throws Exception {
+ String tableName = tableConfig.getTableName();
// Fetch time related configurations from schema and table config.
String pushFrequency =
tableConfig.getValidationConfig().getSegmentPushFrequency();
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
