This is an automated email from the ASF dual-hosted git repository.
jackie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 6d2f749 add segment size metric on segment push (#8387)
6d2f749 is described below
commit 6d2f74978e9fff85d0786ec3d231d157c01a770f
Author: Johan Adami <[email protected]>
AuthorDate: Wed Mar 23 12:50:30 2022 -0400
add segment size metric on segment push (#8387)
Adds an offline segment size metric only when a new segment is pushed. this
is stored as a gauge and based on table name. This is useful to build detectors
in case:
- someone tries to push a segment that is too large
- your segments are organically growing in size and you want to know once
you've hit a threshold
---
.../main/java/org/apache/pinot/common/metrics/ControllerGauge.java | 4 ++++
.../api/resources/PinotSegmentUploadDownloadRestletResource.java | 5 +++++
2 files changed, 9 insertions(+)
diff --git
a/pinot-common/src/main/java/org/apache/pinot/common/metrics/ControllerGauge.java
b/pinot-common/src/main/java/org/apache/pinot/common/metrics/ControllerGauge.java
index ae33acc..f6af993 100644
---
a/pinot-common/src/main/java/org/apache/pinot/common/metrics/ControllerGauge.java
+++
b/pinot-common/src/main/java/org/apache/pinot/common/metrics/ControllerGauge.java
@@ -69,6 +69,10 @@ public enum ControllerGauge implements AbstractMetrics.Gauge
{
// Estimated size of offline table
OFFLINE_TABLE_ESTIMATED_SIZE("OfflineTableEstimatedSize", false),
+ // Size of the last uploaded offline segment
+ LAST_PUSHED_SEGMENT_SIZE("LastPushedSegmentSize", false),
+
+
// Table quota based on setting in table config
TABLE_QUOTA("TableQuotaBasedOnTableConfig", false),
diff --git
a/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotSegmentUploadDownloadRestletResource.java
b/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotSegmentUploadDownloadRestletResource.java
index 253a3fb..1d9d2e7 100644
---
a/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotSegmentUploadDownloadRestletResource.java
+++
b/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotSegmentUploadDownloadRestletResource.java
@@ -61,6 +61,7 @@ import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.tuple.ImmutablePair;
import org.apache.commons.lang3.tuple.Pair;
+import org.apache.pinot.common.metrics.ControllerGauge;
import org.apache.pinot.common.metrics.ControllerMeter;
import org.apache.pinot.common.metrics.ControllerMetrics;
import org.apache.pinot.common.restlet.resources.StartReplaceSegmentsRequest;
@@ -304,6 +305,10 @@ public class PinotSegmentUploadDownloadRestletResource {
completeZkOperations(enableParallelPushProtection, headers,
finalSegmentFile, tableNameWithType, segmentMetadata,
segmentName, zkDownloadUri, moveSegmentToFinalLocation,
crypterClassName, allowRefresh);
+ // We only set this gauge after the segment is successfully pushed.
+ _controllerMetrics.setValueOfTableGauge(tableNameWithType,
ControllerGauge.LAST_PUSHED_SEGMENT_SIZE,
+ FileUtils.sizeOfDirectory(tempSegmentDir));
+
return new SuccessResponse("Successfully uploaded segment: " +
segmentName + " of table: " + tableNameWithType);
} catch (WebApplicationException e) {
throw e;
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
