Jackie-Jiang commented on code in PR #8786:
URL: https://github.com/apache/pinot/pull/8786#discussion_r884027264
##########
pinot-controller/src/main/java/org/apache/pinot/controller/validation/RealtimeSegmentValidationManager.java:
##########
@@ -90,86 +88,83 @@ protected Context preprocess(Properties
periodicTaskProperties) {
@Override
protected void processTable(String tableNameWithType, Context context) {
- TableType tableType =
TableNameBuilder.getTableTypeFromTableName(tableNameWithType);
- if (tableType == TableType.REALTIME) {
+ if (!TableNameBuilder.isRealtimeTableResource(tableNameWithType)) {
+ return;
+ }
- TableConfig tableConfig =
_pinotHelixResourceManager.getTableConfig(tableNameWithType);
- if (tableConfig == null) {
- LOGGER.warn("Failed to find table config for table: {}, skipping
validation", tableNameWithType);
- return;
- }
+ TableConfig tableConfig =
_pinotHelixResourceManager.getTableConfig(tableNameWithType);
+ if (tableConfig == null) {
+ LOGGER.warn("Failed to find table config for table: {}, skipping
validation", tableNameWithType);
+ return;
+ }
+ PartitionLevelStreamConfig streamConfig = new
PartitionLevelStreamConfig(tableConfig.getTableName(),
+ IngestionConfigUtils.getStreamConfigMap(tableConfig));
- if (context._runSegmentLevelValidation) {
- runSegmentLevelValidation(tableConfig);
- }
+ if (context._runSegmentLevelValidation) {
+ runSegmentLevelValidation(tableConfig, streamConfig);
+ }
- PartitionLevelStreamConfig streamConfig = new
PartitionLevelStreamConfig(tableConfig.getTableName(),
- IngestionConfigUtils.getStreamConfigMap(tableConfig));
- if (streamConfig.hasLowLevelConsumerType()) {
- _llcRealtimeSegmentManager.ensureAllPartitionsConsuming(tableConfig,
- streamConfig, context._recreateDeletedConsumingSegment);
- }
+ if (streamConfig.hasLowLevelConsumerType()) {
+ _llcRealtimeSegmentManager.ensureAllPartitionsConsuming(tableConfig,
streamConfig,
+ context._recreateDeletedConsumingSegment);
}
}
- private void runSegmentLevelValidation(TableConfig tableConfig) {
+ private void runSegmentLevelValidation(TableConfig tableConfig,
PartitionLevelStreamConfig streamConfig) {
String realtimeTableName = tableConfig.getTableName();
List<SegmentZKMetadata> segmentsZKMetadata =
_pinotHelixResourceManager.getSegmentsZKMetadata(realtimeTableName);
- boolean countHLCSegments = true; // false if this table has ONLY LLC
segments (i.e. fully migrated)
- StreamConfig streamConfig =
- new StreamConfig(realtimeTableName,
IngestionConfigUtils.getStreamConfigMap(tableConfig));
- if (streamConfig.hasLowLevelConsumerType() &&
!streamConfig.hasHighLevelConsumerType()) {
- countHLCSegments = false;
- }
- // Update the gauge to contain the total document count in the segments
-
_validationMetrics.updateTotalDocumentCountGauge(tableConfig.getTableName(),
- computeRealtimeTotalDocumentInSegments(segmentsZKMetadata,
countHLCSegments));
- if (streamConfig.hasLowLevelConsumerType()
- &&
_llcRealtimeSegmentManager.isDeepStoreLLCSegmentUploadRetryEnabled()) {
+ // Update the total document count gauge
+ // Count HLC segments if there is no LLC consumer configured
+ boolean hasLLC = streamConfig.hasLowLevelConsumerType();
+ _validationMetrics.updateTotalDocumentCountGauge(realtimeTableName,
+ computeTotalDocumentCount(segmentsZKMetadata, hasLLC));
Review Comment:
Reverted it back to the old behavior (if both HLC and LLC consumer are
configured, count HLC segments). No one is using HLC right now, but agree we
should try to keep the old behavior
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
