Jackie-Jiang commented on code in PR #17239: URL: https://github.com/apache/pinot/pull/17239#discussion_r2557863004
########## pinot-common/src/main/java/org/apache/pinot/common/utils/PinotThrottledLogger.java: ########## @@ -0,0 +1,158 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pinot.common.utils; + +import com.google.common.util.concurrent.RateLimiter; +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; +import java.util.function.BiConsumer; +import javax.annotation.Nullable; +import org.apache.pinot.common.metrics.ServerMeter; +import org.apache.pinot.common.metrics.ServerMetrics; +import org.apache.pinot.spi.config.table.ingestion.IngestionConfig; +import org.slf4j.Logger; + + +/** + * Rate-limited exception logger that prevents log flooding while maintaining visibility into errors. + * + * <p>This utility wraps an SLF4J logger and applies per-exception-class rate limiting. Unlike global rate limiting + * which can suffer from the "Noisy Neighbor" problem (high-frequency errors consuming all log quota and starving + * low-frequency critical errors), this implementation maintains independent rate limiters for each exception class. + * + * <p><b>Key Features:</b> + * <ul> + * <li>Class-based fingerprinting: Each exception class gets its own rate limiter</li> + * <li>Suppression tracking: Reports count of dropped logs when rate limit is lifted</li> + * <li>Bounded memory: Exception classes are finite (~10-50 typical)</li> + * </ul> + * + * <p><b>Note:</b> This class is designed for single-threaded access per instance (one TransformPipeline per thread). + * + * <p><b>Example Usage:</b> + * <pre> + * Logger logger = LoggerFactory.getLogger(MyClass.class); + * PinotThrottledLogger throttled = new PinotThrottledLogger(logger, ingestionConfig, tableName); + * + * try { + * // some operation + * } catch (Exception e) { + * throttled.warn("Operation failed for record: " + record, e); + * } + * </pre> + * + * <p><b>Logging Behavior:</b> + * <ul> + * <li><b>Rate limit disabled (≤ 0):</b> All exceptions logged at DEBUG level (backward compatible default)</li> + * <li><b>Within rate limit (> 0, quota available):</b> Logs at WARN/ERROR level with suppression counts</li> + * <li><b>Rate limit exceeded (> 0, quota exhausted):</b> Logs at DEBUG level while tracking suppression counts</li> + * </ul> + * This ensures no exception information is lost - all exceptions are logged at either WARN/ERROR or DEBUG level. + * + * <p><b>Example Output:</b> + * <pre> + * WARN [MyClass] Operation failed for record: {id=1} + * java.lang.NumberFormatException: For input string: "abc" + * at java.lang.NumberFormatException.forInputString(...) + * + * [... 4 more similar logs within 1 minute ...] + * + * [After rate limit window passes and 10,001st exception occurs] + * WARN [MyClass] ... Suppressed 9995 occurrences of NumberFormatException ... + * WARN [MyClass] Operation failed for record: {id=10001} + * java.lang.NumberFormatException: For input string: "xyz" + * </pre> + * + * <p>Meanwhile, if a different exception type occurs (e.g., ConnectException), it logs immediately using its own + * independent rate limiter, ensuring critical errors are never starved by high-frequency errors. + * + * @see org.apache.pinot.spi.config.table.ingestion.IngestionConfig#getIngestionExceptionLogRateLimitPerMin() + */ +public class PinotThrottledLogger { + private final Logger _delegate; + + private final Map<Class<?>, ExceptionState> _stateMap = new HashMap<>(); + private final double _permitsPerSecond; + private final String _tableName; + + public PinotThrottledLogger(Logger delegate, @Nullable IngestionConfig ingestionConfig, @Nullable String tableName) { + this(delegate, getPermitsPerSecond(ingestionConfig), tableName); + } + + public PinotThrottledLogger(Logger delegate, double permitsPerSecond) { + this(delegate, permitsPerSecond, null); + } + + public PinotThrottledLogger(Logger delegate, double permitsPerSecond, @Nullable String tableName) { + _delegate = delegate; + _permitsPerSecond = permitsPerSecond; + _tableName = tableName; + } + + private static double getPermitsPerSecond(IngestionConfig ingestionConfig) { + return Optional.ofNullable(ingestionConfig).orElse(new IngestionConfig()) Review Comment: (minor) Does this involve extra allocation? Old school `null` check might be better ########## pinot-common/src/main/java/org/apache/pinot/common/utils/PinotThrottledLogger.java: ########## @@ -0,0 +1,158 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pinot.common.utils; + +import com.google.common.util.concurrent.RateLimiter; +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; +import java.util.function.BiConsumer; +import javax.annotation.Nullable; +import org.apache.pinot.common.metrics.ServerMeter; +import org.apache.pinot.common.metrics.ServerMetrics; +import org.apache.pinot.spi.config.table.ingestion.IngestionConfig; +import org.slf4j.Logger; + + +/** + * Rate-limited exception logger that prevents log flooding while maintaining visibility into errors. + * + * <p>This utility wraps an SLF4J logger and applies per-exception-class rate limiting. Unlike global rate limiting + * which can suffer from the "Noisy Neighbor" problem (high-frequency errors consuming all log quota and starving + * low-frequency critical errors), this implementation maintains independent rate limiters for each exception class. + * + * <p><b>Key Features:</b> + * <ul> + * <li>Class-based fingerprinting: Each exception class gets its own rate limiter</li> + * <li>Suppression tracking: Reports count of dropped logs when rate limit is lifted</li> + * <li>Bounded memory: Exception classes are finite (~10-50 typical)</li> + * </ul> + * + * <p><b>Note:</b> This class is designed for single-threaded access per instance (one TransformPipeline per thread). + * + * <p><b>Example Usage:</b> + * <pre> + * Logger logger = LoggerFactory.getLogger(MyClass.class); + * PinotThrottledLogger throttled = new PinotThrottledLogger(logger, ingestionConfig, tableName); + * + * try { + * // some operation + * } catch (Exception e) { + * throttled.warn("Operation failed for record: " + record, e); + * } + * </pre> + * + * <p><b>Logging Behavior:</b> + * <ul> + * <li><b>Rate limit disabled (≤ 0):</b> All exceptions logged at DEBUG level (backward compatible default)</li> + * <li><b>Within rate limit (> 0, quota available):</b> Logs at WARN/ERROR level with suppression counts</li> + * <li><b>Rate limit exceeded (> 0, quota exhausted):</b> Logs at DEBUG level while tracking suppression counts</li> + * </ul> + * This ensures no exception information is lost - all exceptions are logged at either WARN/ERROR or DEBUG level. + * + * <p><b>Example Output:</b> + * <pre> + * WARN [MyClass] Operation failed for record: {id=1} + * java.lang.NumberFormatException: For input string: "abc" + * at java.lang.NumberFormatException.forInputString(...) + * + * [... 4 more similar logs within 1 minute ...] + * + * [After rate limit window passes and 10,001st exception occurs] + * WARN [MyClass] ... Suppressed 9995 occurrences of NumberFormatException ... + * WARN [MyClass] Operation failed for record: {id=10001} + * java.lang.NumberFormatException: For input string: "xyz" + * </pre> + * + * <p>Meanwhile, if a different exception type occurs (e.g., ConnectException), it logs immediately using its own + * independent rate limiter, ensuring critical errors are never starved by high-frequency errors. + * + * @see org.apache.pinot.spi.config.table.ingestion.IngestionConfig#getIngestionExceptionLogRateLimitPerMin() + */ +public class PinotThrottledLogger { + private final Logger _delegate; + + private final Map<Class<?>, ExceptionState> _stateMap = new HashMap<>(); + private final double _permitsPerSecond; + private final String _tableName; + + public PinotThrottledLogger(Logger delegate, @Nullable IngestionConfig ingestionConfig, @Nullable String tableName) { + this(delegate, getPermitsPerSecond(ingestionConfig), tableName); + } + + public PinotThrottledLogger(Logger delegate, double permitsPerSecond) { + this(delegate, permitsPerSecond, null); + } + + public PinotThrottledLogger(Logger delegate, double permitsPerSecond, @Nullable String tableName) { + _delegate = delegate; + _permitsPerSecond = permitsPerSecond; + _tableName = tableName; + } + + private static double getPermitsPerSecond(IngestionConfig ingestionConfig) { Review Comment: (minor) ```suggestion private static double getPermitsPerSecond(@Nullable IngestionConfig ingestionConfig) { ``` ########## pinot-common/src/main/java/org/apache/pinot/common/metrics/ServerMeter.java: ########## @@ -245,7 +245,9 @@ public enum ServerMeter implements AbstractMetrics.Meter { TRANSFORMATION_ERROR_COUNT("rows", false), DROPPED_RECORD_COUNT("rows", false), - CORRUPTED_RECORD_COUNT("rows", false); + CORRUPTED_RECORD_COUNT("rows", false), + LOGS_DROPPED_BY_THROTTLED_LOGGER("logs", false, Review Comment: (optional) Do you see this metrics useful given we already collect metrics for incomplete rows? ########## pinot-common/src/main/java/org/apache/pinot/common/utils/PinotThrottledLogger.java: ########## @@ -0,0 +1,158 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pinot.common.utils; + +import com.google.common.util.concurrent.RateLimiter; +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; +import java.util.function.BiConsumer; +import javax.annotation.Nullable; +import org.apache.pinot.common.metrics.ServerMeter; +import org.apache.pinot.common.metrics.ServerMetrics; +import org.apache.pinot.spi.config.table.ingestion.IngestionConfig; +import org.slf4j.Logger; + + +/** + * Rate-limited exception logger that prevents log flooding while maintaining visibility into errors. + * + * <p>This utility wraps an SLF4J logger and applies per-exception-class rate limiting. Unlike global rate limiting + * which can suffer from the "Noisy Neighbor" problem (high-frequency errors consuming all log quota and starving + * low-frequency critical errors), this implementation maintains independent rate limiters for each exception class. + * + * <p><b>Key Features:</b> + * <ul> + * <li>Class-based fingerprinting: Each exception class gets its own rate limiter</li> + * <li>Suppression tracking: Reports count of dropped logs when rate limit is lifted</li> + * <li>Bounded memory: Exception classes are finite (~10-50 typical)</li> + * </ul> + * + * <p><b>Note:</b> This class is designed for single-threaded access per instance (one TransformPipeline per thread). + * + * <p><b>Example Usage:</b> + * <pre> + * Logger logger = LoggerFactory.getLogger(MyClass.class); + * PinotThrottledLogger throttled = new PinotThrottledLogger(logger, ingestionConfig, tableName); + * + * try { + * // some operation + * } catch (Exception e) { + * throttled.warn("Operation failed for record: " + record, e); + * } + * </pre> + * + * <p><b>Logging Behavior:</b> + * <ul> + * <li><b>Rate limit disabled (≤ 0):</b> All exceptions logged at DEBUG level (backward compatible default)</li> + * <li><b>Within rate limit (> 0, quota available):</b> Logs at WARN/ERROR level with suppression counts</li> + * <li><b>Rate limit exceeded (> 0, quota exhausted):</b> Logs at DEBUG level while tracking suppression counts</li> + * </ul> + * This ensures no exception information is lost - all exceptions are logged at either WARN/ERROR or DEBUG level. + * + * <p><b>Example Output:</b> + * <pre> + * WARN [MyClass] Operation failed for record: {id=1} + * java.lang.NumberFormatException: For input string: "abc" + * at java.lang.NumberFormatException.forInputString(...) + * + * [... 4 more similar logs within 1 minute ...] + * + * [After rate limit window passes and 10,001st exception occurs] + * WARN [MyClass] ... Suppressed 9995 occurrences of NumberFormatException ... + * WARN [MyClass] Operation failed for record: {id=10001} + * java.lang.NumberFormatException: For input string: "xyz" + * </pre> + * + * <p>Meanwhile, if a different exception type occurs (e.g., ConnectException), it logs immediately using its own + * independent rate limiter, ensuring critical errors are never starved by high-frequency errors. + * + * @see org.apache.pinot.spi.config.table.ingestion.IngestionConfig#getIngestionExceptionLogRateLimitPerMin() + */ +public class PinotThrottledLogger { Review Comment: (minor) Suggest removing the `Pinot` prefix ########## pinot-common/src/main/java/org/apache/pinot/common/utils/PinotThrottledLogger.java: ########## @@ -0,0 +1,158 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pinot.common.utils; + +import com.google.common.util.concurrent.RateLimiter; +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; +import java.util.function.BiConsumer; +import javax.annotation.Nullable; +import org.apache.pinot.common.metrics.ServerMeter; +import org.apache.pinot.common.metrics.ServerMetrics; +import org.apache.pinot.spi.config.table.ingestion.IngestionConfig; +import org.slf4j.Logger; + + +/** + * Rate-limited exception logger that prevents log flooding while maintaining visibility into errors. + * + * <p>This utility wraps an SLF4J logger and applies per-exception-class rate limiting. Unlike global rate limiting + * which can suffer from the "Noisy Neighbor" problem (high-frequency errors consuming all log quota and starving + * low-frequency critical errors), this implementation maintains independent rate limiters for each exception class. + * + * <p><b>Key Features:</b> + * <ul> + * <li>Class-based fingerprinting: Each exception class gets its own rate limiter</li> + * <li>Suppression tracking: Reports count of dropped logs when rate limit is lifted</li> + * <li>Bounded memory: Exception classes are finite (~10-50 typical)</li> + * </ul> + * + * <p><b>Note:</b> This class is designed for single-threaded access per instance (one TransformPipeline per thread). + * + * <p><b>Example Usage:</b> + * <pre> + * Logger logger = LoggerFactory.getLogger(MyClass.class); + * PinotThrottledLogger throttled = new PinotThrottledLogger(logger, ingestionConfig, tableName); + * + * try { + * // some operation + * } catch (Exception e) { + * throttled.warn("Operation failed for record: " + record, e); + * } + * </pre> + * + * <p><b>Logging Behavior:</b> + * <ul> + * <li><b>Rate limit disabled (≤ 0):</b> All exceptions logged at DEBUG level (backward compatible default)</li> + * <li><b>Within rate limit (> 0, quota available):</b> Logs at WARN/ERROR level with suppression counts</li> + * <li><b>Rate limit exceeded (> 0, quota exhausted):</b> Logs at DEBUG level while tracking suppression counts</li> + * </ul> + * This ensures no exception information is lost - all exceptions are logged at either WARN/ERROR or DEBUG level. + * + * <p><b>Example Output:</b> + * <pre> + * WARN [MyClass] Operation failed for record: {id=1} + * java.lang.NumberFormatException: For input string: "abc" + * at java.lang.NumberFormatException.forInputString(...) + * + * [... 4 more similar logs within 1 minute ...] + * + * [After rate limit window passes and 10,001st exception occurs] + * WARN [MyClass] ... Suppressed 9995 occurrences of NumberFormatException ... + * WARN [MyClass] Operation failed for record: {id=10001} + * java.lang.NumberFormatException: For input string: "xyz" + * </pre> + * + * <p>Meanwhile, if a different exception type occurs (e.g., ConnectException), it logs immediately using its own + * independent rate limiter, ensuring critical errors are never starved by high-frequency errors. + * + * @see org.apache.pinot.spi.config.table.ingestion.IngestionConfig#getIngestionExceptionLogRateLimitPerMin() + */ +public class PinotThrottledLogger { + private final Logger _delegate; + + private final Map<Class<?>, ExceptionState> _stateMap = new HashMap<>(); + private final double _permitsPerSecond; + private final String _tableName; + + public PinotThrottledLogger(Logger delegate, @Nullable IngestionConfig ingestionConfig, @Nullable String tableName) { + this(delegate, getPermitsPerSecond(ingestionConfig), tableName); + } + + public PinotThrottledLogger(Logger delegate, double permitsPerSecond) { + this(delegate, permitsPerSecond, null); + } + + public PinotThrottledLogger(Logger delegate, double permitsPerSecond, @Nullable String tableName) { + _delegate = delegate; + _permitsPerSecond = permitsPerSecond; + _tableName = tableName; + } + + private static double getPermitsPerSecond(IngestionConfig ingestionConfig) { + return Optional.ofNullable(ingestionConfig).orElse(new IngestionConfig()) + .getIngestionExceptionLogRateLimitPerMin() / 60.0; + } + + public void warn(String msg, Throwable t) { + logWithRateLimit(msg, t, _delegate::warn); + } + + public void error(String msg, Throwable t) { + logWithRateLimit(msg, t, _delegate::error); + } + + private void logWithRateLimit(String msg, Throwable t, BiConsumer<String, Throwable> consumer) { + if (_permitsPerSecond <= 0) { Review Comment: Consider modeling this as a separate `DebugOnlyLogger`. We can also have a `UnthrottledLogger` if necessary ########## pinot-common/src/main/java/org/apache/pinot/common/utils/PinotThrottledLogger.java: ########## @@ -0,0 +1,158 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pinot.common.utils; + +import com.google.common.util.concurrent.RateLimiter; +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; +import java.util.function.BiConsumer; +import javax.annotation.Nullable; +import org.apache.pinot.common.metrics.ServerMeter; +import org.apache.pinot.common.metrics.ServerMetrics; +import org.apache.pinot.spi.config.table.ingestion.IngestionConfig; +import org.slf4j.Logger; + + +/** + * Rate-limited exception logger that prevents log flooding while maintaining visibility into errors. + * + * <p>This utility wraps an SLF4J logger and applies per-exception-class rate limiting. Unlike global rate limiting + * which can suffer from the "Noisy Neighbor" problem (high-frequency errors consuming all log quota and starving + * low-frequency critical errors), this implementation maintains independent rate limiters for each exception class. + * + * <p><b>Key Features:</b> + * <ul> + * <li>Class-based fingerprinting: Each exception class gets its own rate limiter</li> + * <li>Suppression tracking: Reports count of dropped logs when rate limit is lifted</li> + * <li>Bounded memory: Exception classes are finite (~10-50 typical)</li> + * </ul> + * + * <p><b>Note:</b> This class is designed for single-threaded access per instance (one TransformPipeline per thread). + * + * <p><b>Example Usage:</b> + * <pre> + * Logger logger = LoggerFactory.getLogger(MyClass.class); + * PinotThrottledLogger throttled = new PinotThrottledLogger(logger, ingestionConfig, tableName); + * + * try { + * // some operation + * } catch (Exception e) { + * throttled.warn("Operation failed for record: " + record, e); + * } + * </pre> + * + * <p><b>Logging Behavior:</b> + * <ul> + * <li><b>Rate limit disabled (≤ 0):</b> All exceptions logged at DEBUG level (backward compatible default)</li> + * <li><b>Within rate limit (> 0, quota available):</b> Logs at WARN/ERROR level with suppression counts</li> + * <li><b>Rate limit exceeded (> 0, quota exhausted):</b> Logs at DEBUG level while tracking suppression counts</li> + * </ul> + * This ensures no exception information is lost - all exceptions are logged at either WARN/ERROR or DEBUG level. + * + * <p><b>Example Output:</b> + * <pre> + * WARN [MyClass] Operation failed for record: {id=1} + * java.lang.NumberFormatException: For input string: "abc" + * at java.lang.NumberFormatException.forInputString(...) + * + * [... 4 more similar logs within 1 minute ...] + * + * [After rate limit window passes and 10,001st exception occurs] + * WARN [MyClass] ... Suppressed 9995 occurrences of NumberFormatException ... + * WARN [MyClass] Operation failed for record: {id=10001} + * java.lang.NumberFormatException: For input string: "xyz" + * </pre> + * + * <p>Meanwhile, if a different exception type occurs (e.g., ConnectException), it logs immediately using its own + * independent rate limiter, ensuring critical errors are never starved by high-frequency errors. + * + * @see org.apache.pinot.spi.config.table.ingestion.IngestionConfig#getIngestionExceptionLogRateLimitPerMin() + */ +public class PinotThrottledLogger { + private final Logger _delegate; + + private final Map<Class<?>, ExceptionState> _stateMap = new HashMap<>(); + private final double _permitsPerSecond; + private final String _tableName; + + public PinotThrottledLogger(Logger delegate, @Nullable IngestionConfig ingestionConfig, @Nullable String tableName) { + this(delegate, getPermitsPerSecond(ingestionConfig), tableName); + } + + public PinotThrottledLogger(Logger delegate, double permitsPerSecond) { + this(delegate, permitsPerSecond, null); + } + + public PinotThrottledLogger(Logger delegate, double permitsPerSecond, @Nullable String tableName) { + _delegate = delegate; + _permitsPerSecond = permitsPerSecond; + _tableName = tableName; + } + + private static double getPermitsPerSecond(IngestionConfig ingestionConfig) { + return Optional.ofNullable(ingestionConfig).orElse(new IngestionConfig()) + .getIngestionExceptionLogRateLimitPerMin() / 60.0; + } + + public void warn(String msg, Throwable t) { + logWithRateLimit(msg, t, _delegate::warn); + } + + public void error(String msg, Throwable t) { + logWithRateLimit(msg, t, _delegate::error); + } + + private void logWithRateLimit(String msg, Throwable t, BiConsumer<String, Throwable> consumer) { + if (_permitsPerSecond <= 0) { + _delegate.debug(msg, t); + return; + } + + final Class<?> exceptionClass = t.getClass(); + final ExceptionState state = _stateMap.computeIfAbsent(exceptionClass, k -> new ExceptionState(_permitsPerSecond)); + + if (state._rateLimiter.tryAcquire()) { + long droppedCount = state._droppedCount; + if (droppedCount > 0) { + consumer.accept(String.format("Dropped %d occurrences of %s", + droppedCount, + exceptionClass.getSimpleName()), null); + state._droppedCount = 0; + } + consumer.accept(msg, t); + } else { + state._droppedCount++; + _delegate.debug(msg, t); + if (_tableName != null) { + ServerMetrics.get().addMeteredTableValue(_tableName, ServerMeter.LOGS_DROPPED_BY_THROTTLED_LOGGER, 1L); + } + } + } + + private static class ExceptionState { + final RateLimiter _rateLimiter; Review Comment: We don't really need the thread-safety from `RateLimiter`. Please verify the overhead of it and see if it could potentially lower the ingestion performance (it is called on a per record per transformer basis in the worst case) ########## pinot-common/src/main/java/org/apache/pinot/common/utils/PinotThrottledLogger.java: ########## @@ -0,0 +1,158 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pinot.common.utils; + +import com.google.common.util.concurrent.RateLimiter; +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; +import java.util.function.BiConsumer; +import javax.annotation.Nullable; +import org.apache.pinot.common.metrics.ServerMeter; +import org.apache.pinot.common.metrics.ServerMetrics; +import org.apache.pinot.spi.config.table.ingestion.IngestionConfig; +import org.slf4j.Logger; + + +/** + * Rate-limited exception logger that prevents log flooding while maintaining visibility into errors. + * + * <p>This utility wraps an SLF4J logger and applies per-exception-class rate limiting. Unlike global rate limiting + * which can suffer from the "Noisy Neighbor" problem (high-frequency errors consuming all log quota and starving + * low-frequency critical errors), this implementation maintains independent rate limiters for each exception class. + * + * <p><b>Key Features:</b> + * <ul> + * <li>Class-based fingerprinting: Each exception class gets its own rate limiter</li> + * <li>Suppression tracking: Reports count of dropped logs when rate limit is lifted</li> + * <li>Bounded memory: Exception classes are finite (~10-50 typical)</li> + * </ul> + * + * <p><b>Note:</b> This class is designed for single-threaded access per instance (one TransformPipeline per thread). + * + * <p><b>Example Usage:</b> + * <pre> + * Logger logger = LoggerFactory.getLogger(MyClass.class); + * PinotThrottledLogger throttled = new PinotThrottledLogger(logger, ingestionConfig, tableName); + * + * try { + * // some operation + * } catch (Exception e) { + * throttled.warn("Operation failed for record: " + record, e); + * } + * </pre> + * + * <p><b>Logging Behavior:</b> + * <ul> + * <li><b>Rate limit disabled (≤ 0):</b> All exceptions logged at DEBUG level (backward compatible default)</li> + * <li><b>Within rate limit (> 0, quota available):</b> Logs at WARN/ERROR level with suppression counts</li> + * <li><b>Rate limit exceeded (> 0, quota exhausted):</b> Logs at DEBUG level while tracking suppression counts</li> + * </ul> + * This ensures no exception information is lost - all exceptions are logged at either WARN/ERROR or DEBUG level. + * + * <p><b>Example Output:</b> + * <pre> + * WARN [MyClass] Operation failed for record: {id=1} + * java.lang.NumberFormatException: For input string: "abc" + * at java.lang.NumberFormatException.forInputString(...) + * + * [... 4 more similar logs within 1 minute ...] + * + * [After rate limit window passes and 10,001st exception occurs] + * WARN [MyClass] ... Suppressed 9995 occurrences of NumberFormatException ... + * WARN [MyClass] Operation failed for record: {id=10001} + * java.lang.NumberFormatException: For input string: "xyz" + * </pre> + * + * <p>Meanwhile, if a different exception type occurs (e.g., ConnectException), it logs immediately using its own + * independent rate limiter, ensuring critical errors are never starved by high-frequency errors. + * + * @see org.apache.pinot.spi.config.table.ingestion.IngestionConfig#getIngestionExceptionLogRateLimitPerMin() + */ +public class PinotThrottledLogger { + private final Logger _delegate; + + private final Map<Class<?>, ExceptionState> _stateMap = new HashMap<>(); + private final double _permitsPerSecond; + private final String _tableName; + + public PinotThrottledLogger(Logger delegate, @Nullable IngestionConfig ingestionConfig, @Nullable String tableName) { + this(delegate, getPermitsPerSecond(ingestionConfig), tableName); + } + + public PinotThrottledLogger(Logger delegate, double permitsPerSecond) { + this(delegate, permitsPerSecond, null); + } + + public PinotThrottledLogger(Logger delegate, double permitsPerSecond, @Nullable String tableName) { + _delegate = delegate; + _permitsPerSecond = permitsPerSecond; + _tableName = tableName; + } + + private static double getPermitsPerSecond(IngestionConfig ingestionConfig) { + return Optional.ofNullable(ingestionConfig).orElse(new IngestionConfig()) + .getIngestionExceptionLogRateLimitPerMin() / 60.0; + } + + public void warn(String msg, Throwable t) { + logWithRateLimit(msg, t, _delegate::warn); + } + + public void error(String msg, Throwable t) { + logWithRateLimit(msg, t, _delegate::error); + } + + private void logWithRateLimit(String msg, Throwable t, BiConsumer<String, Throwable> consumer) { + if (_permitsPerSecond <= 0) { + _delegate.debug(msg, t); + return; + } + + final Class<?> exceptionClass = t.getClass(); Review Comment: (minor, convention) We don't usually use `final` in local variable -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
