hudi-agent commented on code in PR #20189:
URL: https://github.com/apache/hudi/pull/20189#discussion_r4235267799
##########
hudi-common/src/main/java/org/apache/hudi/common/util/RateLimiter.java:
##########
@@ -68,27 +110,71 @@ public boolean tryAcquire(int numPermits) {
}
public boolean acquire(int numOps) {
+ ValidationUtils.checkArgument(numOps > 0 && numOps <= maxPermits,
+ "Number of permits must be between one and the configured maximum");
+ return acquireInternal(numOps);
+ }
+
+ public boolean acquire() {
+ return acquireInternal(1);
+ }
+
+ private boolean acquireInternal(int numOps) {
try {
- while (!semaphore.tryAcquire(numOps)) {
+ while (!stopped && !semaphore.tryAcquire(numOps)) {
+ if (permitWaitCallback != null) {
+ permitWaitCallback.run();
+ }
Thread.sleep(WAIT_BEFORE_NEXT_ACQUIRE_PERMIT_IN_MS);
}
+ ValidationUtils.checkState(!stopped, "Rate limiter is stopped");
log.debug("acquire permits: {}, maxPermits: {}", numOps, maxPermits);
} catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
throw new RuntimeException("Unable to acquire permits", e);
}
return true;
}
- public void stop() {
- scheduler.shutdownNow();
+ public synchronized void stop() {
+ if (stopped) {
+ return;
+ }
+ stopped = true;
+ if (scheduler != null) {
+ scheduler.shutdownNow();
+ }
+ }
+
+ public boolean isStopped() {
+ return stopped;
}
- public void releasePermitsPeriodically() {
- scheduler =
Executors.newScheduledThreadPool(SCHEDULER_CORE_THREAD_POOL_SIZE);
- scheduler.scheduleAtFixedRate(() -> {
- log.debug("Release permits: maxPermits: {}, available: {}", maxPermits,
semaphore.availablePermits());
- semaphore.release(maxPermits - semaphore.availablePermits());
- }, RELEASE_PERMITS_PERIOD_IN_SECONDS, RELEASE_PERMITS_PERIOD_IN_SECONDS,
timePeriod);
+ public synchronized void releasePermitsPeriodically() {
+ ValidationUtils.checkState(!stopped, "Cannot start a stopped rate
limiter");
+ if (scheduler != null) {
+ return;
+ }
+
releasePermitsPeriodically(Executors.newScheduledThreadPool(SCHEDULER_CORE_THREAD_POOL_SIZE,
+ new CustomizedThreadFactory("rate-limiter", true)));
+ }
+
+ private synchronized void
releasePermitsPeriodically(ScheduledExecutorService scheduler) {
+ ValidationUtils.checkState(!stopped, "Cannot start a stopped rate
limiter");
+ if (this.scheduler != null) {
+ return;
+ }
+ this.scheduler = scheduler;
+ try {
+ scheduler.scheduleAtFixedRate(() -> {
+ log.debug("Release permits: maxPermits: {}, available: {}",
maxPermits, semaphore.availablePermits());
+ semaphore.release(maxPermits - semaphore.availablePermits());
+ }, releasePermitsPeriod, releasePermitsPeriod, timePeriod);
+ } catch (RuntimeException e) {
+ this.scheduler = null;
+ scheduler.shutdownNow();
+ throw e;
+ }
Review Comment:
🤖 nit: could you drop the trailing blank line before the closing brace of
this method?
<sub><i>⚠️ AI-generated; verify before applying. React 👍/👎 to flag
quality.</i></sub>
##########
hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/utils/FlinkRateLimiterFactory.java:
##########
@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.sink.utils;
+
+import org.apache.hudi.common.util.RateLimiter;
+import org.apache.hudi.common.util.ValidationUtils;
+import org.apache.hudi.common.util.VisibleForTesting;
+
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * Factory for distributing a job-wide write rate limit across Flink subtasks.
+ */
+public final class FlinkRateLimiterFactory {
+
+ private FlinkRateLimiterFactory() {
+ }
+
+ public static RateLimiter create(long totalRateLimit, int parallelism) {
+ RateLimitConfig rateLimitConfig = resolveRateLimit(totalRateLimit,
parallelism);
+ return RateLimiter.create(
+ rateLimitConfig.getPermits(), rateLimitConfig.getReleasePeriod(),
rateLimitConfig.getTimeUnit());
+ }
+
+ @VisibleForTesting
+ static RateLimiter create(long totalRateLimit, int parallelism,
ScheduledExecutorService scheduler) {
+ RateLimitConfig rateLimitConfig = resolveRateLimit(totalRateLimit,
parallelism);
+ return RateLimiter.create(
+ rateLimitConfig.getPermits(), rateLimitConfig.getReleasePeriod(),
rateLimitConfig.getTimeUnit(), scheduler);
+ }
+
+ static RateLimitConfig resolveRateLimit(long totalRateLimit, int
parallelism) {
+ ValidationUtils.checkArgument(totalRateLimit > 0, "Total rate limit must
be greater than zero");
+ ValidationUtils.checkArgument(parallelism > 0, "Parallelism must be
greater than zero");
+
+ if (totalRateLimit >= parallelism) {
+ long rateLimitPerSubtask = totalRateLimit / parallelism;
+ ValidationUtils.checkArgument(rateLimitPerSubtask <= Integer.MAX_VALUE,
Review Comment:
🤖 non-blocking: The old `(int) totalLimit / parallelism` capped an oversized
`write.rate.limit` at `Integer.MAX_VALUE`, so the job ran effectively
unthrottled; this check now throws in `open()`, and such a job would fail and
restart in a loop after upgrading. Would it make sense to cap the value
instead, e.g. `(int) Math.min(totalRateLimit / parallelism, Integer.MAX_VALUE)`?
<sub><i>⚠️ AI-generated; verify before applying. React 👍/👎 to flag
quality.</i></sub>
##########
hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/utils/FlinkRateLimiterFactory.java:
##########
@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.sink.utils;
+
+import org.apache.hudi.common.util.RateLimiter;
+import org.apache.hudi.common.util.ValidationUtils;
+import org.apache.hudi.common.util.VisibleForTesting;
+
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * Factory for distributing a job-wide write rate limit across Flink subtasks.
+ */
+public final class FlinkRateLimiterFactory {
+
+ private FlinkRateLimiterFactory() {
+ }
+
+ public static RateLimiter create(long totalRateLimit, int parallelism) {
+ RateLimitConfig rateLimitConfig = resolveRateLimit(totalRateLimit,
parallelism);
+ return RateLimiter.create(
+ rateLimitConfig.getPermits(), rateLimitConfig.getReleasePeriod(),
rateLimitConfig.getTimeUnit());
+ }
+
+ @VisibleForTesting
+ static RateLimiter create(long totalRateLimit, int parallelism,
ScheduledExecutorService scheduler) {
+ RateLimitConfig rateLimitConfig = resolveRateLimit(totalRateLimit,
parallelism);
+ return RateLimiter.create(
+ rateLimitConfig.getPermits(), rateLimitConfig.getReleasePeriod(),
rateLimitConfig.getTimeUnit(), scheduler);
+ }
+
+ static RateLimitConfig resolveRateLimit(long totalRateLimit, int
parallelism) {
+ ValidationUtils.checkArgument(totalRateLimit > 0, "Total rate limit must
be greater than zero");
+ ValidationUtils.checkArgument(parallelism > 0, "Parallelism must be
greater than zero");
+
+ if (totalRateLimit >= parallelism) {
+ long rateLimitPerSubtask = totalRateLimit / parallelism;
+ ValidationUtils.checkArgument(rateLimitPerSubtask <= Integer.MAX_VALUE,
+ "Rate limit per subtask exceeds the supported maximum");
+ return new RateLimitConfig((int) rateLimitPerSubtask, 1,
TimeUnit.SECONDS);
+ }
+
+ long periodNanos = divideRoundingUp(
Review Comment:
🤖 nit: would it make sense to add a short comment explaining that when the
limit is below parallelism we use one permit with a longer release period,
since integer division would otherwise give 0 permits?
<sub><i>⚠️ AI-generated; verify before applying. React 👍/👎 to flag
quality.</i></sub>
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
