Logic-32 commented on a change in pull request #2502: Adding storage-throttle module to address "over capacity" issues URL: https://github.com/apache/incubator-zipkin/pull/2502#discussion_r279837011
########## File path: zipkin-server/src/main/java/zipkin2/server/internal/throttle/ThrottledCall.java ########## @@ -0,0 +1,211 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package zipkin2.server.internal.throttle; + +import com.netflix.concurrency.limits.Limiter; +import com.netflix.concurrency.limits.Limiter.Listener; +import java.io.IOException; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Future; +import java.util.concurrent.RejectedExecutionException; +import java.util.function.Supplier; +import zipkin2.Call; +import zipkin2.Callback; +import zipkin2.storage.InMemoryStorage; + +/** + * {@link Call} implementation that is backed by an {@link ExecutorService}. The ExecutorService serves two + * purposes: + * <ol> + * <li>Limits the number of requests that can run in parallel.</li> + * <li>Depending on configuration, can queue up requests to make sure we don't aggressively drop requests that would + * otherwise succeed if given a moment. Bounded queues are safest for this as unbounded ones can lead to heap + * exhaustion and {@link OutOfMemoryError OOM errors}.</li> + * </ol> + * + * @see ThrottledStorageComponent + */ +class ThrottledCall<V> extends Call<V> { + private final ExecutorService executor; + private final Limiter<Void> limiter; + private final Listener limitListener; + /** + * Delegate call needs to be supplied later to avoid having it take action when it is created (like + * {@link InMemoryStorage} and thus avoid being throttled. + */ + private final Supplier<Call<V>> delegate; + private Call<V> call; + private boolean canceled; + + public ThrottledCall(ExecutorService executor, Limiter<Void> limiter, Supplier<Call<V>> delegate) { + this.executor = executor; + this.limiter = limiter; + this.limitListener = limiter.acquire(null).orElseThrow(RejectedExecutionException::new); + this.delegate = delegate; + } + + private ThrottledCall(ThrottledCall other) { + this(other.executor, other.limiter, other.call == null ? other.delegate : () -> other.call.clone()); + } + + @Override + public V execute() throws IOException { + try { + call = delegate.get(); + + // Make sure we throttle + Future<V> future = executor.submit(() -> { + try (AutoCloseable nameReverter = updateThreadName(call.toString())) { + return call.execute(); + } + }); + V result = future.get(); // Still block for the response + + limitListener.onSuccess(); + return result; + } catch (ExecutionException e) { + Throwable cause = e.getCause(); + if (cause instanceof RejectedExecutionException) { + // Storage rejected us, throttle back + limitListener.onDropped(); + } else { + limitListener.onIgnore(); + } + + throw new ThrottleException(cause); + } catch (RuntimeException e) { + limitListener.onIgnore(); + throw e; // E.g. RejectedExecutionException + } catch (Exception e) { + limitListener.onIgnore(); + throw new ThrottleException(e); + } + } + + @Override + public void enqueue(Callback<V> callback) { + try { + executor.execute(() -> { + if (canceled) { + return; + } + + call = delegate.get(); + + // Not using try-with-resources to avoid catching exceptions that occur anywhere outside of close() + AutoCloseable nameReverter = updateThreadName(call.toString()); + try { + ThrottledCallback<V> throttleCallback = new ThrottledCallback<>(callback, limitListener); + call.enqueue(throttleCallback); + + // Need to wait here since the delegate call will run asynchronously also. + // This ensures we don't exceed our throttle/queue limits. + throttleCallback.await(); + } finally { + try { + nameReverter.close(); + } catch (Exception e) { + // swallow + } + } + }); + } catch (Exception e) { + // Ignoring in all cases here because storage itself isn't saying we need to throttle. Though, we may still be + // write bound, but a drop in concurrency won't neccessarily help. + limitListener.onIgnore(); + throw e; // E.g. RejectedExecutionException Review comment: I removed the comment on this line. Line 129 should provide sufficient explanation. ---------------------------------------------------------------- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: [email protected] With regards, Apache Git Services
