bbeaudreault commented on a change in pull request #3271: URL: https://github.com/apache/hadoop/pull/3271#discussion_r695678157
########## File path: hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/client/impl/TestFetchBlockLocationsRetryer.java ########## @@ -0,0 +1,107 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.client.impl; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys; +import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.Retry; +import org.apache.hadoop.hdfs.client.impl.DfsClientConf.FetchBlockLocationsRetryer; +import org.junit.Test; + +public class TestFetchBlockLocationsRetryer { + + private static final double EPSILON = 0.001; + + @Test + public void testIsMaxFailuresExceeded() { + Configuration conf = new Configuration(); + + conf.setInt(HdfsClientConfigKeys.DFS_CLIENT_MAX_BLOCK_ACQUIRE_FAILURES_KEY, 3); + FetchBlockLocationsRetryer retryer = new FetchBlockLocationsRetryer(conf); + + assertFalse(retryer.isMaxFailuresExceeded(1)); + assertTrue(retryer.isMaxFailuresExceeded(3)); + assertTrue(retryer.isMaxFailuresExceeded(5)); + } + + @Test + public void testDefaultRetryPolicy() { Review comment: Per the comment in the original backoff policy: > // Introducing a random factor to the wait time before another retry. > // The wait time is dependent on # of failures and a random factor. > // At the first time of getting a BlockMissingException, the wait time > // is a random number between 0..3000 ms. If the first retry > // still fails, we will wait 3000 ms grace period before the 2nd retry. > // Also at the second retry, the waiting window is expanded to 6000 ms > // alleviating the request rate from the server. Similarly the 3rd retry > // will wait 6000ms grace period before retry and the waiting window is > // expanded to 9000ms. - The first backoff should be between 0-3000ms. - The second should be 3000 plus a random number between 0-6000ms. So the full range is 3000-9000. - The third retry should be 6000 plus a random number between 0-9000ms. So the full range is 6000-15000ms. This test proves that this original retry strategy continues to work with the new code. It's hard to test with randomness, so the random factor is disabled. We're left with only the worst case scenario (if `rand()` returned 1). See the assertions below to see that the results adhere to the original description above. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: common-issues-unsubscr...@hadoop.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org --------------------------------------------------------------------- To unsubscribe, e-mail: common-issues-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-issues-h...@hadoop.apache.org
