Repository: kudu Updated Branches: refs/heads/master d1f53cc32 -> 4a0417018
KUDU-2033 (part 1). Add write and stop/start in a loop to TestLeaderFailover TestLeaderFailover is made more robust by adding a write loop in which the tablet leader is killed and restarted. The main idea is to check that the writes complete and can be read after the leader is killed. Done in a loop so we can be sure that we stay stable through multiple stops and restarts of the leader tablet. Change-Id: Ie93a15084a4c4c0748dc74c005b8313f443a5ba9 Reviewed-on: http://gerrit.cloudera.org:8080/7456 Reviewed-by: Mike Percy <mpe...@apache.org> Tested-by: Mike Percy <mpe...@apache.org> Project: http://git-wip-us.apache.org/repos/asf/kudu/repo Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/4a041701 Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/4a041701 Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/4a041701 Branch: refs/heads/master Commit: 4a0417018b2833ee9f308a294b319d8c7570990f Parents: d1f53cc Author: efancher <e...@cloudera.com> Authored: Tue Jul 18 14:06:16 2017 -0700 Committer: Mike Percy <mpe...@apache.org> Committed: Tue Aug 8 18:34:11 2017 +0000 ---------------------------------------------------------------------- .../org/apache/kudu/client/BaseKuduTest.java | 2 +- .../kudu/client/TestMultipleLeaderFailover.java | 95 ++++++++++++++++++++ .../org/apache/kudu/util/AssertHelpers.java | 41 +++++++++ 3 files changed, 137 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kudu/blob/4a041701/java/kudu-client/src/test/java/org/apache/kudu/client/BaseKuduTest.java ---------------------------------------------------------------------- diff --git a/java/kudu-client/src/test/java/org/apache/kudu/client/BaseKuduTest.java b/java/kudu-client/src/test/java/org/apache/kudu/client/BaseKuduTest.java index 1caa21c..a0a5832 100644 --- a/java/kudu-client/src/test/java/org/apache/kudu/client/BaseKuduTest.java +++ b/java/kudu-client/src/test/java/org/apache/kudu/client/BaseKuduTest.java @@ -382,7 +382,7 @@ public class BaseKuduTest { * @return the RPC port of the given tablet's leader tablet server. * @throws Exception */ - private static int findLeaderTabletServerPort(LocatedTablet tablet) + protected static int findLeaderTabletServerPort(LocatedTablet tablet) throws Exception { LocatedTablet.Replica leader = null; DeadlineTracker deadlineTracker = new DeadlineTracker(); http://git-wip-us.apache.org/repos/asf/kudu/blob/4a041701/java/kudu-client/src/test/java/org/apache/kudu/client/TestMultipleLeaderFailover.java ---------------------------------------------------------------------- diff --git a/java/kudu-client/src/test/java/org/apache/kudu/client/TestMultipleLeaderFailover.java b/java/kudu-client/src/test/java/org/apache/kudu/client/TestMultipleLeaderFailover.java new file mode 100644 index 0000000..526fdb3 --- /dev/null +++ b/java/kudu-client/src/test/java/org/apache/kudu/client/TestMultipleLeaderFailover.java @@ -0,0 +1,95 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.kudu.client; + +import static org.apache.kudu.util.AssertHelpers.assertEventuallyTrue; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +import java.util.List; +import org.apache.kudu.util.AssertHelpers.BooleanExpression; +import org.junit.BeforeClass; +import org.junit.Test; + +public class TestMultipleLeaderFailover extends BaseKuduTest { + + @BeforeClass + public static void setUpBeforeClass() throws Exception { + BaseKuduTest.setUpBeforeClass(); + } + + private void waitUntilRowCount(final KuduTable table, final int rowCount, long timeoutMs) + throws Exception { + assertEventuallyTrue(String.format("Read count should be %s", rowCount), + new BooleanExpression() { + @Override + public boolean get() throws Exception { + AsyncKuduScanner scanner = client.newScannerBuilder(table).build(); + int read_count = countRowsInScan(scanner); + return read_count == rowCount; + } + }, timeoutMs); + } + + /** + * This test writes 3 rows. Then in a loop, it kills the leader, then tries to write inner_row + * rows, and finally restarts the tablet server it killed. Verifying with a read as it goes. + * Finally it counts to make sure we have total_rows_to_insert of them. + */ + @Test(timeout = 100000) + public void testMultipleFailover() throws Exception { + KuduTable table; + CreateTableOptions builder = getBasicCreateTableOptions(); + String tableName = + TestMultipleLeaderFailover.class.getName() + "-" + System.currentTimeMillis(); + createTable(tableName, basicSchema, builder); + + table = openTable(tableName); + KuduSession session = syncClient.newSession(); + final int ROWS_PER_ITERATION = 3; + final int NUM_ITERATIONS = 10; + final int TOTAL_ROWS_TO_INSERT = ROWS_PER_ITERATION + NUM_ITERATIONS * ROWS_PER_ITERATION; + + for (int i = 0; i < ROWS_PER_ITERATION; i++) { + session.apply(createBasicSchemaInsert(table, i)); + } + + waitUntilRowCount(table, ROWS_PER_ITERATION, DEFAULT_SLEEP); + + int currentRows = ROWS_PER_ITERATION; + for (int i = 0; i < NUM_ITERATIONS; i++) { + List<LocatedTablet> tablets = table.getTabletsLocations(DEFAULT_SLEEP); + assertEquals(1, tablets.size()); + int leaderPort = findLeaderTabletServerPort(tablets.get(0)); + miniCluster.killTabletServerOnPort(leaderPort); + + for (int j = 0; j < ROWS_PER_ITERATION; j++) { + OperationResponse resp = session.apply(createBasicSchemaInsert(table, currentRows)); + if (resp.hasRowError()) { + fail("Encountered a row error " + resp.getRowError()); + } + currentRows++; + } + + miniCluster.restartDeadTabletServerOnPort(leaderPort); + // Read your writes hasn't been enabled, so we need to use a helper function to poll. + waitUntilRowCount(table, currentRows, DEFAULT_SLEEP); + + } + waitUntilRowCount(table, TOTAL_ROWS_TO_INSERT, DEFAULT_SLEEP); + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/kudu/blob/4a041701/java/kudu-client/src/test/java/org/apache/kudu/util/AssertHelpers.java ---------------------------------------------------------------------- diff --git a/java/kudu-client/src/test/java/org/apache/kudu/util/AssertHelpers.java b/java/kudu-client/src/test/java/org/apache/kudu/util/AssertHelpers.java new file mode 100644 index 0000000..ff7d67e --- /dev/null +++ b/java/kudu-client/src/test/java/org/apache/kudu/util/AssertHelpers.java @@ -0,0 +1,41 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.kudu.util; + +import static org.junit.Assert.assertTrue; + +public class AssertHelpers { + public interface BooleanExpression { + boolean get() throws Exception; + } + + // A looping check. It's mainly useful for scanners, since writes may take a little time to show + // up. + public static void assertEventuallyTrue(String description, BooleanExpression expression, + long timeoutMillis) throws Exception { + long deadlineNanos = System.nanoTime() + timeoutMillis * 1000000; + boolean success; + + do { + success = expression.get(); + if (success) break; + Thread.sleep(50); // Sleep for 50ms + } while (System.nanoTime() < deadlineNanos); + + assertTrue(description, success); + } +}
