[2/4] kudu git commit: java: Expose test name in BaseKuduTest
java: Expose test name in BaseKuduTest Expose the currently-running test method name in BaseKuduTest, since that seems to be the least-worst way to do it in JUnit 4. We also use the method name for the default test table name in TestKuduClient and other tests that included the test name in table names. Change-Id: I63049288e1ab6b32fe629a5c009f2d7d523fd5c0 Reviewed-on: http://gerrit.cloudera.org:8080/10808 Tested-by: Kudu Jenkins Reviewed-by: Grant Henke Project: http://git-wip-us.apache.org/repos/asf/kudu/repo Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/8155d8c4 Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/8155d8c4 Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/8155d8c4 Branch: refs/heads/master Commit: 8155d8c4102d1c36465ab83e680bd1314f80be1f Parents: 90bc812 Author: Mike Percy Authored: Fri Jun 22 20:17:24 2018 -0700 Committer: Mike Percy Committed: Tue Jul 3 02:57:48 2018 + -- .../org/apache/kudu/client/BaseKuduTest.java| 22 ++ .../org/apache/kudu/client/TestKuduClient.java | 4 +- .../org/apache/kudu/client/TestKuduSession.java | 57 ++ .../org/apache/kudu/client/TestKuduTable.java | 79 +--- 4 files changed, 83 insertions(+), 79 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kudu/blob/8155d8c4/java/kudu-client/src/test/java/org/apache/kudu/client/BaseKuduTest.java -- diff --git a/java/kudu-client/src/test/java/org/apache/kudu/client/BaseKuduTest.java b/java/kudu-client/src/test/java/org/apache/kudu/client/BaseKuduTest.java index eed9481..cd28736 100644 --- a/java/kudu-client/src/test/java/org/apache/kudu/client/BaseKuduTest.java +++ b/java/kudu-client/src/test/java/org/apache/kudu/client/BaseKuduTest.java @@ -29,6 +29,8 @@ import com.google.common.net.HostAndPort; import com.stumbleupon.async.Deferred; import org.junit.AfterClass; import org.junit.BeforeClass; +import org.junit.Rule; +import org.junit.rules.TestName; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -63,6 +65,9 @@ public class BaseKuduTest { protected static final Schema basicSchema = getBasicSchema(); protected static final Schema allTypesSchema = getSchemaWithAllTypes(); + @Rule + public TestName testName = new TestName(); + @BeforeClass public static void setUpBeforeClass() throws Exception { LOG.info("Setting up before class..."); @@ -111,6 +116,23 @@ public class BaseKuduTest { syncClient = client.syncClient(); } + /** + * Returns the method name of the currently-running JUnit test. + * @return a test method name + */ + protected String getTestMethodName() { +return testName.getMethodName(); + } + + /** + * Returns the method name of the currently-running JUnit test with a concatenated millisecond + * timestamp. Useful for table names in tests that are automatically retried. + * @return a test method name with a millisecond timestamp appended + */ + protected String getTestMethodNameWithTimestamp() { +return getTestMethodName() + "-" + System.currentTimeMillis(); + } + protected static KuduTable createTable(String tableName, Schema schema, CreateTableOptions builder) throws KuduException { LOG.info("Creating table: {}", tableName); http://git-wip-us.apache.org/repos/asf/kudu/blob/8155d8c4/java/kudu-client/src/test/java/org/apache/kudu/client/TestKuduClient.java -- diff --git a/java/kudu-client/src/test/java/org/apache/kudu/client/TestKuduClient.java b/java/kudu-client/src/test/java/org/apache/kudu/client/TestKuduClient.java index c35902b..3d84cb1 100644 --- a/java/kudu-client/src/test/java/org/apache/kudu/client/TestKuduClient.java +++ b/java/kudu-client/src/test/java/org/apache/kudu/client/TestKuduClient.java @@ -72,8 +72,8 @@ public class TestKuduClient extends BaseKuduTest { private String tableName; @Before - public void setTableName() { -tableName = TestKuduClient.class.getName() + "-" + System.currentTimeMillis(); + public void setUp() { +tableName = getTestMethodNameWithTimestamp(); } /** http://git-wip-us.apache.org/repos/asf/kudu/blob/8155d8c4/java/kudu-client/src/test/java/org/apache/kudu/client/TestKuduSession.java -- diff --git a/java/kudu-client/src/test/java/org/apache/kudu/client/TestKuduSession.java b/java/kudu-client/src/test/java/org/apache/kudu/client/TestKuduSession.java index 74438dc..d1afea7 100644 --- a/java/kudu-client/src/test/java/org/apache/kudu/client/TestKuduSession.java +++ b/java/kudu-client/src/test/java/org/apache/kudu/client/TestKuduSession.java @@ -31,34 +31,34 @@ import static org.junit.Assert.fail;
[4/4] kudu git commit: Add scan request timeout to Kudu scan token API
Add scan request timeout to Kudu scan token API Includes Java and C++ tests. Change-Id: I151cd1da27419361f507e9ad29276c2b50153077 Reviewed-on: http://gerrit.cloudera.org:8080/10653 Reviewed-by: Dan Burkert Tested-by: Mike Percy Project: http://git-wip-us.apache.org/repos/asf/kudu/repo Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/1ae050e4 Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/1ae050e4 Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/1ae050e4 Branch: refs/heads/master Commit: 1ae050e4d57bc333de28bcbc62e072e8bafd04b3 Parents: 0cda8c8 Author: Mike Percy Authored: Thu Jun 7 23:14:20 2018 -0700 Committer: Mike Percy Committed: Tue Jul 3 02:58:35 2018 + -- .../apache/kudu/client/AsyncKuduScanner.java| 8 +++ .../org/apache/kudu/client/KuduScanToken.java | 5 .../org/apache/kudu/client/KuduScanner.java | 8 +++ .../org/apache/kudu/client/TestScanToken.java | 20 .../org/apache/kudu/util/ClientTestUtil.java| 23 ++ src/kudu/client/client.proto| 5 src/kudu/client/scan_token-internal.cc | 5 src/kudu/client/scan_token-test.cc | 25 8 files changed, 99 insertions(+) -- http://git-wip-us.apache.org/repos/asf/kudu/blob/1ae050e4/java/kudu-client/src/main/java/org/apache/kudu/client/AsyncKuduScanner.java -- diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/AsyncKuduScanner.java b/java/kudu-client/src/main/java/org/apache/kudu/client/AsyncKuduScanner.java index 0d13fa6..dd61bf4 100644 --- a/java/kudu-client/src/main/java/org/apache/kudu/client/AsyncKuduScanner.java +++ b/java/kudu-client/src/main/java/org/apache/kudu/client/AsyncKuduScanner.java @@ -379,6 +379,14 @@ public final class AsyncKuduScanner { } /** + * Returns the scan request timeout for this scanner. + * @return the scan request timeout, in milliseconds + */ + public long getScanRequestTimeout() { +return scanRequestTimeout; + } + + /** * Returns the projection schema of this scanner. If specific columns were * not specified during scanner creation, the table schema is returned. * @return the projection schema for this scanner http://git-wip-us.apache.org/repos/asf/kudu/blob/1ae050e4/java/kudu-client/src/main/java/org/apache/kudu/client/KuduScanToken.java -- diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/KuduScanToken.java b/java/kudu-client/src/main/java/org/apache/kudu/client/KuduScanToken.java index 615deed..7e0e2d9 100644 --- a/java/kudu-client/src/main/java/org/apache/kudu/client/KuduScanToken.java +++ b/java/kudu-client/src/main/java/org/apache/kudu/client/KuduScanToken.java @@ -257,6 +257,10 @@ public class KuduScanToken implements Comparable { builder.batchSizeBytes(message.getBatchSizeBytes()); } +if (message.hasScanRequestTimeoutMs()) { + builder.scanRequestTimeout(message.getScanRequestTimeoutMs()); +} + return builder.build(); } @@ -367,6 +371,7 @@ public class KuduScanToken implements Comparable { proto.setCacheBlocks(cacheBlocks); proto.setFaultTolerant(isFaultTolerant); proto.setBatchSizeBytes(batchSizeBytes); + proto.setScanRequestTimeoutMs(scanRequestTimeout); try { PartitionPruner pruner = PartitionPruner.create(this); http://git-wip-us.apache.org/repos/asf/kudu/blob/1ae050e4/java/kudu-client/src/main/java/org/apache/kudu/client/KuduScanner.java -- diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/KuduScanner.java b/java/kudu-client/src/main/java/org/apache/kudu/client/KuduScanner.java index 7437d1e..13602a5 100644 --- a/java/kudu-client/src/main/java/org/apache/kudu/client/KuduScanner.java +++ b/java/kudu-client/src/main/java/org/apache/kudu/client/KuduScanner.java @@ -129,6 +129,14 @@ public class KuduScanner { } /** + * Returns the current value of the scanner's scan request timeout. + * @return the timeout value, in milliseconds + */ + public long getScanRequestTimeout() { +return asyncScanner.getScanRequestTimeout(); + } + + /** * A Builder class to build {@link KuduScanner}. * Use {@link KuduClient#newScannerBuilder} in order to get a builder instance. */ http://git-wip-us.apache.org/repos/asf/kudu/blob/1ae050e4/java/kudu-client/src/test/java/org/apache/kudu/client/TestScanToken.java -- diff --git a/java/kudu-client/src/test/java/org/apache/kudu/client/TestScanToken.java
[1/4] kudu git commit: java: Move schema creation helpers out of KuduClientTest
Repository: kudu Updated Branches: refs/heads/master 95182a6df -> 1ae050e4d java: Move schema creation helpers out of KuduClientTest These helpers are useful in other tests too. Also add missing license header file to ClientTestUtil.java Change-Id: I461a198fb2d30a9a893aee24bbcfe2b493d3350b Reviewed-on: http://gerrit.cloudera.org:8080/10807 Reviewed-by: Dan Burkert Tested-by: Kudu Jenkins Project: http://git-wip-us.apache.org/repos/asf/kudu/repo Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/90bc8127 Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/90bc8127 Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/90bc8127 Branch: refs/heads/master Commit: 90bc81275cd781700934c8e6b4975aec78d0ae76 Parents: 95182a6 Author: Mike Percy Authored: Fri Jun 22 18:42:35 2018 -0700 Committer: Mike Percy Committed: Tue Jul 3 02:53:38 2018 + -- .../org/apache/kudu/client/TestKuduClient.java | 70 .../org/apache/kudu/util/ClientTestUtil.java| 59 + 2 files changed, 73 insertions(+), 56 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kudu/blob/90bc8127/java/kudu-client/src/test/java/org/apache/kudu/client/TestKuduClient.java -- diff --git a/java/kudu-client/src/test/java/org/apache/kudu/client/TestKuduClient.java b/java/kudu-client/src/test/java/org/apache/kudu/client/TestKuduClient.java index 01269d3..c35902b 100644 --- a/java/kudu-client/src/test/java/org/apache/kudu/client/TestKuduClient.java +++ b/java/kudu-client/src/test/java/org/apache/kudu/client/TestKuduClient.java @@ -23,6 +23,10 @@ import static org.apache.kudu.client.KuduPredicate.ComparisonOp.LESS; import static org.apache.kudu.client.KuduPredicate.ComparisonOp.LESS_EQUAL; import static org.apache.kudu.util.ClientTestUtil.countRowsInScan; import static org.apache.kudu.util.ClientTestUtil.createBasicSchemaInsert; +import static org.apache.kudu.util.ClientTestUtil.createManyStringsSchema; +import static org.apache.kudu.util.ClientTestUtil.createSchemaWithBinaryColumns; +import static org.apache.kudu.util.ClientTestUtil.createSchemaWithDecimalColumns; +import static org.apache.kudu.util.ClientTestUtil.createSchemaWithTimestampColumns; import static org.apache.kudu.util.ClientTestUtil.getBasicCreateTableOptions; import static org.apache.kudu.util.ClientTestUtil.getBasicTableOptionsWithNonCoveredRange; import static org.apache.kudu.util.ClientTestUtil.scanTableToStrings; @@ -58,7 +62,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.kudu.ColumnSchema; -import org.apache.kudu.ColumnTypeAttributes.ColumnTypeAttributesBuilder; import org.apache.kudu.Schema; import org.apache.kudu.Type; import org.apache.kudu.util.CapturingLogAppender; @@ -73,51 +76,6 @@ public class TestKuduClient extends BaseKuduTest { tableName = TestKuduClient.class.getName() + "-" + System.currentTimeMillis(); } - private Schema createManyStringsSchema() { -ArrayList columns = new ArrayList(4); -columns.add(new ColumnSchema.ColumnSchemaBuilder("key", Type.STRING).key(true).build()); -columns.add(new ColumnSchema.ColumnSchemaBuilder("c1", Type.STRING).build()); -columns.add(new ColumnSchema.ColumnSchemaBuilder("c2", Type.STRING).build()); -columns.add(new ColumnSchema.ColumnSchemaBuilder("c3", Type.STRING).nullable(true).build()); -columns.add(new ColumnSchema.ColumnSchemaBuilder("c4", Type.STRING).nullable(true).build()); -return new Schema(columns); - } - - private Schema createSchemaWithBinaryColumns() { -ArrayList columns = new ArrayList(); -columns.add(new ColumnSchema.ColumnSchemaBuilder("key", Type.BINARY).key(true).build()); -columns.add(new ColumnSchema.ColumnSchemaBuilder("c1", Type.STRING).build()); -columns.add(new ColumnSchema.ColumnSchemaBuilder("c2", Type.DOUBLE).build()); -columns.add(new ColumnSchema.ColumnSchemaBuilder("c3", Type.BINARY).nullable(true).build()); -return new Schema(columns); - } - - private Schema createSchemaWithTimestampColumns() { -ArrayList columns = new ArrayList(); -columns.add(new ColumnSchema.ColumnSchemaBuilder("key", Type.UNIXTIME_MICROS).key(true).build()); -columns.add(new ColumnSchema.ColumnSchemaBuilder("c1", Type.UNIXTIME_MICROS).nullable(true).build()); -return new Schema(columns); - } - - private Schema createSchemaWithDecimalColumns() { -ArrayList columns = new ArrayList(); -columns.add(new ColumnSchema.ColumnSchemaBuilder("key", Type.DECIMAL).key(true) -.typeAttributes( -new ColumnTypeAttributesBuilder() - .precision(org.apache.kudu.util.DecimalUtil.MAX_DECIMAL64_PRECISION).build() -).build()); -columns.add(new ColumnSchema.ColumnSchemaBuilder("c1",
[3/4] kudu git commit: java: Move scan token tests into their own class
java: Move scan token tests into their own class A follow-up commit will add additional tests there. Change-Id: Iefc23f0193cb24a5a5554881eedbaac15929 Reviewed-on: http://gerrit.cloudera.org:8080/10724 Tested-by: Kudu Jenkins Reviewed-by: Dan Burkert Project: http://git-wip-us.apache.org/repos/asf/kudu/repo Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/0cda8c8e Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/0cda8c8e Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/0cda8c8e Branch: refs/heads/master Commit: 0cda8c8ee9dbd982763c3706a6defde1fffc6d00 Parents: 8155d8c Author: Mike Percy Authored: Fri Jun 22 19:44:14 2018 -0700 Committer: Mike Percy Committed: Tue Jul 3 02:58:04 2018 + -- .../org/apache/kudu/client/TestKuduClient.java | 228 -- .../org/apache/kudu/client/TestScanToken.java | 235 +++ .../org/apache/kudu/util/ClientTestUtil.java| 45 3 files changed, 280 insertions(+), 228 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kudu/blob/0cda8c8e/java/kudu-client/src/test/java/org/apache/kudu/client/TestKuduClient.java -- diff --git a/java/kudu-client/src/test/java/org/apache/kudu/client/TestKuduClient.java b/java/kudu-client/src/test/java/org/apache/kudu/client/TestKuduClient.java index 3d84cb1..ce88fd7 100644 --- a/java/kudu-client/src/test/java/org/apache/kudu/client/TestKuduClient.java +++ b/java/kudu-client/src/test/java/org/apache/kudu/client/TestKuduClient.java @@ -49,10 +49,8 @@ import java.util.concurrent.Callable; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; -import java.util.concurrent.atomic.AtomicInteger; import com.google.common.collect.ImmutableList; -import com.google.common.collect.Iterators; import com.stumbleupon.async.Deferred; import org.apache.kudu.util.TimestampUtil; @@ -586,238 +584,12 @@ public class TestKuduClient extends BaseKuduTest { ).size()); } - /** - * Counts the rows in the provided scan tokens. - */ - private int countScanTokenRows(List tokens) throws Exception { -final AtomicInteger count = new AtomicInteger(0); -List threads = new ArrayList<>(); -for (final KuduScanToken token : tokens) { - final byte[] serializedToken = token.serialize(); - Thread thread = new Thread(new Runnable() { -@Override -public void run() { - try (KuduClient contextClient = new KuduClient.KuduClientBuilder(masterAddresses) - .defaultAdminOperationTimeoutMs(DEFAULT_SLEEP) - .build()) { -KuduScanner scanner = KuduScanToken.deserializeIntoScanner(serializedToken, contextClient); -try { - int localCount = 0; - while (scanner.hasMoreRows()) { -localCount += Iterators.size(scanner.nextRows()); - } - count.addAndGet(localCount); -} finally { - scanner.close(); -} - } catch (Exception e) { -LOG.error("exception in parallel token scanner", e); - } -} - }); - thread.run(); - threads.add(thread); -} - -for (Thread thread : threads) { - thread.join(); -} -return count.get(); - } - @Test public void testGetAuthnToken() throws Exception { byte[] token = client.exportAuthenticationCredentials().join(); assertNotNull(token); } - /** - * Tests scan tokens by creating a set of scan tokens, serializing them, and - * then executing them in parallel with separate client instances. This - * simulates the normal usecase of scan tokens being created at a central - * planner and distributed to remote task executors. - */ - @Test - public void testScanTokens() throws Exception { -int saveFetchTablets = AsyncKuduClient.FETCH_TABLETS_PER_RANGE_LOOKUP; -try { - // For this test, make sure that we cover the case that not all tablets - // are returned in a single batch. - AsyncKuduClient.FETCH_TABLETS_PER_RANGE_LOOKUP = 4; - - Schema schema = createManyStringsSchema(); - CreateTableOptions createOptions = new CreateTableOptions(); - createOptions.addHashPartitions(ImmutableList.of("key"), 8); - - PartialRow splitRow = schema.newPartialRow(); - splitRow.addString("key", "key_50"); - createOptions.addSplitRow(splitRow); - - syncClient.createTable(tableName, schema, createOptions); - - KuduSession session = syncClient.newSession(); - session.setFlushMode(SessionConfiguration.FlushMode.AUTO_FLUSH_BACKGROUND); - KuduTable table = syncClient.openTable(tableName); - for (int i = 0; i < 100; i++) { -Insert insert = table.newInsert(); -
kudu git commit: KUDU-2191: Adjust upgrade tool to tolerate downgraded metadata
Repository: kudu Updated Branches: refs/heads/master f5c802ced -> 95182a6df KUDU-2191: Adjust upgrade tool to tolerate downgraded metadata Currently, TestHmsDowngrade keeps on failing as the metadata upgrade tool does not handle tables being downgraded properly. This issue didn't surface because commit 214dbc249 didn't rebase. This commit corrects the upgrade tool to better handle this scenario. I also looped this test for 500 times, no failures found for TestHmsDowngrade. Dist-test: http://dist-test.cloudera.org/job?job_id=hao.hao.1530566672.124129 Change-Id: Ie9a3b154e77db4eda72b9c4b53861f9fe06f09a3 Reviewed-on: http://gerrit.cloudera.org:8080/10844 Reviewed-by: Andrew Wong Tested-by: Kudu Jenkins Project: http://git-wip-us.apache.org/repos/asf/kudu/repo Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/95182a6d Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/95182a6d Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/95182a6d Branch: refs/heads/master Commit: 95182a6dff9da1f268d60aed705a267ed2946df6 Parents: f5c802c Author: Hao Hao Authored: Fri Jun 29 14:12:31 2018 -0700 Committer: Hao Hao Committed: Mon Jul 2 22:34:30 2018 + -- src/kudu/tools/tool_action_hms.cc | 29 +++-- 1 file changed, 15 insertions(+), 14 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kudu/blob/95182a6d/src/kudu/tools/tool_action_hms.cc -- diff --git a/src/kudu/tools/tool_action_hms.cc b/src/kudu/tools/tool_action_hms.cc index d0e3c4b..f06367b 100644 --- a/src/kudu/tools/tool_action_hms.cc +++ b/src/kudu/tools/tool_action_hms.cc @@ -158,20 +158,21 @@ Status AlterLegacyKuduTables(KuduClient* kudu_client, if (hms_table->parameters[HmsClient::kStorageHandlerKey] == HmsClient::kLegacyKuduStorageHandler) { string new_table_name = Substitute("$0.$1", hms_table->dbName, hms_table->tableName); -bool exist; -RETURN_NOT_OK(kudu_client->TableExists(new_table_name, )); -if (!exist) { - // TODO(Hao): Use notification listener to avoid race conditions. - s = AlterKuduTableOnly(kudu_client, table_name, new_table_name).AndThen([&] { -return hms_catalog->UpgradeLegacyImpalaTable(kudu_table->id(), -hms_table->dbName, hms_table->tableName, -client::SchemaFromKuduSchema(kudu_table->schema())); - }); -} else { - LOG(WARNING) << Substitute("Failed to upgrade legacy Impala table '$0.$1' " - "(Kudu table name: $2), because a Kudu table with " - "name '$0.$1' already exists'.", hms_table->dbName, - hms_table->tableName, table_name); +// Hive-compatible table name implies the table has been upgraded previously and +// then downgraded. In this case, we only upgrade the legacy Impala table. +if (table_name != new_table_name) { + s = AlterKuduTableOnly(kudu_client, table_name, new_table_name); + if (s.IsAlreadyPresent()) { +s = s.CloneAndPrepend(Substitute("Failed to upgrade legacy Impala table '$0.$1' " + "(Kudu table name: $2), because a Kudu table with " + "name '$0.$1' already exists'.", hms_table->dbName, + hms_table->tableName, table_name)); + } +} +if (s.ok()) { + s = hms_catalog->UpgradeLegacyImpalaTable( + kudu_table->id(), hms_table->dbName, hms_table->tableName, + client::SchemaFromKuduSchema(kudu_table->schema())); } } } else {