This is an automated email from the ASF dual-hosted git repository. abukor pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/kudu.git
commit 59813014099a3c433734c7c58268ddfb80d69055 Author: Attila Bukor <[email protected]> AuthorDate: Tue Jun 30 16:36:56 2020 +0200 KUDU-3090 Ownership support in Java client Change-Id: I083ad9750ce1b3ae31bb510b700d1204fcdf291d Reviewed-on: http://gerrit.cloudera.org:8080/16125 Tested-by: Kudu Jenkins Reviewed-by: Grant Henke <[email protected]> --- java/kudu-client/build.gradle | 5 -- .../org/apache/kudu/client/AlterTableOptions.java | 10 +++ .../org/apache/kudu/client/AsyncKuduClient.java | 5 +- .../org/apache/kudu/client/CreateTableOptions.java | 11 +--- .../apache/kudu/client/GetTableSchemaRequest.java | 3 +- .../apache/kudu/client/GetTableSchemaResponse.java | 14 ++++- .../java/org/apache/kudu/client/KuduScanToken.java | 4 +- .../java/org/apache/kudu/client/KuduTable.java | 14 ++++- .../org/apache/kudu/client/TestAlterTable.java | 26 ++++++++ .../apache/kudu/client/TestAsyncKuduClient.java | 2 +- .../kudu/client/TestHiveMetastoreIntegration.java | 72 ---------------------- .../org/apache/kudu/client/TestKeyEncoding.java | 6 +- 12 files changed, 77 insertions(+), 95 deletions(-) diff --git a/java/kudu-client/build.gradle b/java/kudu-client/build.gradle index f681ac2..759dee0 100644 --- a/java/kudu-client/build.gradle +++ b/java/kudu-client/build.gradle @@ -38,11 +38,6 @@ dependencies { optional libs.yetusAnnotations testCompile project(":kudu-test-utils") - testCompile libs.hiveMetastore - // The HMS client relies on the MR client-core artifact for JobConf, but only - // specifies it as an optional dependency. Gradle doesn't pull in optional - // dependencies, so we have to depend on it directly. - testCompile libs.hadoopMRClientCore testCompile libs.junit testCompile libs.log4j testCompile libs.log4jSlf4jImpl diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/AlterTableOptions.java b/java/kudu-client/src/main/java/org/apache/kudu/client/AlterTableOptions.java index 8071257..20aa80f 100644 --- a/java/kudu-client/src/main/java/org/apache/kudu/client/AlterTableOptions.java +++ b/java/kudu-client/src/main/java/org/apache/kudu/client/AlterTableOptions.java @@ -54,6 +54,16 @@ public class AlterTableOptions { } /** + * Change a table's owner. + * @param owner the new table owner + * @return this instance + */ + public AlterTableOptions setOwner(String owner) { + pb.setNewTableOwner(owner); + return this; + } + + /** * Add a new column. * @param colSchema the schema of the new column * @return this instance diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/AsyncKuduClient.java b/java/kudu-client/src/main/java/org/apache/kudu/client/AsyncKuduClient.java index 112c9c2..9260189 100644 --- a/java/kudu-client/src/main/java/org/apache/kudu/client/AsyncKuduClient.java +++ b/java/kudu-client/src/main/java/org/apache/kudu/client/AsyncKuduClient.java @@ -374,7 +374,7 @@ public class AsyncKuduClient implements AutoCloseable { this.bootstrap = b.createBootstrap(); this.masterAddresses = b.masterAddresses; this.masterTable = new KuduTable(this, MASTER_TABLE_NAME_PLACEHOLDER, - MASTER_TABLE_NAME_PLACEHOLDER, null, null, 1, null); + MASTER_TABLE_NAME_PLACEHOLDER, null, null, 1, null, null); this.defaultOperationTimeoutMs = b.defaultOperationTimeoutMs; this.defaultAdminOperationTimeoutMs = b.defaultAdminOperationTimeoutMs; this.statisticsDisabled = b.statisticsDisabled; @@ -792,7 +792,8 @@ public class AsyncKuduClient implements AutoCloseable { resp.getSchema(), resp.getPartitionSchema(), resp.getNumReplicas(), - resp.getExtraConfig()); + resp.getExtraConfig(), + resp.getOwner()); }); } diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/CreateTableOptions.java b/java/kudu-client/src/main/java/org/apache/kudu/client/CreateTableOptions.java index 630d73e..94dff93 100644 --- a/java/kudu-client/src/main/java/org/apache/kudu/client/CreateTableOptions.java +++ b/java/kudu-client/src/main/java/org/apache/kudu/client/CreateTableOptions.java @@ -245,22 +245,17 @@ public class CreateTableOptions { } /** - * Set the table owner as the provided username in configured external catalogs - * such as the Hive Metastore. Overrides the default of the currently logged-in - * username or Kerberos principal. + * Set the table owner as the provided username. + * Overrides the default of the currently logged-in username or Kerberos principal. * * This is an unstable method because it is not yet clear whether this should * be supported directly in the long run, rather than requiring the table creator * to re-assign ownership explicitly. * - * @param owner the username to set as the table owner in external catalogs + * @param owner the username to set as the table owner. * @return this instance */ - @InterfaceAudience.LimitedPrivate("Impala") - @InterfaceStability.Unstable public CreateTableOptions setOwner(String owner) { - Preconditions.checkArgument(!Strings.isNullOrEmpty(owner), - "table owner must not be null or empty"); pb.setOwner(owner); return this; } diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/GetTableSchemaRequest.java b/java/kudu-client/src/main/java/org/apache/kudu/client/GetTableSchemaRequest.java index 4aac027..433ac4f 100644 --- a/java/kudu-client/src/main/java/org/apache/kudu/client/GetTableSchemaRequest.java +++ b/java/kudu-client/src/main/java/org/apache/kudu/client/GetTableSchemaRequest.java @@ -100,7 +100,8 @@ public class GetTableSchemaRequest extends KuduRpc<GetTableSchemaResponse> { respBuilder.getNumReplicas(), ProtobufHelper.pbToPartitionSchema(respBuilder.getPartitionSchema(), schema), respBuilder.hasAuthzToken() ? respBuilder.getAuthzToken() : null, - respBuilder.getExtraConfigsMap()); + respBuilder.getExtraConfigsMap(), + respBuilder.hasOwner() ? respBuilder.getOwner() : ""); return new Pair<GetTableSchemaResponse, Object>( response, respBuilder.hasError() ? respBuilder.getError() : null); } diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/GetTableSchemaResponse.java b/java/kudu-client/src/main/java/org/apache/kudu/client/GetTableSchemaResponse.java index 22a1a2b..26b8723 100644 --- a/java/kudu-client/src/main/java/org/apache/kudu/client/GetTableSchemaResponse.java +++ b/java/kudu-client/src/main/java/org/apache/kudu/client/GetTableSchemaResponse.java @@ -34,6 +34,7 @@ public class GetTableSchemaResponse extends KuduRpcResponse { private final int numReplicas; private final SignedTokenPB authzToken; private final Map<String, String> extraConfig; + private final String owner; /** * @param elapsedMillis Time in milliseconds since RPC creation to now @@ -45,6 +46,7 @@ public class GetTableSchemaResponse extends KuduRpcResponse { * @param partitionSchema the table's partition schema * @param authzToken an authorization token for use with this table * @param extraConfig the table's extra configuration properties + * @param owner the table's owner */ GetTableSchemaResponse(long elapsedMillis, String tsUUID, @@ -54,7 +56,8 @@ public class GetTableSchemaResponse extends KuduRpcResponse { int numReplicas, PartitionSchema partitionSchema, SignedTokenPB authzToken, - Map<String, String> extraConfig) { + Map<String, String> extraConfig, + String owner) { super(elapsedMillis, tsUUID); this.schema = schema; this.partitionSchema = partitionSchema; @@ -63,6 +66,7 @@ public class GetTableSchemaResponse extends KuduRpcResponse { this.numReplicas = numReplicas; this.authzToken = authzToken; this.extraConfig = extraConfig; + this.owner = owner; } /** @@ -120,4 +124,12 @@ public class GetTableSchemaResponse extends KuduRpcResponse { public Map<String, String> getExtraConfig() { return extraConfig; } + + /** + * Get the owner for the table. + * @return the table's owner + */ + public String getOwner() { + return owner; + } } diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/KuduScanToken.java b/java/kudu-client/src/main/java/org/apache/kudu/client/KuduScanToken.java index 62e7bfe..bdf8fd1 100644 --- a/java/kudu-client/src/main/java/org/apache/kudu/client/KuduScanToken.java +++ b/java/kudu-client/src/main/java/org/apache/kudu/client/KuduScanToken.java @@ -354,7 +354,8 @@ public class KuduScanToken implements Comparable<KuduScanToken> { ProtobufHelper.pbToPartitionSchema(tableMetadata.getPartitionSchema(), schema); return new KuduTable(client.asyncClient, tableMetadata.getTableName(), tableMetadata.getTableId(), schema, partitionSchema, - tableMetadata.getNumReplicas(), tableMetadata.getExtraConfigsMap()); + tableMetadata.getNumReplicas(), tableMetadata.getExtraConfigsMap(), + tableMetadata.getOwner()); } else if (message.hasTableId()) { return client.openTableById(message.getTableId()); } else { @@ -482,6 +483,7 @@ public class KuduScanToken implements Comparable<KuduScanToken> { Client.TableMetadataPB tableMetadataPB = Client.TableMetadataPB.newBuilder() .setTableId(table.getTableId()) .setTableName(table.getName()) + .setOwner(table.getOwner()) .setNumReplicas(table.getNumReplicas()) .setSchema(ProtobufHelper.schemaToPb(table.getSchema())) .setPartitionSchema(ProtobufHelper.partitionSchemaToPb(table.getPartitionSchema())) diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/KuduTable.java b/java/kudu-client/src/main/java/org/apache/kudu/client/KuduTable.java index 5d76bf5..2488015 100644 --- a/java/kudu-client/src/main/java/org/apache/kudu/client/KuduTable.java +++ b/java/kudu-client/src/main/java/org/apache/kudu/client/KuduTable.java @@ -50,6 +50,7 @@ public class KuduTable { private final String tableId; private final int numReplicas; private final Map<String, String> extraConfig; + private final String owner; /** * Package-private constructor, use {@link KuduClient#openTable(String)} to get an instance. @@ -60,10 +61,11 @@ public class KuduTable { * @param partitionSchema this table's partition schema * @param numReplicas this table's replication factor * @param extraConfig this table's extra configuration properties + * @param owner this table's owner */ KuduTable(AsyncKuduClient client, String name, String tableId, Schema schema, PartitionSchema partitionSchema, int numReplicas, - Map<String, String> extraConfig) { + Map<String, String> extraConfig, String owner) { this.schema = schema; this.partitionSchema = partitionSchema; this.client = client; @@ -71,6 +73,7 @@ public class KuduTable { this.tableId = tableId; this.numReplicas = numReplicas; this.extraConfig = extraConfig; + this.owner = owner; } /** @@ -127,6 +130,15 @@ public class KuduTable { } /** + * Get this table's owner. + * @return this table's owner or an empty string if the table was created without owner on a + * version of Kudu that didn't automatically assign an owner. + */ + public String getOwner() { + return owner; + } + + /** * Get the async client that created this instance. * @return an async kudu client */ diff --git a/java/kudu-client/src/test/java/org/apache/kudu/client/TestAlterTable.java b/java/kudu-client/src/test/java/org/apache/kudu/client/TestAlterTable.java index 715ced4..6f05a25 100644 --- a/java/kudu-client/src/test/java/org/apache/kudu/client/TestAlterTable.java +++ b/java/kudu-client/src/test/java/org/apache/kudu/client/TestAlterTable.java @@ -68,6 +68,16 @@ public class TestAlterTable { * with the provided bounds. */ private KuduTable createTable(List<Pair<Integer, Integer>> bounds) throws KuduException { + return createTable(bounds, null); + } + + /** + * Creates a new table with two int columns, c0 and c1. c0 is the primary key. + * The table is hash partitioned on c0 into two buckets, and range partitioned + * with the provided bounds and the specified owner. + */ + private KuduTable createTable(List<Pair<Integer, Integer>> bounds, String owner) + throws KuduException { // Create initial table with single range partition covering the entire key // space, and two hash buckets. ArrayList<ColumnSchema> columns = new ArrayList<>(1); @@ -93,6 +103,10 @@ public class TestAlterTable { createOptions.addRangePartition(lower, upper); } + if (owner != null) { + createOptions.setOwner(owner); + } + return client.createTable(tableName, schema, createOptions); } @@ -564,4 +578,16 @@ public class TestAlterTable { Assert.assertTrue(thrown.getMessage() .contains("number of columns 11 is greater than the permitted maximum 10")); } + + @Test + public void testAlterChangeOwner() throws Exception { + String originalOwner = "alice"; + KuduTable table = createTable(ImmutableList.of(), originalOwner); + assertEquals(originalOwner, table.getOwner()); + + String newOwner = "bob"; + client.alterTable(table.getName(), new AlterTableOptions().setOwner(newOwner)); + table = client.openTable(table.getName()); + assertEquals(newOwner, table.getOwner()); + } } diff --git a/java/kudu-client/src/test/java/org/apache/kudu/client/TestAsyncKuduClient.java b/java/kudu-client/src/test/java/org/apache/kudu/client/TestAsyncKuduClient.java index a7fdb9b..b1eb798 100644 --- a/java/kudu-client/src/test/java/org/apache/kudu/client/TestAsyncKuduClient.java +++ b/java/kudu-client/src/test/java/org/apache/kudu/client/TestAsyncKuduClient.java @@ -176,7 +176,7 @@ public class TestAsyncKuduClient { // Test that a tablet full of unreachable replicas won't make us retry. try { KuduTable badTable = new KuduTable(asyncClient, "Invalid table name", - "Invalid table ID", null, null, 3, null); + "Invalid table ID", null, null, 3, null, null); asyncClient.discoverTablets(badTable, null, requestBatchSize, tabletLocations, tsInfos, 1000); fail("This should have failed quickly"); diff --git a/java/kudu-client/src/test/java/org/apache/kudu/client/TestHiveMetastoreIntegration.java b/java/kudu-client/src/test/java/org/apache/kudu/client/TestHiveMetastoreIntegration.java deleted file mode 100644 index daf2396..0000000 --- a/java/kudu-client/src/test/java/org/apache/kudu/client/TestHiveMetastoreIntegration.java +++ /dev/null @@ -1,72 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.kudu.client; - -import static org.junit.Assert.assertEquals; - -import org.apache.hadoop.hive.conf.HiveConf; -import org.apache.hadoop.hive.metastore.HiveMetaStoreClient; -import org.apache.hadoop.hive.metastore.IMetaStoreClient; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; - -import org.apache.kudu.test.ClientTestUtil; -import org.apache.kudu.test.KuduTestHarness; - -public class TestHiveMetastoreIntegration { - - @Rule - public KuduTestHarness harness = new KuduTestHarness( - KuduTestHarness.getBaseClusterBuilder().enableHiveMetastoreIntegration()); - - private KuduClient client; - - @Before - public void setUp() { - client = harness.getClient(); - } - - @Test(timeout = 100000) - public void testOverrideTableOwner() throws Exception { - // Create a table with an overridden owner. - String tableName = "default.testOverrideTableOwner"; - String owner = "alice"; - CreateTableOptions options = ClientTestUtil.getBasicCreateTableOptions(); - options.setOwner(owner); - client.createTable(tableName, ClientTestUtil.getBasicSchema(), options); - - // Create an HMS client. Kudu doesn't provide an API to look up the table owner, - // so it's necessary to use the HMS APIs directly. - HiveMetastoreConfig hmsConfig = client.getHiveMetastoreConfig(); - HiveConf hiveConf = new HiveConf(); - hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, hmsConfig.getHiveMetastoreUris()); - hiveConf.setBoolVar( - HiveConf.ConfVars.METASTORE_USE_THRIFT_SASL, - hmsConfig.getHiveMetastoreSaslEnabled()); - - // Check that the owner of the table in the HMS matches. - IMetaStoreClient hmsClient = new HiveMetaStoreClient(hiveConf); - assertEquals(owner, hmsClient.getTable("default", "testOverrideTableOwner").getOwner()); - - // Altering the table should not result in a change of ownership. - client.alterTable( - tableName, new AlterTableOptions().renameTable("default.testOverrideTableOwner_renamed")); - assertEquals(owner, hmsClient.getTable("default", "testOverrideTableOwner_renamed").getOwner()); - } -} diff --git a/java/kudu-client/src/test/java/org/apache/kudu/client/TestKeyEncoding.java b/java/kudu-client/src/test/java/org/apache/kudu/client/TestKeyEncoding.java index 7aff44f..dae90d4 100644 --- a/java/kudu-client/src/test/java/org/apache/kudu/client/TestKeyEncoding.java +++ b/java/kudu-client/src/test/java/org/apache/kudu/client/TestKeyEncoding.java @@ -118,7 +118,7 @@ public class TestKeyEncoding { Schema schemaOneString = buildSchema(new ColumnSchema.ColumnSchemaBuilder("key", Type.STRING).key(true)); KuduTable table = new KuduTable(null, "one", "one", schemaOneString, - defaultPartitionSchema(schemaOneString), 3, null); + defaultPartitionSchema(schemaOneString), 3, null, null); Insert oneKeyInsert = new Insert(table); PartialRow row = oneKeyInsert.getRow(); row.addString("key", "foo"); @@ -128,7 +128,7 @@ public class TestKeyEncoding { new ColumnSchema.ColumnSchemaBuilder("key", Type.STRING).key(true), new ColumnSchema.ColumnSchemaBuilder("key2", Type.STRING).key(true)); KuduTable table2 = new KuduTable(null, "two", "two", schemaTwoString, - defaultPartitionSchema(schemaTwoString), 3, null); + defaultPartitionSchema(schemaTwoString), 3, null, null); Insert twoKeyInsert = new Insert(table2); row = twoKeyInsert.getRow(); row.addString("key", "foo"); @@ -147,7 +147,7 @@ public class TestKeyEncoding { new ColumnSchema.ColumnSchemaBuilder("key2", Type.STRING).key(true)); PartitionSchema partitionSchemaIntString = defaultPartitionSchema(schemaIntString); KuduTable table3 = new KuduTable(null, "three", "three", - schemaIntString, partitionSchemaIntString, 3, null); + schemaIntString, partitionSchemaIntString, 3, null, null); Insert small = new Insert(table3); row = small.getRow(); row.addInt("key", 20);
