This is an automated email from the ASF dual-hosted git repository.
granthenke pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kudu.git
The following commit(s) were added to refs/heads/master by this push:
new faa8c5d KUDU-3187: Enhance the HMS plugin to check if synchronization
is enabled
faa8c5d is described below
commit faa8c5d4f21ab8bc3e42995d0202246e2a5417b6
Author: Grant Henke <[email protected]>
AuthorDate: Sun Aug 30 12:19:43 2020 -0500
KUDU-3187: Enhance the HMS plugin to check if synchronization is enabled
This patch enhances the HMS plugin to check if synchronization
is enabled on the Kudu cluster backing the table. This can
simplify enabling HMS synchronization because the plugin
jar can be added to the HMS classpath unconditionally.
This also helps support cases where multiple Kudu clusters
use the same HMS and one is synchronized and the other isn’t.
This patch introduces a new environment property,
`KUDU_HMS_SYNC_ENABLED`, to the plugin which allows
for faster tests that do not need to setup a Kudu cluster.
It could also be useful to disable this new functionality as
a workaround if issues are seen.
Additonally, this patch adjusts the mini HMS to add
security properties configuration. This is required now
that the Kudu client runs in the plugin and communicates
with the cluster. See these commits for context:
-
https://github.com/apache/kudu/commit/0ee93e31a1febb987c72e7392a69b2584e6f38ed
-
https://github.com/apache/kudu/commit/3343144fefaad5a30e95e21297c64c78e308fa1f
I also manually verified the following behavior on a real cluster:
- `KUDU_HMS_SYNC_ENABLED` environment variable
- CREATE/ALTER/DROP on non-sync cluster works
- - With plugin jar used by the HMS
- CREATE/ALTER/DROP on sync cluster fails with clear message
- Authn via Kerberos
Change-Id: Ib3588d72af1bb499202b47fca50a08876e13ea37
Reviewed-on: http://gerrit.cloudera.org:8080/16388
Tested-by: Kudu Jenkins
Reviewed-by: Andrew Wong <[email protected]>
---
java/kudu-hive/build.gradle | 6 +-
.../kudu/hive/metastore/KuduMetastorePlugin.java | 162 +++++++++++++++++----
.../hive/metastore/TestKuduMetastorePlugin.java | 63 ++++++--
src/kudu/hms/hms_catalog-test.cc | 6 +
src/kudu/hms/hms_client-test.cc | 11 ++
src/kudu/hms/mini_hms.cc | 25 ++++
src/kudu/hms/mini_hms.h | 9 ++
7 files changed, 239 insertions(+), 43 deletions(-)
diff --git a/java/kudu-hive/build.gradle b/java/kudu-hive/build.gradle
index e18eac6..b86d912 100644
--- a/java/kudu-hive/build.gradle
+++ b/java/kudu-hive/build.gradle
@@ -14,11 +14,11 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
+apply from: "$rootDir/gradle/shadow.gradle"
dependencies {
- // NOTE: The build currently expects all dependencies will be available
- // via the hive-metastore classpath. Therefore, all dependencies should
- // be provided scope.
+ compile project(path: ":kudu-client", configuration: "shadow")
+
provided libs.hiveMetastore
provided libs.slf4jApi
// Transitively required through hive-metastore,
diff --git
a/java/kudu-hive/src/main/java/org/apache/kudu/hive/metastore/KuduMetastorePlugin.java
b/java/kudu-hive/src/main/java/org/apache/kudu/hive/metastore/KuduMetastorePlugin.java
index 2acc8e1..814107a 100644
---
a/java/kudu-hive/src/main/java/org/apache/kudu/hive/metastore/KuduMetastorePlugin.java
+++
b/java/kudu-hive/src/main/java/org/apache/kudu/hive/metastore/KuduMetastorePlugin.java
@@ -17,8 +17,11 @@
package org.apache.kudu.hive.metastore;
+import java.io.IOException;
+import java.security.PrivilegedExceptionAction;
import java.util.Map;
import java.util.Objects;
+import java.util.concurrent.ConcurrentHashMap;
import com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.conf.Configuration;
@@ -32,6 +35,11 @@ import
org.apache.hadoop.hive.metastore.events.AlterTableEvent;
import org.apache.hadoop.hive.metastore.events.CreateTableEvent;
import org.apache.hadoop.hive.metastore.events.DropTableEvent;
import org.apache.hadoop.hive.metastore.events.ListenerEvent;
+import org.apache.hadoop.security.UserGroupInformation;
+
+import org.apache.kudu.client.HiveMetastoreConfig;
+import org.apache.kudu.client.KuduClient;
+import org.apache.kudu.client.KuduException;
/**
* The {@code KuduMetastorePlugin} intercepts DDL operations on Kudu table
entries
@@ -91,6 +99,15 @@ public class KuduMetastorePlugin extends
MetaStoreEventListener {
// System env to track if the HMS plugin validation should be skipped.
static final String SKIP_VALIDATION_ENV = "KUDU_SKIP_HMS_PLUGIN_VALIDATION";
+ // System env to force sync enabled/disabled without a call to the master.
+ // This is useful for testing and could be useful as an escape hatch if there
+ // are too many requests to the master.
+ static final String SYNC_ENABLED_ENV = "KUDU_HMS_SYNC_ENABLED";
+
+ // Maps lists of master addresses to KuduClients to cache clients.
+ private static final Map<String, KuduClient> KUDU_CLIENTS =
+ new ConcurrentHashMap<String, KuduClient>();
+
public KuduMetastorePlugin(Configuration config) {
super(config);
}
@@ -117,22 +134,34 @@ public class KuduMetastorePlugin extends
MetaStoreEventListener {
return;
}
- if (!isKuduMasterAction(tableEvent)) {
- throw new MetaException("Kudu tables may not be created through Hive");
+ // In the case of table creation all Kudu tables must have
+ // the master addresses property. We explicitly check for it here
+ // so that `kuduSyncEnabled` below doesn't return false for Kudu
+ // tables that are missing the master addresses property.
+ checkMasterAddrsProperty(table);
+
+ // Only validate tables for clusters with HMS sync enabled.
+ if (!kuduSyncEnabled(tableEvent, table)) {
+ return;
}
checkKuduProperties(table);
+
+ if (!isKuduMasterAction(tableEvent)) {
+ throw new MetaException("Kudu tables may not be created through Hive");
+ }
}
@Override
public void onDropTable(DropTableEvent tableEvent) throws MetaException {
super.onDropTable(tableEvent);
- Table table = tableEvent.getTable();
if (skipsValidation()) {
return;
}
+ Table table = tableEvent.getTable();
+
// Only validate synchronized tables.
if (!isSynchronizedTable(table)) {
return;
@@ -142,7 +171,25 @@ public class KuduMetastorePlugin extends
MetaStoreEventListener {
String targetTableId = environmentContext == null ? null :
environmentContext.getProperties().get(KUDU_TABLE_ID_KEY);
+ // Allow non-Kudu tables to be dropped.
+ if (!isKuduTable(table)) {
+ // However, make sure it doesn't have a table id from the context.
+ // The table id is only meant for Kudu tables and set by the Kudu master.
+ if (targetTableId != null) {
+ throw new MetaException("Kudu table ID does not match the non-Kudu HMS
entry");
+ }
+ return;
+ }
+
+ // Only validate tables for clusters with HMS sync enabled.
+ if (!kuduSyncEnabled(tableEvent, table)) {
+ return;
+ }
+
// If this request doesn't specify a Kudu table ID then allow it to
proceed.
+ // Drop table requests that don't come from the Kudu master may not set
the table ID,
+ // e.g. when dropping tables via Hive, or dropping orphaned HMS entries.
+ // Such tables are dropped in Kudu by name via the notification listener.
if (targetTableId == null) {
return;
}
@@ -150,11 +197,6 @@ public class KuduMetastorePlugin extends
MetaStoreEventListener {
// The kudu.master_event property isn't checked, because the kudu.table_id
// property already implies this event is coming from a Kudu Master.
- // Check that the table being dropped is a Kudu table.
- if (!isKuduTable(table)) {
- throw new MetaException("Kudu table ID does not match the non-Kudu HMS
entry");
- }
-
// Check that the table's ID matches the request's table ID.
if (!targetTableId.equals(table.getParameters().get(KUDU_TABLE_ID_KEY))) {
throw new MetaException("Kudu table ID does not match the HMS entry");
@@ -172,11 +214,22 @@ public class KuduMetastorePlugin extends
MetaStoreEventListener {
Table oldTable = tableEvent.getOldTable();
Table newTable = tableEvent.getNewTable();
+ // Allow non-Kudu tables to be altered.
+ if (!isKuduTable(oldTable) && !isLegacyKuduTable(oldTable)) {
+ // Allow non-Kudu tables to be altered without introducing Kudu-specific
+ // properties.
+ checkNoKuduProperties(newTable);
+ return;
+ }
+
+ // Only validate tables for clusters with HMS sync enabled.
+ if (!kuduSyncEnabled(tableEvent, oldTable)) {
+ return;
+ }
+
// Prevent altering the table type (managed/external) of Kudu tables (or
via
// altering table properties 'EXTERNAL' or `external.table.purge`).
- // This can cause orphaned tables and the Sentry integration depends on
having a
- // synchronized table for each Kudu table to prevent security issues due
to overlapping
- // names with Kudu tables and tables in the HMS.
+ // This can cause orphaned tables.
// Note: This doesn't prevent altering the table type for legacy tables
// because they should continue to work as they always have primarily for
// migration purposes.
@@ -228,10 +281,6 @@ public class KuduMetastorePlugin extends
MetaStoreEventListener {
throw new MetaException("Kudu table ID does not match the existing
HMS entry");
}
}
- } else {
- // Allow non-Kudu tables to be altered without introducing Kudu-specific
- // properties.
- checkNoKuduProperties(newTable);
}
}
@@ -240,7 +289,7 @@ public class KuduMetastorePlugin extends
MetaStoreEventListener {
* @param table the table to check
* @return {@code true} if the table is a Kudu table, otherwise {@code false}
*/
- private boolean isKuduTable(Table table) {
+ private static boolean isKuduTable(Table table) {
String storageHandler =
table.getParameters().get(hive_metastoreConstants.META_TABLE_STORAGE);
return KUDU_STORAGE_HANDLER.equals(storageHandler);
}
@@ -253,7 +302,7 @@ public class KuduMetastorePlugin extends
MetaStoreEventListener {
* @return {@code true} if the table is a legacy Kudu table,
* otherwise {@code false}
*/
- private boolean isLegacyKuduTable(Table table) {
+ private static boolean isLegacyKuduTable(Table table) {
return LEGACY_KUDU_STORAGE_HANDLER.equals(table.getParameters()
.get(hive_metastoreConstants.META_TABLE_STORAGE));
}
@@ -265,7 +314,7 @@ public class KuduMetastorePlugin extends
MetaStoreEventListener {
* @return {@code true} if the table is an external table,
* otherwise {@code false}
*/
- private boolean isExternalTable(Table table) {
+ private static boolean isExternalTable(Table table) {
String isExternal = table.getParameters().get(EXTERNAL_TABLE_KEY);
if (isExternal == null) {
return false;
@@ -282,7 +331,7 @@ public class KuduMetastorePlugin extends
MetaStoreEventListener {
* @return {@code true} if the table is a managed table or has
external.table.purge = true,
* otherwise {@code false}
*/
- private boolean isPurgeTable(Table table) {
+ private static boolean isPurgeTable(Table table) {
boolean externalPurge =
Boolean.parseBoolean(table.getParameters().getOrDefault(EXTERNAL_PURGE_KEY,
"false"));
return TableType.MANAGED_TABLE.name().equals(table.getTableType()) ||
externalPurge;
@@ -295,7 +344,7 @@ public class KuduMetastorePlugin extends
MetaStoreEventListener {
* @return {@code true} if the table is a managed table or an external table
with
* `external.table.purge = true`, otherwise {@code false}
*/
- private boolean isSynchronizedTable(Table table) {
+ private static boolean isSynchronizedTable(Table table) {
return TableType.MANAGED_TABLE.name().equals(table.getTableType()) ||
(isExternalTable(table) && isPurgeTable(table));
}
@@ -304,7 +353,7 @@ public class KuduMetastorePlugin extends
MetaStoreEventListener {
* Checks that the Kudu table entry contains the required Kudu table
properties.
* @param table the table to check
*/
- private void checkKuduProperties(Table table) throws MetaException {
+ private static void checkKuduProperties(Table table) throws MetaException {
if (!isKuduTable(table)) {
throw new MetaException(String.format(
"Kudu table entry must contain a Kudu storage handler property
(%s=%s)",
@@ -316,6 +365,14 @@ public class KuduMetastorePlugin extends
MetaStoreEventListener {
throw new MetaException(String.format(
"Kudu table entry must contain a table ID property (%s)",
KUDU_TABLE_ID_KEY));
}
+ checkMasterAddrsProperty(table);
+ }
+
+ /**
+ * Checks that the Kudu table entry contains the `kudu.master_addresses`
property.
+ * @param table the table to check
+ */
+ private static void checkMasterAddrsProperty(Table table) throws
MetaException {
String masterAddresses = table.getParameters().get(KUDU_MASTER_ADDRS_KEY);
if (masterAddresses == null || masterAddresses.isEmpty()) {
throw new MetaException(String.format(
@@ -327,7 +384,7 @@ public class KuduMetastorePlugin extends
MetaStoreEventListener {
* Checks that the non-Kudu table entry does not contain Kudu-specific table
properties.
* @param table the table to check
*/
- private void checkNoKuduProperties(Table table) throws MetaException {
+ private static void checkNoKuduProperties(Table table) throws MetaException {
if (isKuduTable(table)) {
throw new MetaException(String.format(
"non-Kudu table entry must not contain the Kudu storage handler
(%s=%s)",
@@ -347,7 +404,7 @@ public class KuduMetastorePlugin extends
MetaStoreEventListener {
* @param oldTable the table to be altered
* @param newTable the new altered table
*/
- private void checkOnlyKuduMasterCanAlterSchema(AlterTableEvent tableEvent,
+ private static void checkOnlyKuduMasterCanAlterSchema(AlterTableEvent
tableEvent,
Table oldTable, Table newTable) throws MetaException {
if (!isKuduMasterAction(tableEvent) &&
!oldTable.getSd().getCols().equals(newTable.getSd().getCols())) {
@@ -358,7 +415,7 @@ public class KuduMetastorePlugin extends
MetaStoreEventListener {
/**
* Returns true if the event is from the Kudu Master.
*/
- private boolean isKuduMasterAction(ListenerEvent event) {
+ private static boolean isKuduMasterAction(ListenerEvent event) {
EnvironmentContext environmentContext = event.getEnvironmentContext();
if (environmentContext == null) {
return false;
@@ -380,7 +437,7 @@ public class KuduMetastorePlugin extends
MetaStoreEventListener {
* Returns true if the table ID should be verified on an event.
* Defaults to true.
*/
- private boolean checkTableID(ListenerEvent event) {
+ private static boolean checkTableID(ListenerEvent event) {
EnvironmentContext environmentContext = event.getEnvironmentContext();
if (environmentContext == null) {
return true;
@@ -409,4 +466,59 @@ public class KuduMetastorePlugin extends
MetaStoreEventListener {
}
return true;
}
+
+ /**
+ * Returns true if HMS synchronization is configured on the Kudu cluster
+ * backing the HMS table.
+ */
+ private static boolean kuduSyncEnabled(ListenerEvent event, Table table)
throws MetaException {
+ // If SYNC_ENABLED_ENV is set, use it instead of contacting the Kudu
master.
+ String envEnabled = System.getenv(SYNC_ENABLED_ENV);
+ if (envEnabled != null && !envEnabled.isEmpty()) {
+ return Integer.parseInt(envEnabled) == 1;
+ }
+
+ // If the request is from the Kudu Master, we know HMS sync is enabled
+ // and can avoid another request.
+ if (isKuduMasterAction(event)) {
+ return true;
+ }
+
+ String masterAddresses = table.getParameters().get(KUDU_MASTER_ADDRS_KEY);
+ if (masterAddresses == null || masterAddresses.isEmpty()) {
+ // A table without master addresses is not synchronized,
+ // it may not even be a Kudu table.
+ return false;
+ }
+
+ KuduClient kuduClient = getKuduClient(masterAddresses);
+ HiveMetastoreConfig hmsConfig;
+ try {
+ hmsConfig = kuduClient.getHiveMetastoreConfig();
+ } catch (KuduException e) {
+ throw new MetaException(
+ String.format("Error determining if Kudu's integration with " +
+ "the Hive Metastore is enabled: %s", e.getMessage()));
+ }
+
+ // If the HiveMetastoreConfig is not null, then the HMS synchronization
+ // is enabled in the Kudu cluster.
+ return hmsConfig != null;
+ }
+
+ private static KuduClient getKuduClient(String kuduMasters) {
+ KuduClient client = KUDU_CLIENTS.get(kuduMasters);
+ if (client == null) {
+ try {
+ client = UserGroupInformation.getLoginUser().doAs(
+ (PrivilegedExceptionAction<KuduClient>) () ->
+ new KuduClient.KuduClientBuilder(kuduMasters).build()
+ );
+ } catch (IOException | InterruptedException e) {
+ throw new RuntimeException("Failed to create the Kudu client");
+ }
+ KUDU_CLIENTS.put(kuduMasters, client);
+ }
+ return client;
+ }
}
diff --git
a/java/kudu-hive/src/test/java/org/apache/kudu/hive/metastore/TestKuduMetastorePlugin.java
b/java/kudu-hive/src/test/java/org/apache/kudu/hive/metastore/TestKuduMetastorePlugin.java
index 0730d12..4b6c4c9 100644
---
a/java/kudu-hive/src/test/java/org/apache/kudu/hive/metastore/TestKuduMetastorePlugin.java
+++
b/java/kudu-hive/src/test/java/org/apache/kudu/hive/metastore/TestKuduMetastorePlugin.java
@@ -38,34 +38,27 @@ import org.apache.hadoop.hive.metastore.api.SerDeInfo;
import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
import org.apache.hadoop.hive.metastore.api.Table;
import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants;
-import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils;
import org.apache.thrift.TException;
-import org.hamcrest.CoreMatchers;
import org.junit.After;
-import org.junit.Before;
-import org.junit.Rule;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.kudu.test.junit.RetryRule;
+import org.apache.kudu.test.cluster.MiniKuduCluster;
public class TestKuduMetastorePlugin {
private static final Logger LOG =
LoggerFactory.getLogger(TestKuduMetastorePlugin.class);
private HiveConf clientConf;
private HiveMetaStoreClient client;
+ private MiniKuduCluster miniCluster;
private EnvironmentContext masterContext() {
return new
EnvironmentContext(ImmutableMap.of(KuduMetastorePlugin.KUDU_MASTER_EVENT,
"true"));
}
- @Rule
- public RetryRule retryRule = new RetryRule();
-
- @Before
- public void setUp() throws Exception {
+ public void startCluster(boolean syncEnabled) throws Exception {
HiveConf metastoreConf = new HiveConf();
// Avoids a dependency on the default partition expression class, which is
// contained in the hive-exec jar.
@@ -116,12 +109,26 @@ public class TestKuduMetastorePlugin {
clientConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" +
msPort);
client = new HiveMetaStoreClient(clientConf);
+
+ MiniKuduCluster.MiniKuduClusterBuilder mcb = new
MiniKuduCluster.MiniKuduClusterBuilder();
+ if (syncEnabled) {
+ mcb.addMasterServerFlag("--hive_metastore_uris=thrift://localhost:" +
msPort);
+ }
+ miniCluster = mcb.numMasterServers(3)
+ .numTabletServers(0)
+ .build();
}
@After
public void tearDown() {
- if (client != null) {
- client.close();
+ try {
+ if (client != null) {
+ client.close();
+ }
+ } finally {
+ if (miniCluster != null) {
+ miniCluster.shutdown();
+ }
}
}
@@ -141,7 +148,7 @@ public class TestKuduMetastorePlugin {
table.putToParameters(KuduMetastorePlugin.KUDU_TABLE_ID_KEY,
UUID.randomUUID().toString());
table.putToParameters(KuduMetastorePlugin.KUDU_MASTER_ADDRS_KEY,
- "localhost");
+ miniCluster.getMasterAddressesAsString());
}
// The HMS will NPE if the storage descriptor and partition keys aren't
set...
@@ -172,6 +179,7 @@ public class TestKuduMetastorePlugin {
@Test
public void testCreateTableHandler() throws Exception {
+ startCluster(/* syncEnabled */ true);
// A non-Kudu table with a Kudu table ID should be rejected.
try {
Table table = newTable("table");
@@ -240,6 +248,7 @@ public class TestKuduMetastorePlugin {
@Test
public void testAlterTableHandler() throws Exception {
+ startCluster(/* syncEnabled */ true);
// Test altering a Kudu (or a legacy) table.
Table initTable = newTable("table");
client.createTable(initTable, masterContext());
@@ -401,7 +410,7 @@ public class TestKuduMetastorePlugin {
alteredTable.putToParameters(KuduMetastorePlugin.KUDU_TABLE_NAME,
"legacy_table");
alteredTable.putToParameters(KuduMetastorePlugin.KUDU_MASTER_ADDRS_KEY,
- "localhost");
+ miniCluster.getMasterAddressesAsString());
client.alter_table(table.getDbName(), table.getTableName(),
alteredTable);
}
} finally {
@@ -455,7 +464,7 @@ public class TestKuduMetastorePlugin {
alteredTable.putToParameters(KuduMetastorePlugin.KUDU_TABLE_ID_KEY,
UUID.randomUUID().toString());
alteredTable.putToParameters(KuduMetastorePlugin.KUDU_MASTER_ADDRS_KEY,
- "localhost");
+ miniCluster.getMasterAddressesAsString());
client.alter_table(legacyTable.getDbName(),
legacyTable.getTableName(),
alteredTable);
}
@@ -482,6 +491,7 @@ public class TestKuduMetastorePlugin {
@Test
public void testLegacyTableHandler() throws Exception {
+ startCluster(/* syncEnabled */ true);
// Test creating a legacy Kudu table without context succeeds.
Table table = newLegacyTable("legacy_table");
client.createTable(table);
@@ -508,6 +518,7 @@ public class TestKuduMetastorePlugin {
@Test
public void testDropTableHandler() throws Exception {
+ startCluster(/* syncEnabled */ true);
// Test dropping a Kudu table.
Table table = newTable("table");
client.createTable(table, masterContext());
@@ -588,4 +599,26 @@ public class TestKuduMetastorePlugin {
client.dropTable(table.getDbName(), table.getTableName());
}
}
+
+ @Test
+ public void testSyncDisabled() throws Exception {
+ startCluster(/* syncEnabled */ false);
+
+ // A Kudu table should should be allowed to be created via Hive.
+ Table table = newTable("table");
+ client.createTable(table);
+
+ // A Kudu table should should be allowed to be altered via Hive.
+ // Add a column to the original table.
+ table.getSd().addToCols(new FieldSchema("b", "int", ""));
+ // Also change the location to avoid MetastoreDefaultTransformer validation
+ // that exists in some Hive versions.
+ table.getSd().setLocation(String.format("%s/%s/%s",
+ clientConf.get(HiveConf.ConfVars.METASTOREWAREHOUSE.varname),
+ table.getDbName(), table.getTableName()));
+ client.alter_table(table.getDbName(), table.getTableName(), table);
+
+ // A Kudu table should should be allowed to be dropped via Hive.
+ client.dropTable(table.getDbName(), table.getTableName());
+ }
}
diff --git a/src/kudu/hms/hms_catalog-test.cc b/src/kudu/hms/hms_catalog-test.cc
index eac64c7..d0692c0 100644
--- a/src/kudu/hms/hms_catalog-test.cc
+++ b/src/kudu/hms/hms_catalog-test.cc
@@ -132,6 +132,12 @@ class HmsCatalogTest : public KuduTest {
thrift::ClientOptions hms_client_opts;
hms_.reset(new hms::MiniHms());
+
+ // Set the `KUDU_HMS_SYNC_ENABLED` environment variable in the
+ // HMS environment to manually enable HMS synchronization checks.
+ // This means we don't need to stand up a Kudu Cluster for this test.
+ hms_->AddEnvVar("KUDU_HMS_SYNC_ENABLED", "1");
+
if (enable_kerberos) {
kdc_.reset(new MiniKdc(MiniKdcOptions()));
ASSERT_OK(kdc_->Start());
diff --git a/src/kudu/hms/hms_client-test.cc b/src/kudu/hms/hms_client-test.cc
index 487234c..96484ff 100644
--- a/src/kudu/hms/hms_client-test.cc
+++ b/src/kudu/hms/hms_client-test.cc
@@ -131,6 +131,11 @@ TEST_P(HmsClientTest, TestHmsOperations) {
hms_client_opts.service_principal = "hive";
}
+ // Set the `KUDU_HMS_SYNC_ENABLED` environment variable in the
+ // HMS environment to manually enable HMS synchronization checks.
+ // This means we don't need to stand up a Kudu Cluster for this test.
+ hms.AddEnvVar("KUDU_HMS_SYNC_ENABLED", "1");
+
ASSERT_OK(hms.Start());
HmsClient client(hms.address(), hms_client_opts);
@@ -449,6 +454,12 @@ TEST_F(HmsClientTest, TestDeserializeJsonTable) {
TEST_F(HmsClientTest, TestCaseSensitivity) {
MiniKdc kdc;
MiniHms hms;
+
+ // Set the `KUDU_HMS_SYNC_ENABLED` environment variable in the
+ // HMS environment to manually enable HMS synchronization checks.
+ // This means we don't need to stand up a Kudu Cluster for this test.
+ hms.AddEnvVar("KUDU_HMS_SYNC_ENABLED", "1");
+
ASSERT_OK(hms.Start());
HmsClient client(hms.address(), thrift::ClientOptions());
diff --git a/src/kudu/hms/mini_hms.cc b/src/kudu/hms/mini_hms.cc
index 1158e30..e8f1279 100644
--- a/src/kudu/hms/mini_hms.cc
+++ b/src/kudu/hms/mini_hms.cc
@@ -74,6 +74,10 @@ void MiniHms::EnableKuduPlugin(bool enable) {
enable_kudu_plugin_ = enable;
}
+void MiniHms::AddEnvVar(const string& key, const string& value) {
+ extra_env_vars_.insert({key, value});
+}
+
void MiniHms::SetDataRoot(string data_root) {
CHECK(!hms_process_);
data_root_ = std::move(data_root);
@@ -102,6 +106,7 @@ Status MiniHms::Start() {
data_root_ = GetTestDataDirectory();
}
+ RETURN_NOT_OK(CreateSecurityProperties());
RETURN_NOT_OK(CreateHiveSite());
RETURN_NOT_OK(CreateCoreSite());
RETURN_NOT_OK(CreateLogConfig());
@@ -127,6 +132,10 @@ Status MiniHms::Start() {
// directory. We want it in data_root_.
java_options += Substitute(" -Dderby.stream.error.file=$0/derby.log",
data_root_);
+ // Add the security properties file generated by
`CreateSecurityProperties()`.
+ java_options += Substitute("
-Djava.security.properties=$0/security.properties",
+ data_root_);
+
if (!krb5_conf_.empty()) {
java_options += Substitute(" -Djava.security.krb5.conf=$0", krb5_conf_);
}
@@ -142,6 +151,7 @@ Status MiniHms::Start() {
// TODO(ghenke): Remove after HADOOP-15966 is available (Hadoop 3.1.3+)
{ "HADOOP_OS_TYPE", "Linux" }
};
+ env_vars.insert(extra_env_vars_.begin(), extra_env_vars_.end());
// Run the schematool to initialize the database if not yet initialized.
// Instead of running slow 'schematool -dbType derby -info' to check whether
@@ -205,6 +215,21 @@ string MiniHms::uris() const {
return Substitute("thrift://127.0.0.1:$0", port_);
}
+Status MiniHms::CreateSecurityProperties() const {
+ // Default Java security settings are restrictive with regard to RSA key
+ // length. Since Kudu masters and tablet servers in MiniKuduCluster use
+ // smaller RSA keys to shorten runtime of tests, it's necessary to override
+ // those default security settings to allow for using relaxed cryptography,
+ // particularly smaller RSA keys.
+ string security_file_contents =
+ "jdk.certpath.disabledAlgorithms = MD2, RC4, MD5\n"
+ "jdk.tls.disabledAlgorithms = SSLv3, RC4, MD5\n);";
+
+ return WriteStringToFile(Env::Default(),
+ security_file_contents,
+ JoinPathSegments(data_root_,
"security.properties"));
+}
+
Status MiniHms::CreateHiveSite() const {
const string listeners =
Substitute("org.apache.hive.hcatalog.listener.DbNotificationListener$0",
diff --git a/src/kudu/hms/mini_hms.h b/src/kudu/hms/mini_hms.h
index 1dc64d6..4c2be11 100644
--- a/src/kudu/hms/mini_hms.h
+++ b/src/kudu/hms/mini_hms.h
@@ -18,6 +18,7 @@
#pragma once
#include <cstdint>
+#include <map>
#include <memory>
#include <string>
@@ -49,6 +50,9 @@ class MiniHms {
// Configures the mini HMS to enable or disable the Kudu plugin.
void EnableKuduPlugin(bool enable);
+ // Add extra environment variables to the HMS environment.
+ void AddEnvVar(const std::string& key, const std::string& value);
+
// Configures the mini HMS to store its data in the provided path. If not
set,
// it uses a test-only temporary directory.
void SetDataRoot(std::string data_root);
@@ -85,6 +89,9 @@ class MiniHms {
private:
+ // Creates a security.properties file for use via
`-Djava.security.properties` in the mini HMS.
+ Status CreateSecurityProperties() const WARN_UNUSED_RESULT;
+
// Creates a hive-site.xml for the mini HMS.
Status CreateHiveSite() const WARN_UNUSED_RESULT;
@@ -110,6 +117,8 @@ class MiniHms {
// Whether to enable the Kudu listener plugin.
bool enable_kudu_plugin_ = true;
+
+ std::map<std::string, std::string> extra_env_vars_ = {};
};
} // namespace hms
