This is an automated email from the ASF dual-hosted git repository.
lzljs3620320 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-paimon.git
The following commit(s) were added to refs/heads/master by this push:
new a011fa7d4 [hive] Use default hive-site.xml if hive-conf-dir is not set
(#1024)
a011fa7d4 is described below
commit a011fa7d454a794320c82150e530733ae7965d53
Author: tsreaper <[email protected]>
AuthorDate: Tue Apr 25 21:35:48 2023 +0800
[hive] Use default hive-site.xml if hive-conf-dir is not set (#1024)
---
.../java/org/apache/paimon/hive/HiveCatalog.java | 18 +++++----
.../org/apache/paimon/hive/HiveCatalogTest.java | 31 +++++++++++++++
.../test/resources/hadoop-conf-dir/core-site.xml | 0
.../src/test/resources/hive-conf-dir/hive-site.xml | 0
.../apache/paimon/hive/Hive23CatalogITCase.java | 12 ------
.../apache/paimon/hive/Hive31CatalogITCase.java | 12 ------
.../apache/paimon/hive/HiveCatalogITCaseBase.java | 14 -------
.../apache/paimon/hive/HiveCatalogOptionsTest.java | 46 ----------------------
8 files changed, 41 insertions(+), 92 deletions(-)
diff --git
a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java
b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java
index 9cfb7964b..d8c75dd79 100644
---
a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java
+++
b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java
@@ -583,15 +583,15 @@ public class HiveCatalog extends AbstractCatalog {
if (hadoopConf == null) {
hadoopConf = new Configuration();
}
- // ignore all the static conf file URLs that HiveConf may have set
- HiveConf.setHiveSiteLocation(null);
- HiveConf.setLoadMetastoreConfig(false);
- HiveConf.setLoadHiveServer2Config(false);
- HiveConf hiveConf = new HiveConf(hadoopConf, HiveConf.class);
LOG.info("Setting hive conf dir as {}", hiveConfDir);
-
if (hiveConfDir != null) {
+ // ignore all the static conf file URLs that HiveConf may have set
+ HiveConf.setHiveSiteLocation(null);
+ HiveConf.setLoadMetastoreConfig(false);
+ HiveConf.setLoadHiveServer2Config(false);
+ HiveConf hiveConf = new HiveConf(hadoopConf, HiveConf.class);
+
org.apache.hadoop.fs.Path hiveSite =
new org.apache.hadoop.fs.Path(hiveConfDir, HIVE_SITE_FILE);
if (!hiveSite.toUri().isAbsolute()) {
@@ -606,9 +606,11 @@ public class HiveCatalog extends AbstractCatalog {
"Failed to load hive-site.xml from specified path:" +
hiveSite, e);
}
hiveConf.addResource(hiveSite);
- }
- return hiveConf;
+ return hiveConf;
+ } else {
+ return new HiveConf(hadoopConf, HiveConf.class);
+ }
}
public static boolean isEmbeddedMetastore(HiveConf hiveConf) {
diff --git
a/paimon-hive/paimon-hive-catalog/src/test/java/org/apache/paimon/hive/HiveCatalogTest.java
b/paimon-hive/paimon-hive-catalog/src/test/java/org/apache/paimon/hive/HiveCatalogTest.java
index 9b9ef1be8..cc490c7ed 100644
---
a/paimon-hive/paimon-hive-catalog/src/test/java/org/apache/paimon/hive/HiveCatalogTest.java
+++
b/paimon-hive/paimon-hive-catalog/src/test/java/org/apache/paimon/hive/HiveCatalogTest.java
@@ -83,4 +83,35 @@ public class HiveCatalogTest extends CatalogTestBase {
.hasRootCauseInstanceOf(IllegalStateException.class)
.hasRootCauseMessage("Table name[NEW_TABLE] cannot contain
upper case");
}
+
+ private static final String HADOOP_CONF_DIR =
+
Thread.currentThread().getContextClassLoader().getResource("hadoop-conf-dir").getPath();
+
+ private static final String HIVE_CONF_DIR =
+
Thread.currentThread().getContextClassLoader().getResource("hive-conf-dir").getPath();
+
+ @Test
+ public void testHadoopConfDir() {
+ HiveConf hiveConf = HiveCatalog.createHiveConf(null, HADOOP_CONF_DIR);
+ assertThat(hiveConf.get("fs.defaultFS")).isEqualTo("dummy-fs");
+ }
+
+ @Test
+ public void testHiveConfDir() {
+ try {
+ testHiveConfDirImpl();
+ } finally {
+ cleanUpHiveConfDir();
+ }
+ }
+
+ private void testHiveConfDirImpl() {
+ HiveConf hiveConf = HiveCatalog.createHiveConf(HIVE_CONF_DIR, null);
+ assertThat(hiveConf.get("hive.metastore.uris")).isEqualTo("dummy-hms");
+ }
+
+ private void cleanUpHiveConfDir() {
+ // reset back to default value
+
HiveConf.setHiveSiteLocation(HiveConf.class.getClassLoader().getResource("hive-site.xml"));
+ }
}
diff --git
a/paimon-hive/paimon-hive-connector-common/src/test/resources/hadoop-conf-dir/core-site.xml
b/paimon-hive/paimon-hive-catalog/src/test/resources/hadoop-conf-dir/core-site.xml
similarity index 100%
rename from
paimon-hive/paimon-hive-connector-common/src/test/resources/hadoop-conf-dir/core-site.xml
rename to
paimon-hive/paimon-hive-catalog/src/test/resources/hadoop-conf-dir/core-site.xml
diff --git
a/paimon-hive/paimon-hive-connector-common/src/test/resources/hive-conf-dir/hive-site.xml
b/paimon-hive/paimon-hive-catalog/src/test/resources/hive-conf-dir/hive-site.xml
similarity index 100%
rename from
paimon-hive/paimon-hive-connector-common/src/test/resources/hive-conf-dir/hive-site.xml
rename to
paimon-hive/paimon-hive-catalog/src/test/resources/hive-conf-dir/hive-site.xml
diff --git
a/paimon-hive/paimon-hive-connector-2.3/src/test/java/org/apache/paimon/hive/Hive23CatalogITCase.java
b/paimon-hive/paimon-hive-connector-2.3/src/test/java/org/apache/paimon/hive/Hive23CatalogITCase.java
index f064ac312..b2d38da4b 100644
---
a/paimon-hive/paimon-hive-connector-2.3/src/test/java/org/apache/paimon/hive/Hive23CatalogITCase.java
+++
b/paimon-hive/paimon-hive-connector-2.3/src/test/java/org/apache/paimon/hive/Hive23CatalogITCase.java
@@ -72,10 +72,6 @@ public class Hive23CatalogITCase extends
HiveCatalogITCaseBase {
" 'metastore' = 'hive',",
" 'uri' = '',",
" 'warehouse' = '" + path + "',",
- " 'hive-conf-dir' = '"
- +
hiveShell.getBaseDir().getRoot().getPath()
- + HIVE_CONF
- + "',",
" 'metastore.client.class' = '"
+
TestHiveMetaStoreClient.class.getName()
+ "'",
@@ -100,10 +96,6 @@ public class Hive23CatalogITCase extends
HiveCatalogITCaseBase {
" 'metastore' = 'hive',",
" 'uri' = '',",
" 'warehouse' = '" + path + "',",
- " 'hive-conf-dir' = '"
- + hiveShell.getBaseDir().getRoot().getPath()
- + HIVE_CONF
- + "',",
" 'metastore.client.class' = '"
+ CreateFailHiveMetaStoreClient.class.getName()
+ "'",
@@ -135,10 +127,6 @@ public class Hive23CatalogITCase extends
HiveCatalogITCaseBase {
" 'metastore' = 'hive',",
" 'uri' = '',",
" 'warehouse' = '" + path + "',",
- " 'hive-conf-dir' = '"
- +
hiveShell.getBaseDir().getRoot().getPath()
- + HIVE_CONF
- + "',",
" 'metastore.client.class' = '"
+
AlterFailHiveMetaStoreClient.class.getName()
+ "'",
diff --git
a/paimon-hive/paimon-hive-connector-3.1/src/test/java/org/apache/paimon/hive/Hive31CatalogITCase.java
b/paimon-hive/paimon-hive-connector-3.1/src/test/java/org/apache/paimon/hive/Hive31CatalogITCase.java
index 6ccaa871b..84860dcde 100644
---
a/paimon-hive/paimon-hive-connector-3.1/src/test/java/org/apache/paimon/hive/Hive31CatalogITCase.java
+++
b/paimon-hive/paimon-hive-connector-3.1/src/test/java/org/apache/paimon/hive/Hive31CatalogITCase.java
@@ -72,10 +72,6 @@ public class Hive31CatalogITCase extends
HiveCatalogITCaseBase {
" 'metastore' = 'hive',",
" 'uri' = '',",
" 'warehouse' = '" + path + "',",
- " 'hive-conf-dir' = '"
- +
hiveShell.getBaseDir().getRoot().getPath()
- + HIVE_CONF
- + "',",
" 'metastore.client.class' = '"
+
TestHiveMetaStoreClient.class.getName()
+ "'",
@@ -100,10 +96,6 @@ public class Hive31CatalogITCase extends
HiveCatalogITCaseBase {
" 'metastore' = 'hive',",
" 'uri' = '',",
" 'warehouse' = '" + path + "',",
- " 'hive-conf-dir' = '"
- + hiveShell.getBaseDir().getRoot().getPath()
- + HIVE_CONF
- + "',",
" 'metastore.client.class' = '"
+ CreateFailHiveMetaStoreClient.class.getName()
+ "'",
@@ -135,10 +127,6 @@ public class Hive31CatalogITCase extends
HiveCatalogITCaseBase {
" 'metastore' = 'hive',",
" 'uri' = '',",
" 'warehouse' = '" + path + "',",
- " 'hive-conf-dir' = '"
- +
hiveShell.getBaseDir().getRoot().getPath()
- + HIVE_CONF
- + "',",
" 'metastore.client.class' = '"
+
AlterFailHiveMetaStoreClient.class.getName()
+ "'",
diff --git
a/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveCatalogITCaseBase.java
b/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveCatalogITCaseBase.java
index 575a51fe6..1065a51db 100644
---
a/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveCatalogITCaseBase.java
+++
b/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveCatalogITCaseBase.java
@@ -68,8 +68,6 @@ public abstract class HiveCatalogITCaseBase {
@HiveSQL(files = {})
protected static HiveShell hiveShell;
- public static final String HIVE_CONF = "/hive-conf";
-
@Before
public void before() throws Exception {
hiveShell.execute("CREATE DATABASE IF NOT EXISTS test_db");
@@ -88,10 +86,6 @@ public abstract class HiveCatalogITCaseBase {
" 'type' = 'paimon',",
" 'metastore' = 'hive',",
" 'uri' = '',",
- " 'hive-conf-dir' = '"
- +
hiveShell.getBaseDir().getRoot().getPath()
- + HIVE_CONF
- + "',",
" 'warehouse' = '" + path + "',",
" 'lock.enabled' = 'true'",
")"))
@@ -235,10 +229,6 @@ public abstract class HiveCatalogITCaseBase {
" 'metastore' = 'hive',",
" 'uri' = '',",
" 'warehouse' = '" + path + "',",
- " 'hive-conf-dir' = '"
- +
hiveShell.getBaseDir().getRoot().getPath()
- + HIVE_CONF
- + "',",
" 'lock.enabled' = 'true',",
" 'table.type' = 'EXTERNAL'",
")"))
@@ -538,10 +528,6 @@ public abstract class HiveCatalogITCaseBase {
" 'type' = 'paimon',",
" 'metastore' = 'hive',",
" 'uri' = '',",
- " 'hive-conf-dir' = '"
- +
hiveShell.getBaseDir().getRoot().getPath()
- + HIVE_CONF
- + "',",
" 'warehouse' = '" + path + "',",
" 'lock.enabled' = 'true',",
" 'table-default.opt1' = 'value1',",
diff --git
a/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveCatalogOptionsTest.java
b/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveCatalogOptionsTest.java
deleted file mode 100644
index f9cf0a96b..000000000
---
a/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveCatalogOptionsTest.java
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.paimon.hive;
-
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.junit.Test;
-
-import static org.assertj.core.api.Assertions.assertThat;
-
-/** Test for HiveCatalogOptions. */
-public class HiveCatalogOptionsTest {
-
- public static final String HADOOP_CONF_DIR =
-
Thread.currentThread().getContextClassLoader().getResource("hadoop-conf-dir").getPath();
-
- public static final String HIVE_CONF_DIR =
-
Thread.currentThread().getContextClassLoader().getResource("hive-conf-dir").getPath();
-
- @Test
- public void testHadoopConfDir() {
- HiveConf hiveConf = HiveCatalog.createHiveConf(null, HADOOP_CONF_DIR);
- assertThat(hiveConf.get("fs.defaultFS")).isEqualTo("dummy-fs");
- }
-
- @Test
- public void testHiveConfDir() {
- HiveConf hiveConf = HiveCatalog.createHiveConf(HIVE_CONF_DIR, null);
- assertThat(hiveConf.get("hive.metastore.uris")).isEqualTo("dummy-hms");
- }
-}
