This is an automated email from the ASF dual-hosted git repository.
lzljs3620320 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new 7c9f5e638 [hive] Add AWS Glue Catalog Support (#4283)
7c9f5e638 is described below
commit 7c9f5e6385b2acec9de02abcfb40de366aff8366
Author: Lucian <[email protected]>
AuthorDate: Fri Oct 11 11:24:57 2024 +0800
[hive] Add AWS Glue Catalog Support (#4283)
---
.../hive/RetryingMetaStoreClientFactory.java | 72 +++++++++++++++++-----
.../hive/CustomConstructorMetastoreClient.java | 54 ++++++++++++++++
.../apache/paimon/hive/Hive23CatalogITCase.java | 28 +++++++++
.../hive/CustomConstructorMetastoreClient.java | 54 ++++++++++++++++
.../apache/paimon/hive/Hive31CatalogITCase.java | 28 +++++++++
5 files changed, 219 insertions(+), 17 deletions(-)
diff --git
a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/RetryingMetaStoreClientFactory.java
b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/RetryingMetaStoreClientFactory.java
index bc5a88705..2d62f56bc 100644
---
a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/RetryingMetaStoreClientFactory.java
+++
b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/RetryingMetaStoreClientFactory.java
@@ -21,6 +21,7 @@ package org.apache.paimon.hive;
import org.apache.paimon.shade.guava30.com.google.common.collect.ImmutableMap;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.common.JavaUtils;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.HiveMetaHookLoader;
import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
@@ -29,6 +30,7 @@ import
org.apache.hadoop.hive.metastore.RetryingMetaStoreClient;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
+import java.util.Arrays;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.Map.Entry;
@@ -75,6 +77,16 @@ public class RetryingMetaStoreClientFactory {
new ConcurrentHashMap<>(),
clientClassName,
true))
+ .put(
+ new Class<?>[] {
+ HiveConf.class,
+ Class[].class,
+ Object[].class,
+ ConcurrentHashMap.class,
+ String.class
+ },
+ RetryingMetaStoreClientFactory
+
::constructorDetectedHiveMetastoreProxySupplier)
// for hive 3.x
.put(
new Class<?>[] {
@@ -103,23 +115,8 @@ public class RetryingMetaStoreClientFactory {
ConcurrentHashMap.class,
String.class
},
- (getProxyMethod, hiveConf, clientClassName) ->
- (IMetaStoreClient)
- getProxyMethod.invoke(
- null,
- hiveConf,
- new Class[] {
- HiveConf.class,
-
HiveMetaHookLoader.class,
- Boolean.class
- },
- new Object[] {
- hiveConf,
- (HiveMetaHookLoader)
(tbl -> null),
- true
- },
- new ConcurrentHashMap<>(),
- clientClassName))
+ RetryingMetaStoreClientFactory
+
::constructorDetectedHiveMetastoreProxySupplier)
.build();
// If clientClassName is HiveMetaStoreClient,
@@ -175,4 +172,45 @@ public class RetryingMetaStoreClientFactory {
IMetaStoreClient get(Method getProxyMethod, Configuration conf, String
clientClassName)
throws IllegalAccessException, IllegalArgumentException,
InvocationTargetException;
}
+
+ /** Detect the client class whether it has the proper constructor. */
+ private static IMetaStoreClient
constructorDetectedHiveMetastoreProxySupplier(
+ Method getProxyMethod, Configuration hiveConf, String
clientClassName)
+ throws IllegalAccessException, IllegalArgumentException,
InvocationTargetException {
+
+ try {
+ Class<?> baseClass = Class.forName(clientClassName, false,
JavaUtils.getClassLoader());
+
+ // Configuration.class or HiveConf.class
+ Class<?> firstParamType = getProxyMethod.getParameterTypes()[0];
+
+ Class<?>[] fullParams =
+ new Class[] {firstParamType, HiveMetaHookLoader.class,
Boolean.TYPE};
+ Object[] fullParamValues =
+ new Object[] {hiveConf, (HiveMetaHookLoader) (tbl ->
null), Boolean.TRUE};
+
+ for (int i = fullParams.length; i >= 1; i--) {
+ try {
+ baseClass.getConstructor(Arrays.copyOfRange(fullParams, 0,
i));
+ return (IMetaStoreClient)
+ getProxyMethod.invoke(
+ null,
+ hiveConf,
+ Arrays.copyOfRange(fullParams, 0, i),
+ Arrays.copyOfRange(fullParamValues, 0, i),
+ new ConcurrentHashMap<>(),
+ clientClassName);
+ } catch (NoSuchMethodException ignored) {
+ }
+ }
+
+ } catch (ClassNotFoundException e) {
+ throw new RuntimeException(e);
+ }
+
+ throw new IllegalArgumentException(
+ "Failed to create the desired metastore client with proper
constructors (class name: "
+ + clientClassName
+ + ")");
+ }
}
diff --git
a/paimon-hive/paimon-hive-connector-2.3/src/test/java/org/apache/paimon/hive/CustomConstructorMetastoreClient.java
b/paimon-hive/paimon-hive-connector-2.3/src/test/java/org/apache/paimon/hive/CustomConstructorMetastoreClient.java
new file mode 100644
index 000000000..c9ddc1be4
--- /dev/null
+++
b/paimon-hive/paimon-hive-connector-2.3/src/test/java/org/apache/paimon/hive/CustomConstructorMetastoreClient.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.hive;
+
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.HiveMetaHookLoader;
+import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
+import org.apache.hadoop.hive.metastore.IMetaStoreClient;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+
+/** A {@link HiveMetaStoreClient} Factory to test custom Hive metastore
client. */
+public class CustomConstructorMetastoreClient {
+
+ /**
+ * A {@link HiveMetaStoreClient} to test custom Hive metastore client with
(HiveConf,
+ * HiveMetaHookLoader) constructor.
+ */
+ public static class TwoParameterConstructorMetastoreClient extends
HiveMetaStoreClient
+ implements IMetaStoreClient {
+
+ public TwoParameterConstructorMetastoreClient(HiveConf conf,
HiveMetaHookLoader hookLoader)
+ throws MetaException {
+ super(conf, hookLoader);
+ }
+ }
+
+ /**
+ * A {@link HiveMetaStoreClient} to test custom Hive metastore client with
(HiveConf)
+ * constructor.
+ */
+ public static class OneParameterConstructorMetastoreClient extends
HiveMetaStoreClient
+ implements IMetaStoreClient {
+
+ public OneParameterConstructorMetastoreClient(HiveConf conf) throws
MetaException {
+ super(conf);
+ }
+ }
+}
diff --git
a/paimon-hive/paimon-hive-connector-2.3/src/test/java/org/apache/paimon/hive/Hive23CatalogITCase.java
b/paimon-hive/paimon-hive-connector-2.3/src/test/java/org/apache/paimon/hive/Hive23CatalogITCase.java
index 204f779d7..c24e4a608 100644
---
a/paimon-hive/paimon-hive-connector-2.3/src/test/java/org/apache/paimon/hive/Hive23CatalogITCase.java
+++
b/paimon-hive/paimon-hive-connector-2.3/src/test/java/org/apache/paimon/hive/Hive23CatalogITCase.java
@@ -84,6 +84,34 @@ public class Hive23CatalogITCase extends
HiveCatalogITCaseBase {
Row.of(TestHiveMetaStoreClient.MOCK_DATABASE)));
}
+ @Test
+ public void testCustomConstructorMetastoreClient() throws Exception {
+ path = folder.newFolder().toURI().toString();
+ EnvironmentSettings settings =
EnvironmentSettings.newInstance().inBatchMode().build();
+ Class<?>[] customConstructorMetastoreClientClass = {
+
CustomConstructorMetastoreClient.TwoParameterConstructorMetastoreClient.class,
+
CustomConstructorMetastoreClient.OneParameterConstructorMetastoreClient.class
+ };
+ for (Class<?> clazz : customConstructorMetastoreClientClass) {
+ tEnv = TableEnvironmentImpl.create(settings);
+ tEnv.executeSql(
+ String.join(
+ "\n",
+ "CREATE CATALOG my_hive WITH (",
+ " 'type' = 'paimon',",
+ " 'metastore' = 'hive',",
+ " 'uri' = '',",
+ " 'default-database' = 'test_db',",
+ " 'warehouse' = '" + path + "',",
+ " 'metastore.client.class' = '" +
clazz.getName() + "'",
+ ")"))
+ .await();
+ tEnv.executeSql("USE CATALOG my_hive").await();
+ assertThat(collect("SHOW DATABASES"))
+ .isEqualTo(Arrays.asList(Row.of("default"),
Row.of("test_db")));
+ }
+ }
+
@Test
public void testCreateExistTableInHive() throws Exception {
tEnv.executeSql(
diff --git
a/paimon-hive/paimon-hive-connector-3.1/src/test/java/org/apache/paimon/hive/CustomConstructorMetastoreClient.java
b/paimon-hive/paimon-hive-connector-3.1/src/test/java/org/apache/paimon/hive/CustomConstructorMetastoreClient.java
new file mode 100644
index 000000000..35ff3f8bb
--- /dev/null
+++
b/paimon-hive/paimon-hive-connector-3.1/src/test/java/org/apache/paimon/hive/CustomConstructorMetastoreClient.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.hive;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.metastore.HiveMetaHookLoader;
+import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
+import org.apache.hadoop.hive.metastore.IMetaStoreClient;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+
+/** A {@link HiveMetaStoreClient} Factory to test custom Hive metastore
client. */
+public class CustomConstructorMetastoreClient {
+
+ /**
+ * A {@link HiveMetaStoreClient} to test custom Hive metastore client with
(HiveConf,
+ * HiveMetaHookLoader) constructor.
+ */
+ public static class TwoParameterConstructorMetastoreClient extends
HiveMetaStoreClient
+ implements IMetaStoreClient {
+
+ public TwoParameterConstructorMetastoreClient(
+ Configuration conf, HiveMetaHookLoader hookLoader) throws
MetaException {
+ super(conf, hookLoader);
+ }
+ }
+
+ /**
+ * A {@link HiveMetaStoreClient} to test custom Hive metastore client with
(HiveConf)
+ * constructor.
+ */
+ public static class OneParameterConstructorMetastoreClient extends
HiveMetaStoreClient
+ implements IMetaStoreClient {
+
+ public OneParameterConstructorMetastoreClient(Configuration conf)
throws MetaException {
+ super(conf);
+ }
+ }
+}
diff --git
a/paimon-hive/paimon-hive-connector-3.1/src/test/java/org/apache/paimon/hive/Hive31CatalogITCase.java
b/paimon-hive/paimon-hive-connector-3.1/src/test/java/org/apache/paimon/hive/Hive31CatalogITCase.java
index 1cd17553f..7b3ab5626 100644
---
a/paimon-hive/paimon-hive-connector-3.1/src/test/java/org/apache/paimon/hive/Hive31CatalogITCase.java
+++
b/paimon-hive/paimon-hive-connector-3.1/src/test/java/org/apache/paimon/hive/Hive31CatalogITCase.java
@@ -83,6 +83,34 @@ public class Hive31CatalogITCase extends
HiveCatalogITCaseBase {
Row.of(TestHiveMetaStoreClient.MOCK_DATABASE)));
}
+ @Test
+ public void testCustomConstructorMetastoreClient() throws Exception {
+ path = folder.newFolder().toURI().toString();
+ EnvironmentSettings settings =
EnvironmentSettings.newInstance().inBatchMode().build();
+ Class<?>[] customConstructorMetastoreClientClass = {
+
CustomConstructorMetastoreClient.TwoParameterConstructorMetastoreClient.class,
+
CustomConstructorMetastoreClient.OneParameterConstructorMetastoreClient.class
+ };
+
+ for (Class<?> clazz : customConstructorMetastoreClientClass) {
+ tEnv = TableEnvironmentImpl.create(settings);
+ tEnv.executeSql(
+ String.join(
+ "\n",
+ "CREATE CATALOG my_hive WITH (",
+ " 'type' = 'paimon',",
+ " 'metastore' = 'hive',",
+ " 'uri' = '',",
+ " 'warehouse' = '" + path + "',",
+ " 'metastore.client.class' = '" +
clazz.getName() + "'",
+ ")"))
+ .await();
+ tEnv.executeSql("USE CATALOG my_hive").await();
+ assertThat(collect("SHOW DATABASES"))
+ .isEqualTo(Arrays.asList(Row.of("default"),
Row.of("test_db")));
+ }
+ }
+
@Test
public void testCreateExistTableInHive() throws Exception {
tEnv.executeSql(
