This is an automated email from the ASF dual-hosted git repository.
zhangliang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/shardingsphere.git
The following commit(s) were added to refs/heads/master by this push:
new a12eb276133 Fix the issue that ShardingSphere cannot connect to
HiveServer2 using remote Hive Metastore Server (#33837)
a12eb276133 is described below
commit a12eb276133be769afc6a79fce08c33e9d3a6988
Author: Ling Hengqian <[email protected]>
AuthorDate: Sat Nov 30 11:45:18 2024 +0800
Fix the issue that ShardingSphere cannot connect to HiveServer2 using
remote Hive Metastore Server (#33837)
---
RELEASE-NOTES.md | 1 +
.../optional-plugins/hiveserver2/_index.cn.md | 49 +++++++++-
.../optional-plugins/hiveserver2/_index.en.md | 49 +++++++++-
.../hadoop-common/3.3.6/reflect-config.json | 8 ++
.../hadoop-common/3.3.6/resource-config.json | 6 --
.../hive-jdbc/4.0.1/reflect-config.json | 11 +++
.../proxy-config.json | 4 +
.../reflect-config.json | 104 ++++++++++++++++-----
.../resource-config.json | 70 +++++++++++++-
.../reflect-config.json | 5 -
.../resource-config.json | 2 +-
pom.xml | 1 +
.../pipeline/cases/PipelineContainerComposer.java | 3 +-
test/native/native-image-filter/extra-filter.json | 1 +
test/native/pom.xml | 11 +++
...overyTest.java => StandaloneMetastoreTest.java} | 95 ++++---------------
.../hive/ZookeeperServiceDiscoveryTest.java | 12 +--
.../reflect-config.json | 8 ++
.../yaml/jdbc/databases/hive/standalone-hms.yaml | 64 +++++++++++++
19 files changed, 380 insertions(+), 124 deletions(-)
diff --git a/RELEASE-NOTES.md b/RELEASE-NOTES.md
index 41639f697bc..c24280170f2 100644
--- a/RELEASE-NOTES.md
+++ b/RELEASE-NOTES.md
@@ -46,6 +46,7 @@
1. Encrypt: Fix merge exception without encrypt rule in database -
[#33708](https://github.com/apache/shardingsphere/pull/33708)
1. SQL Parser: Fix mysql parse zone unreserved keyword error -
[#33720](https://github.com/apache/shardingsphere/pull/33720)
1. Proxy: Fix BatchUpdateException when execute INSERT INTO ON DUPLICATE KEY
UPDATE in proxy adapter -
[#33796](https://github.com/apache/shardingsphere/pull/33796)
+1. Infra: Fix the issue that ShardingSphere cannot connect to HiveServer2
using remote Hive Metastore Server -
[#33837](https://github.com/apache/shardingsphere/pull/33837)
### Change Logs
diff --git
a/docs/document/content/user-manual/shardingsphere-jdbc/optional-plugins/hiveserver2/_index.cn.md
b/docs/document/content/user-manual/shardingsphere-jdbc/optional-plugins/hiveserver2/_index.cn.md
index 97fc19a4f6a..ab5a4e35a6f 100644
---
a/docs/document/content/user-manual/shardingsphere-jdbc/optional-plugins/hiveserver2/_index.cn.md
+++
b/docs/document/content/user-manual/shardingsphere-jdbc/optional-plugins/hiveserver2/_index.cn.md
@@ -40,6 +40,17 @@ ShardingSphere 对 HiveServer2 JDBC Driver 的支持位于可选模块中。
<artifactId>hive-service</artifactId>
<version>4.0.1</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-client-core</artifactId>
+ <version>3.3.6</version>
+ <exclusions>
+ <exclusion>
+ <groupId>*</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
</dependencies>
```
@@ -81,6 +92,17 @@ ShardingSphere 对 HiveServer2 JDBC Driver 的支持位于可选模块中。
</exclusion>
</exclusions>
</dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-client-core</artifactId>
+ <version>3.3.6</version>
+ <exclusions>
+ <exclusion>
+ <groupId>*</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
</dependencies>
```
@@ -427,8 +449,31 @@ ShardingSphere 仅针对 HiveServer2 `4.0.1` 进行集成测试。
### Hadoop 限制
用户仅可使用 Hadoop `3.3.6` 来作为 HiveServer2 JDBC Driver `4.0.1` 的底层 Hadoop 依赖。
-HiveServer2 JDBC Driver `4.0.1` 不支持 Hadoop `3.4.1`，
-参考 https://github.com/apache/hive/pull/5500 。
+HiveServer2 JDBC Driver `4.0.1` 不支持 Hadoop `3.4.1`， 参考
https://github.com/apache/hive/pull/5500 。
+
+对于 HiveServer2 JDBC Driver `org.apache.hive:hive-jdbc:4.0.1` 或 `classifier` 为
`standalone` 的 `org.apache.hive:hive-jdbc:4.0.1`，
+实际上并不额外依赖 `org.apache.hadoop:hadoop-mapreduce-client-core:3.3.6`。
+
+但 `org.apache.shardingsphere:shardingsphere-infra-database-hive` 的
+`org.apache.shardingsphere.infra.database.hive.metadata.data.loader.HiveMetaDataLoader`
会使用 `org.apache.hadoop.hive.conf.HiveConf`，
+这进一步使用了 `org.apache.hadoop:hadoop-mapreduce-client-core:3.3.6` 的
`org.apache.hadoop.mapred.JobConf` 类。
+
+ShardingSphere 仅需要使用 `org.apache.hadoop.mapred.JobConf` 类，
+因此排除 `org.apache.hadoop:hadoop-mapreduce-client-core:3.3.6` 的所有额外依赖是合理行为。
+
+```xml
+<dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-client-core</artifactId>
+ <version>3.3.6</version>
+ <exclusions>
+ <exclusion>
+ <groupId>*</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ </exclusions>
+</dependency>
+```
### SQL 限制
diff --git
a/docs/document/content/user-manual/shardingsphere-jdbc/optional-plugins/hiveserver2/_index.en.md
b/docs/document/content/user-manual/shardingsphere-jdbc/optional-plugins/hiveserver2/_index.en.md
index 3cbcff19cb1..b9de7a27997 100644
---
a/docs/document/content/user-manual/shardingsphere-jdbc/optional-plugins/hiveserver2/_index.en.md
+++
b/docs/document/content/user-manual/shardingsphere-jdbc/optional-plugins/hiveserver2/_index.en.md
@@ -41,6 +41,17 @@ The possible Maven dependencies are as follows.
<artifactId>hive-service</artifactId>
<version>4.0.1</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-client-core</artifactId>
+ <version>3.3.6</version>
+ <exclusions>
+ <exclusion>
+ <groupId>*</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
</dependencies>
```
@@ -83,6 +94,17 @@ The following is an example of a possible configuration,
</exclusion>
</exclusions>
</dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-client-core</artifactId>
+ <version>3.3.6</version>
+ <exclusions>
+ <exclusion>
+ <groupId>*</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
</dependencies>
```
@@ -433,8 +455,31 @@ Reference https://issues.apache.org/jira/browse/HIVE-28418.
### Hadoop Limitations
Users can only use Hadoop `3.3.6` as the underlying Hadoop dependency of
HiveServer2 JDBC Driver `4.0.1`.
-HiveServer2 JDBC Driver `4.0.1` does not support Hadoop `3.4.1`,
-Reference https://github.com/apache/hive/pull/5500.
+HiveServer2 JDBC Driver `4.0.1` does not support Hadoop `3.4.1`. Reference
https://github.com/apache/hive/pull/5500 .
+
+For HiveServer2 JDBC Driver `org.apache.hive:hive-jdbc:4.0.1` or
`org.apache.hive:hive-jdbc:4.0.1` with `classifier` as `standalone`,
+there is actually no additional dependency on
`org.apache.hadoop:hadoop-mapreduce-client-core:3.3.6`.
+
+But `org.apache.shardingsphere:shardingsphere-infra-database-hive`'s
+`org.apache.shardingsphere.infra.database.hive.metadata.data.loader.HiveMetaDataLoader`
uses `org.apache.hadoop.hive.conf.HiveConf`,
+which further uses `org.apache.hadoop:hadoop-mapreduce-client-core:3.3.6`'s
`org.apache.hadoop.mapred.JobConf` class.
+
+ShardingSphere only needs to use the `org.apache.hadoop.mapred.JobConf` class,
+so it is reasonable to exclude all additional dependencies of
`org.apache.hadoop:hadoop-mapreduce-client-core:3.3.6`.
+
+```xml
+<dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-client-core</artifactId>
+ <version>3.3.6</version>
+ <exclusions>
+ <exclusion>
+ <groupId>*</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ </exclusions>
+</dependency>
+```
### SQL Limitations
diff --git
a/infra/reachability-metadata/src/main/resources/META-INF/native-image/org.apache.hadoop/hadoop-common/3.3.6/reflect-config.json
b/infra/reachability-metadata/src/main/resources/META-INF/native-image/org.apache.hadoop/hadoop-common/3.3.6/reflect-config.json
new file mode 100644
index 00000000000..2920c4f5163
--- /dev/null
+++
b/infra/reachability-metadata/src/main/resources/META-INF/native-image/org.apache.hadoop/hadoop-common/3.3.6/reflect-config.json
@@ -0,0 +1,8 @@
+[
+{
+
"condition":{"typeReachable":"org.apache.hadoop.security.UserGroupInformation"},
+ "name":"org.apache.hadoop.security.UserGroupInformation$UgiMetrics",
+ "allDeclaredFields": true,
+ "allDeclaredMethods": true
+}
+]
diff --git
a/infra/reachability-metadata/src/main/resources/META-INF/native-image/org.apache.hadoop/hadoop-common/3.3.6/resource-config.json
b/infra/reachability-metadata/src/main/resources/META-INF/native-image/org.apache.hadoop/hadoop-common/3.3.6/resource-config.json
index a8e2a9038aa..30741259e02 100644
---
a/infra/reachability-metadata/src/main/resources/META-INF/native-image/org.apache.hadoop/hadoop-common/3.3.6/resource-config.json
+++
b/infra/reachability-metadata/src/main/resources/META-INF/native-image/org.apache.hadoop/hadoop-common/3.3.6/resource-config.json
@@ -3,12 +3,6 @@
"includes":[{
"condition":{"typeReachable":"org.apache.hadoop.conf.Configuration"},
"pattern":"\\Qhadoop-site.xml\\E"
- }, {
- "condition":{"typeReachable":"org.apache.hadoop.conf.Configuration"},
- "pattern":"\\Qcore-default.xml\\E"
- }, {
- "condition":{"typeReachable":"org.apache.hadoop.conf.Configuration"},
- "pattern":"\\Qcore-site.xml\\E"
}]},
"bundles":[]
}
diff --git
a/infra/reachability-metadata/src/main/resources/META-INF/native-image/org.apache.hive/hive-jdbc/4.0.1/reflect-config.json
b/infra/reachability-metadata/src/main/resources/META-INF/native-image/org.apache.hive/hive-jdbc/4.0.1/reflect-config.json
new file mode 100644
index 00000000000..3dd25918d61
--- /dev/null
+++
b/infra/reachability-metadata/src/main/resources/META-INF/native-image/org.apache.hive/hive-jdbc/4.0.1/reflect-config.json
@@ -0,0 +1,11 @@
+[
+{
+
"condition":{"typeReachable":"org.apache.hadoop.hive.metastore.HiveMetaStoreClient"},
+ "name":"org.apache.hadoop.hive.metastore.DefaultMetaStoreFilterHookImpl",
+
"methods":[{"name":"<init>","parameterTypes":["org.apache.hadoop.conf.Configuration"]
}]
+},
+{
+
"condition":{"typeReachable":"org.apache.hadoop.hive.metastore.conf.MetastoreConf"},
+ "name":"org.apache.hadoop.hive.metastore.DefaultMetaStoreFilterHookImpl"
+}
+]
diff --git
a/infra/reachability-metadata/src/main/resources/META-INF/native-image/org.apache.shardingsphere/generated-reachability-metadata/proxy-config.json
b/infra/reachability-metadata/src/main/resources/META-INF/native-image/org.apache.shardingsphere/generated-reachability-metadata/proxy-config.json
index d230828e54b..f2fb40f7040 100644
---
a/infra/reachability-metadata/src/main/resources/META-INF/native-image/org.apache.shardingsphere/generated-reachability-metadata/proxy-config.json
+++
b/infra/reachability-metadata/src/main/resources/META-INF/native-image/org.apache.shardingsphere/generated-reachability-metadata/proxy-config.json
@@ -3,6 +3,10 @@
"condition":{"typeReachable":"org.apache.shardingsphere.proxy.initializer.BootstrapInitializer"},
"interfaces":["java.sql.Connection"]
},
+ {
+
"condition":{"typeReachable":"org.apache.shardingsphere.infra.database.hive.metadata.data.loader.HiveMetaDataLoader"},
+ "interfaces":["org.apache.hadoop.metrics2.MetricsSystem$Callback"]
+ },
{
"condition":{"typeReachable":"org.apache.shardingsphere.driver.jdbc.core.datasource.ShardingSphereDataSource"},
"interfaces":["org.apache.hive.service.rpc.thrift.TCLIService$Iface"]
diff --git
a/infra/reachability-metadata/src/main/resources/META-INF/native-image/org.apache.shardingsphere/generated-reachability-metadata/reflect-config.json
b/infra/reachability-metadata/src/main/resources/META-INF/native-image/org.apache.shardingsphere/generated-reachability-metadata/reflect-config.json
index a6d7095620d..3988f03fe69 100644
---
a/infra/reachability-metadata/src/main/resources/META-INF/native-image/org.apache.shardingsphere/generated-reachability-metadata/reflect-config.json
+++
b/infra/reachability-metadata/src/main/resources/META-INF/native-image/org.apache.shardingsphere/generated-reachability-metadata/reflect-config.json
@@ -16,7 +16,7 @@
"name":"[Lcom.github.dockerjava.api.model.VolumesFrom;"
},
{
-
"condition":{"typeReachable":"org.apache.shardingsphere.driver.jdbc.core.connection.DriverDatabaseConnectionManager$$Lambda/0x00007f547fcab290"},
+
"condition":{"typeReachable":"org.apache.shardingsphere.driver.jdbc.core.connection.DriverDatabaseConnectionManager$$Lambda/0x00007f1137caca88"},
"name":"[Lcom.zaxxer.hikari.util.ConcurrentBag$IConcurrentBagEntry;"
},
{
@@ -75,10 +75,6 @@
"condition":{"typeReachable":"org.apache.shardingsphere.mode.repository.standalone.jdbc.JDBCRepository"},
"name":"[Lcom.zaxxer.hikari.util.ConcurrentBag$IConcurrentBagEntry;"
},
-{
-
"condition":{"typeReachable":"org.apache.shardingsphere.proxy.backend.connector.jdbc.datasource.JDBCBackendDataSource"},
- "name":"[Lcom.zaxxer.hikari.util.ConcurrentBag$IConcurrentBagEntry;"
-},
{
"condition":{"typeReachable":"org.apache.shardingsphere.proxy.frontend.postgresql.command.query.extended.Portal"},
"name":"[Lcom.zaxxer.hikari.util.ConcurrentBag$IConcurrentBagEntry;"
@@ -99,6 +95,10 @@
"condition":{"typeReachable":"org.apache.shardingsphere.mode.repository.standalone.jdbc.JDBCRepository"},
"name":"[Ljava.sql.Statement;"
},
+{
+
"condition":{"typeReachable":"org.apache.shardingsphere.infra.database.hive.metadata.data.loader.HiveMetaDataLoader"},
+ "name":"com.sun.security.auth.UnixPrincipal"
+},
{
"condition":{"typeReachable":"org.apache.shardingsphere.infra.util.yaml.YamlEngine"},
"name":"java.beans.PropertyVetoException"
@@ -283,6 +283,12 @@
"condition":{"typeReachable":"org.apache.shardingsphere.infra.expr.groovy.GroovyInlineExpressionParser"},
"name":"java.lang.Number"
},
+{
+
"condition":{"typeReachable":"org.apache.shardingsphere.infra.database.hive.metadata.data.loader.HiveMetaDataLoader"},
+ "name":"java.lang.Object",
+ "allDeclaredFields":true,
+ "queryAllDeclaredMethods":true
+},
{
"condition":{"typeReachable":"org.apache.shardingsphere.infra.expr.groovy.GroovyInlineExpressionParser"},
"name":"java.lang.Object",
@@ -1105,6 +1111,11 @@
"condition":{"typeReachable":"org.apache.shardingsphere.infra.spi.type.ordered.OrderedSPILoader"},
"name":"org.apache.shardingsphere.encrypt.checker.config.EncryptRuleConfigurationChecker"
},
+{
+
"condition":{"typeReachable":"org.apache.shardingsphere.driver.jdbc.core.statement.ShardingSphereStatement"},
+
"name":"org.apache.shardingsphere.encrypt.checker.sql.EncryptSupportedSQLCheckersBuilder",
+ "methods":[{"name":"<init>","parameterTypes":[] }]
+},
{
"condition":{"typeReachable":"org.apache.shardingsphere.proxy.backend.connector.StandardDatabaseConnector"},
"name":"org.apache.shardingsphere.encrypt.checker.sql.EncryptSupportedSQLCheckersBuilder",
@@ -1307,7 +1318,7 @@
"methods":[{"name":"<init>","parameterTypes":[] }]
},
{
-
"condition":{"typeReachable":"org.apache.shardingsphere.sharding.checker.ShardingRuleConfigurationChecker"},
+
"condition":{"typeReachable":"org.apache.shardingsphere.sharding.checker.config.ShardingRuleConfigurationChecker"},
"name":"org.apache.shardingsphere.infra.algorithm.keygen.snowflake.SnowflakeKeyGenerateAlgorithm",
"methods":[{"name":"<init>","parameterTypes":[] }]
},
@@ -1322,7 +1333,7 @@
"methods":[{"name":"<init>","parameterTypes":[] }]
},
{
-
"condition":{"typeReachable":"org.apache.shardingsphere.sharding.checker.ShardingRuleConfigurationChecker"},
+
"condition":{"typeReachable":"org.apache.shardingsphere.sharding.checker.config.ShardingRuleConfigurationChecker"},
"name":"org.apache.shardingsphere.infra.algorithm.keygen.uuid.UUIDKeyGenerateAlgorithm",
"methods":[{"name":"<init>","parameterTypes":[] }]
},
@@ -2069,7 +2080,7 @@
"queryAllDeclaredMethods":true
},
{
-
"condition":{"typeReachable":"org.apache.shardingsphere.mode.manager.cluster.listener.DatabaseMetaDataChangedListener$$Lambda/0x00007f547fb27268"},
+
"condition":{"typeReachable":"org.apache.shardingsphere.mode.manager.cluster.listener.DatabaseMetaDataChangedListener$$Lambda/0x00007f1137b2dc80"},
"name":"org.apache.shardingsphere.mode.manager.cluster.event.subscriber.dispatch.MetaDataChangedSubscriber"
},
{
@@ -2675,7 +2686,7 @@
"methods":[{"name":"<init>","parameterTypes":[] }]
},
{
-
"condition":{"typeReachable":"org.apache.shardingsphere.sharding.checker.ShardingRuleConfigurationChecker"},
+
"condition":{"typeReachable":"org.apache.shardingsphere.sharding.checker.config.ShardingRuleConfigurationChecker"},
"name":"org.apache.shardingsphere.sharding.algorithm.sharding.classbased.ClassBasedShardingAlgorithm",
"methods":[{"name":"<init>","parameterTypes":[] }]
},
@@ -2685,7 +2696,7 @@
"methods":[{"name":"<init>","parameterTypes":[] }]
},
{
-
"condition":{"typeReachable":"org.apache.shardingsphere.sharding.checker.ShardingRuleConfigurationChecker"},
+
"condition":{"typeReachable":"org.apache.shardingsphere.sharding.checker.config.ShardingRuleConfigurationChecker"},
"name":"org.apache.shardingsphere.sharding.algorithm.sharding.datetime.AutoIntervalShardingAlgorithm",
"methods":[{"name":"<init>","parameterTypes":[] }]
},
@@ -2695,7 +2706,7 @@
"methods":[{"name":"<init>","parameterTypes":[] }]
},
{
-
"condition":{"typeReachable":"org.apache.shardingsphere.sharding.checker.ShardingRuleConfigurationChecker"},
+
"condition":{"typeReachable":"org.apache.shardingsphere.sharding.checker.config.ShardingRuleConfigurationChecker"},
"name":"org.apache.shardingsphere.sharding.algorithm.sharding.datetime.IntervalShardingAlgorithm",
"methods":[{"name":"<init>","parameterTypes":[] }]
},
@@ -2705,7 +2716,7 @@
"methods":[{"name":"<init>","parameterTypes":[] }]
},
{
-
"condition":{"typeReachable":"org.apache.shardingsphere.sharding.checker.ShardingRuleConfigurationChecker"},
+
"condition":{"typeReachable":"org.apache.shardingsphere.sharding.checker.config.ShardingRuleConfigurationChecker"},
"name":"org.apache.shardingsphere.sharding.algorithm.sharding.hint.HintInlineShardingAlgorithm",
"methods":[{"name":"<init>","parameterTypes":[] }]
},
@@ -2715,7 +2726,7 @@
"methods":[{"name":"<init>","parameterTypes":[] }]
},
{
-
"condition":{"typeReachable":"org.apache.shardingsphere.sharding.checker.ShardingRuleConfigurationChecker"},
+
"condition":{"typeReachable":"org.apache.shardingsphere.sharding.checker.config.ShardingRuleConfigurationChecker"},
"name":"org.apache.shardingsphere.sharding.algorithm.sharding.inline.ComplexInlineShardingAlgorithm",
"methods":[{"name":"<init>","parameterTypes":[] }]
},
@@ -2725,7 +2736,7 @@
"methods":[{"name":"<init>","parameterTypes":[] }]
},
{
-
"condition":{"typeReachable":"org.apache.shardingsphere.sharding.checker.ShardingRuleConfigurationChecker"},
+
"condition":{"typeReachable":"org.apache.shardingsphere.sharding.checker.config.ShardingRuleConfigurationChecker"},
"name":"org.apache.shardingsphere.sharding.algorithm.sharding.inline.InlineShardingAlgorithm",
"methods":[{"name":"<init>","parameterTypes":[] }]
},
@@ -2735,7 +2746,7 @@
"methods":[{"name":"<init>","parameterTypes":[] }]
},
{
-
"condition":{"typeReachable":"org.apache.shardingsphere.sharding.checker.ShardingRuleConfigurationChecker"},
+
"condition":{"typeReachable":"org.apache.shardingsphere.sharding.checker.config.ShardingRuleConfigurationChecker"},
"name":"org.apache.shardingsphere.sharding.algorithm.sharding.mod.HashModShardingAlgorithm",
"methods":[{"name":"<init>","parameterTypes":[] }]
},
@@ -2745,7 +2756,7 @@
"methods":[{"name":"<init>","parameterTypes":[] }]
},
{
-
"condition":{"typeReachable":"org.apache.shardingsphere.sharding.checker.ShardingRuleConfigurationChecker"},
+
"condition":{"typeReachable":"org.apache.shardingsphere.sharding.checker.config.ShardingRuleConfigurationChecker"},
"name":"org.apache.shardingsphere.sharding.algorithm.sharding.mod.ModShardingAlgorithm",
"methods":[{"name":"<init>","parameterTypes":[] }]
},
@@ -2755,7 +2766,7 @@
"methods":[{"name":"<init>","parameterTypes":[] }]
},
{
-
"condition":{"typeReachable":"org.apache.shardingsphere.sharding.checker.ShardingRuleConfigurationChecker"},
+
"condition":{"typeReachable":"org.apache.shardingsphere.sharding.checker.config.ShardingRuleConfigurationChecker"},
"name":"org.apache.shardingsphere.sharding.algorithm.sharding.range.BoundaryBasedRangeShardingAlgorithm",
"methods":[{"name":"<init>","parameterTypes":[] }]
},
@@ -2765,7 +2776,7 @@
"methods":[{"name":"<init>","parameterTypes":[] }]
},
{
-
"condition":{"typeReachable":"org.apache.shardingsphere.sharding.checker.ShardingRuleConfigurationChecker"},
+
"condition":{"typeReachable":"org.apache.shardingsphere.sharding.checker.config.ShardingRuleConfigurationChecker"},
"name":"org.apache.shardingsphere.sharding.algorithm.sharding.range.VolumeBasedRangeShardingAlgorithm",
"methods":[{"name":"<init>","parameterTypes":[] }]
},
@@ -2780,7 +2791,17 @@
},
{
"condition":{"typeReachable":"org.apache.shardingsphere.infra.spi.type.ordered.OrderedSPILoader"},
-
"name":"org.apache.shardingsphere.sharding.checker.ShardingRuleConfigurationChecker"
+
"name":"org.apache.shardingsphere.sharding.checker.config.ShardingRuleConfigurationChecker"
+},
+{
+
"condition":{"typeReachable":"org.apache.shardingsphere.driver.jdbc.core.statement.ShardingSphereStatement"},
+
"name":"org.apache.shardingsphere.sharding.checker.sql.ShardingSupportedSQLCheckersBuilder",
+ "methods":[{"name":"<init>","parameterTypes":[] }]
+},
+{
+
"condition":{"typeReachable":"org.apache.shardingsphere.proxy.backend.connector.StandardDatabaseConnector"},
+
"name":"org.apache.shardingsphere.sharding.checker.sql.ShardingSupportedSQLCheckersBuilder",
+ "methods":[{"name":"<init>","parameterTypes":[] }]
},
{
"condition":{"typeReachable":"org.apache.shardingsphere.sqlfederation.engine.SQLFederationEngine"},
@@ -3056,6 +3077,16 @@
"condition":{"typeReachable":"org.apache.shardingsphere.infra.util.yaml.YamlEngine"},
"name":"org.apache.shardingsphere.sharding.yaml.engine.construct.NoneShardingStrategyConfigurationYamlConstruct"
},
+{
+
"condition":{"typeReachable":"org.apache.shardingsphere.driver.jdbc.core.statement.ShardingSphereStatement"},
+
"name":"org.apache.shardingsphere.single.checker.sql.SingleSupportedSQLCheckersBuilder",
+ "methods":[{"name":"<init>","parameterTypes":[] }]
+},
+{
+
"condition":{"typeReachable":"org.apache.shardingsphere.proxy.backend.connector.StandardDatabaseConnector"},
+
"name":"org.apache.shardingsphere.single.checker.sql.SingleSupportedSQLCheckersBuilder",
+ "methods":[{"name":"<init>","parameterTypes":[] }]
+},
{
"condition":{"typeReachable":"org.apache.shardingsphere.sqlfederation.engine.SQLFederationEngine"},
"name":"org.apache.shardingsphere.single.decider.SingleSQLFederationDecider"
@@ -3328,6 +3359,11 @@
"name":"org.apache.shardingsphere.sql.parser.statement.clickhouse.dml.ClickHouseInsertStatement",
"methods":[{"name":"<init>","parameterTypes":[] }]
},
+{
+
"condition":{"typeReachable":"org.apache.shardingsphere.driver.jdbc.core.statement.ShardingSpherePreparedStatement"},
+
"name":"org.apache.shardingsphere.sql.parser.statement.clickhouse.dml.ClickHouseSelectStatement",
+ "methods":[{"name":"<init>","parameterTypes":[] }]
+},
{
"condition":{"typeReachable":"org.apache.shardingsphere.driver.jdbc.core.statement.ShardingSphereStatement"},
"name":"org.apache.shardingsphere.sql.parser.statement.clickhouse.dml.ClickHouseSelectStatement",
@@ -3338,6 +3374,11 @@
"name":"org.apache.shardingsphere.sql.parser.statement.hive.dml.HiveInsertStatement",
"methods":[{"name":"<init>","parameterTypes":[] }]
},
+{
+
"condition":{"typeReachable":"org.apache.shardingsphere.driver.jdbc.core.statement.ShardingSpherePreparedStatement"},
+
"name":"org.apache.shardingsphere.sql.parser.statement.hive.dml.HiveSelectStatement",
+ "methods":[{"name":"<init>","parameterTypes":[] }]
+},
{
"condition":{"typeReachable":"org.apache.shardingsphere.proxy.backend.handler.ProxyBackendHandlerFactory"},
"name":"org.apache.shardingsphere.sql.parser.statement.mysql.dml.MySQLDeleteStatement",
@@ -3353,6 +3394,11 @@
"name":"org.apache.shardingsphere.sql.parser.statement.mysql.dml.MySQLInsertStatement",
"methods":[{"name":"<init>","parameterTypes":[] }]
},
+{
+
"condition":{"typeReachable":"org.apache.shardingsphere.driver.jdbc.core.statement.ShardingSpherePreparedStatement"},
+
"name":"org.apache.shardingsphere.sql.parser.statement.mysql.dml.MySQLSelectStatement",
+ "methods":[{"name":"<init>","parameterTypes":[] }]
+},
{
"condition":{"typeReachable":"org.apache.shardingsphere.driver.jdbc.core.statement.ShardingSphereStatement"},
"name":"org.apache.shardingsphere.sql.parser.statement.mysql.dml.MySQLSelectStatement",
@@ -3368,6 +3414,11 @@
"name":"org.apache.shardingsphere.sql.parser.statement.opengauss.dml.OpenGaussInsertStatement",
"methods":[{"name":"<init>","parameterTypes":[] }]
},
+{
+
"condition":{"typeReachable":"org.apache.shardingsphere.driver.jdbc.core.statement.ShardingSpherePreparedStatement"},
+
"name":"org.apache.shardingsphere.sql.parser.statement.opengauss.dml.OpenGaussSelectStatement",
+ "methods":[{"name":"<init>","parameterTypes":[] }]
+},
{
"condition":{"typeReachable":"org.apache.shardingsphere.driver.jdbc.core.statement.ShardingSphereStatement"},
"name":"org.apache.shardingsphere.sql.parser.statement.opengauss.dml.OpenGaussSelectStatement",
@@ -3388,6 +3439,11 @@
"name":"org.apache.shardingsphere.sql.parser.statement.postgresql.dml.PostgreSQLInsertStatement",
"methods":[{"name":"<init>","parameterTypes":[] }]
},
+{
+
"condition":{"typeReachable":"org.apache.shardingsphere.driver.jdbc.core.statement.ShardingSpherePreparedStatement"},
+
"name":"org.apache.shardingsphere.sql.parser.statement.postgresql.dml.PostgreSQLSelectStatement",
+ "methods":[{"name":"<init>","parameterTypes":[] }]
+},
{
"condition":{"typeReachable":"org.apache.shardingsphere.driver.jdbc.core.statement.ShardingSphereStatement"},
"name":"org.apache.shardingsphere.sql.parser.statement.postgresql.dml.PostgreSQLSelectStatement",
@@ -3403,6 +3459,11 @@
"name":"org.apache.shardingsphere.sql.parser.statement.sqlserver.dml.SQLServerInsertStatement",
"methods":[{"name":"<init>","parameterTypes":[] }]
},
+{
+
"condition":{"typeReachable":"org.apache.shardingsphere.driver.jdbc.core.statement.ShardingSpherePreparedStatement"},
+
"name":"org.apache.shardingsphere.sql.parser.statement.sqlserver.dml.SQLServerSelectStatement",
+ "methods":[{"name":"<init>","parameterTypes":[] }]
+},
{
"condition":{"typeReachable":"org.apache.shardingsphere.driver.jdbc.core.statement.ShardingSphereStatement"},
"name":"org.apache.shardingsphere.sql.parser.statement.sqlserver.dml.SQLServerSelectStatement",
@@ -3588,10 +3649,5 @@
{
"condition":{"typeReachable":"org.apache.shardingsphere.proxy.initializer.BootstrapInitializer"},
"name":"org.apache.shardingsphere.transaction.yaml.config.YamlTransactionRuleConfigurationCustomizer"
-},
-{
-
"condition":{"typeReachable":"org.apache.shardingsphere.infra.executor.kernel.ExecutorEngine"},
- "name":"sun.security.provider.SecureRandom",
- "methods":[{"name":"<init>","parameterTypes":[] },
{"name":"<init>","parameterTypes":["java.security.SecureRandomParameters"] }]
}
]
\ No newline at end of file
diff --git
a/infra/reachability-metadata/src/main/resources/META-INF/native-image/org.apache.shardingsphere/generated-reachability-metadata/resource-config.json
b/infra/reachability-metadata/src/main/resources/META-INF/native-image/org.apache.shardingsphere/generated-reachability-metadata/resource-config.json
index c42c1ecf3f6..4355d10bc0b 100644
---
a/infra/reachability-metadata/src/main/resources/META-INF/native-image/org.apache.shardingsphere/generated-reachability-metadata/resource-config.json
+++
b/infra/reachability-metadata/src/main/resources/META-INF/native-image/org.apache.shardingsphere/generated-reachability-metadata/resource-config.json
@@ -79,7 +79,7 @@
"condition":{"typeReachable":"org.apache.shardingsphere.driver.jdbc.core.connection.DriverDatabaseConnectionManager"},
"pattern":"\\QMETA-INF/services/com.clickhouse.client.ClickHouseClient\\E"
}, {
-
"condition":{"typeReachable":"org.apache.shardingsphere.driver.jdbc.core.connection.DriverDatabaseConnectionManager$$Lambda/0x00007f547fb36cb8"},
+
"condition":{"typeReachable":"org.apache.shardingsphere.driver.jdbc.core.connection.DriverDatabaseConnectionManager$$Lambda/0x00007f1137b39f20"},
"pattern":"\\QMETA-INF/services/com.clickhouse.client.ClickHouseClient\\E"
}, {
"condition":{"typeReachable":"org.apache.shardingsphere.driver.jdbc.core.datasource.ShardingSphereDataSource"},
@@ -114,6 +114,9 @@
}, {
"condition":{"typeReachable":"org.apache.shardingsphere.db.protocol.mysql.constant.MySQLCharacterSet"},
"pattern":"\\QMETA-INF/services/java.nio.charset.spi.CharsetProvider\\E"
+ }, {
+
"condition":{"typeReachable":"org.apache.shardingsphere.infra.database.hive.metadata.data.loader.HiveMetaDataLoader"},
+ "pattern":"\\QMETA-INF/services/java.nio.charset.spi.CharsetProvider\\E"
}, {
"condition":{"typeReachable":"org.apache.shardingsphere.proxy.backend.postgresql.handler.admin.executor.variable.charset.PostgreSQLCharacterSets"},
"pattern":"\\QMETA-INF/services/java.nio.charset.spi.CharsetProvider\\E"
@@ -123,12 +126,18 @@
}, {
"condition":{"typeReachable":"org.apache.shardingsphere.driver.jdbc.core.datasource.ShardingSphereDataSource"},
"pattern":"\\QMETA-INF/services/javax.xml.datatype.DatatypeFactory\\E"
+ }, {
+
"condition":{"typeReachable":"org.apache.shardingsphere.infra.database.hive.metadata.data.loader.HiveMetaDataLoader"},
+
"pattern":"\\QMETA-INF/services/javax.xml.parsers.DocumentBuilderFactory\\E"
}, {
"condition":{"typeReachable":"org.apache.shardingsphere.mode.repository.standalone.jdbc.sql.JDBCRepositorySQLLoader"},
"pattern":"\\QMETA-INF/services/javax.xml.stream.XMLInputFactory\\E"
}, {
"condition":{"typeReachable":"org.apache.shardingsphere.mode.repository.standalone.jdbc.sql.JDBCRepositorySQLLoader"},
"pattern":"\\QMETA-INF/services/javax.xml.stream.XMLOutputFactory\\E"
+ }, {
+
"condition":{"typeReachable":"org.apache.shardingsphere.infra.database.hive.metadata.data.loader.HiveMetaDataLoader"},
+ "pattern":"\\QMETA-INF/services/javax.xml.transform.TransformerFactory\\E"
}, {
"condition":{"typeReachable":"org.apache.shardingsphere.transaction.base.seata.at.SeataATShardingSphereTransactionManager"},
"pattern":"\\QMETA-INF/services/org.apache.seata.config.ConfigurationProvider\\E"
@@ -235,7 +244,7 @@
"condition":{"typeReachable":"org.apache.shardingsphere.proxy.backend.handler.ProxyBackendHandlerFactory"},
"pattern":"\\QMETA-INF/services/org.apache.shardingsphere.infra.executor.checker.SQLExecutionChecker\\E"
}, {
-
"condition":{"typeReachable":"org.apache.shardingsphere.infra.executor.kernel.ExecutorEngine"},
+
"condition":{"typeReachable":"org.apache.shardingsphere.proxy.frontend.mysql.command.query.text.query.MySQLComQueryPacketExecutor"},
"pattern":"\\QMETA-INF/services/org.apache.shardingsphere.infra.executor.sql.hook.SQLExecutionHook\\E"
}, {
"condition":{"typeReachable":"org.apache.shardingsphere.infra.executor.sql.prepare.AbstractExecutionPrepareEngine"},
@@ -387,15 +396,42 @@
}, {
"condition":{"typeReachable":"org.apache.shardingsphere.driver.jdbc.core.datasource.ShardingSphereDataSource"},
"pattern":"\\Qcontainer-license-acceptance.txt\\E"
+ }, {
+
"condition":{"typeReachable":"org.apache.shardingsphere.infra.database.hive.metadata.data.loader.HiveMetaDataLoader"},
+ "pattern":"\\Qcore-default.xml\\E"
+ }, {
+
"condition":{"typeReachable":"org.apache.shardingsphere.infra.database.hive.metadata.data.loader.HiveMetaDataLoader"},
+ "pattern":"\\Qcore-site.xml\\E"
}, {
"condition":{"typeReachable":"org.apache.shardingsphere.infra.autogen.version.ShardingSphereVersion"},
"pattern":"\\Qcurrent-git-commit.properties\\E"
+ }, {
+
"condition":{"typeReachable":"org.apache.shardingsphere.infra.database.hive.metadata.data.loader.HiveMetaDataLoader"},
+ "pattern":"\\Qhive-default.xml\\E"
+ }, {
+
"condition":{"typeReachable":"org.apache.shardingsphere.infra.database.hive.metadata.data.loader.HiveMetaDataLoader"},
+ "pattern":"\\Qhive-site.xml\\E"
+ }, {
+
"condition":{"typeReachable":"org.apache.shardingsphere.infra.database.hive.metadata.data.loader.HiveMetaDataLoader"},
+ "pattern":"\\Qhivemetastore-site.xml\\E"
+ }, {
+
"condition":{"typeReachable":"org.apache.shardingsphere.infra.database.hive.metadata.data.loader.HiveMetaDataLoader"},
+ "pattern":"\\Qhiveserver2-site.xml\\E"
}, {
"condition":{"typeReachable":"org.apache.shardingsphere.transaction.xa.atomikos.manager.AtomikosTransactionManagerProvider"},
"pattern":"\\Qjta.properties\\E"
}, {
"condition":{"typeReachable":"org.apache.shardingsphere.transaction.base.seata.at.SeataATShardingSphereTransactionManager"},
"pattern":"\\Qlib/sqlparser/druid.jar\\E"
+ }, {
+
"condition":{"typeReachable":"org.apache.shardingsphere.infra.database.hive.metadata.data.loader.HiveMetaDataLoader"},
+ "pattern":"\\Qmapred-default.xml\\E"
+ }, {
+
"condition":{"typeReachable":"org.apache.shardingsphere.infra.database.hive.metadata.data.loader.HiveMetaDataLoader"},
+ "pattern":"\\Qmapred-site.xml\\E"
+ }, {
+
"condition":{"typeReachable":"org.apache.shardingsphere.infra.database.hive.metadata.data.loader.HiveMetaDataLoader"},
+ "pattern":"\\Qorg/apache/hadoop/hive/conf/HiveConf.class\\E"
}, {
"condition":{"typeReachable":"org.apache.shardingsphere.proxy.initializer.BootstrapInitializer"},
"pattern":"\\Qorg/h2/util/data.zip\\E"
@@ -1968,6 +2004,9 @@
}, {
"condition":{"typeReachable":"org.apache.shardingsphere.infra.url.classpath.ClassPathURLLoader"},
"pattern":"\\Qtest-native/yaml/jdbc/databases/hive/iceberg.yaml\\E"
+ }, {
+
"condition":{"typeReachable":"org.apache.shardingsphere.infra.url.classpath.ClassPathURLLoader"},
+ "pattern":"\\Qtest-native/yaml/jdbc/databases/hive/standalone-hms.yaml\\E"
}, {
"condition":{"typeReachable":"org.apache.shardingsphere.infra.url.classpath.ClassPathURLLoader"},
"pattern":"\\Qtest-native/yaml/jdbc/databases/hive/zsd.yaml\\E"
@@ -2022,9 +2061,36 @@
}, {
"condition":{"typeReachable":"org.apache.shardingsphere.mode.repository.cluster.etcd.EtcdRepository"},
"pattern":"\\Qvertx-default-jul-logging.properties\\E"
+ }, {
+
"condition":{"typeReachable":"org.apache.shardingsphere.infra.database.hive.metadata.data.loader.HiveMetaDataLoader"},
+ "pattern":"\\Qyarn-default.xml\\E"
+ }, {
+
"condition":{"typeReachable":"org.apache.shardingsphere.infra.database.hive.metadata.data.loader.HiveMetaDataLoader"},
+ "pattern":"\\Qyarn-site.xml\\E"
+ }, {
+
"condition":{"typeReachable":"org.apache.shardingsphere.infra.database.hive.metadata.data.loader.HiveMetaDataLoader"},
+
"pattern":"java.xml:\\Qcom/sun/org/apache/xml/internal/serializer/Encodings.properties\\E"
+ }, {
+
"condition":{"typeReachable":"org.apache.shardingsphere.infra.database.hive.metadata.data.loader.HiveMetaDataLoader"},
+
"pattern":"java.xml:\\Qcom/sun/org/apache/xml/internal/serializer/XMLEntities.properties\\E"
+ }, {
+
"condition":{"typeReachable":"org.apache.shardingsphere.infra.database.hive.metadata.data.loader.HiveMetaDataLoader"},
+
"pattern":"java.xml:\\Qcom/sun/org/apache/xml/internal/serializer/XMLEntities_zh.properties\\E"
+ }, {
+
"condition":{"typeReachable":"org.apache.shardingsphere.infra.database.hive.metadata.data.loader.HiveMetaDataLoader"},
+
"pattern":"java.xml:\\Qcom/sun/org/apache/xml/internal/serializer/XMLEntities_zh_CN.properties\\E"
+ }, {
+
"condition":{"typeReachable":"org.apache.shardingsphere.infra.database.hive.metadata.data.loader.HiveMetaDataLoader"},
+
"pattern":"java.xml:\\Qcom/sun/org/apache/xml/internal/serializer/XMLEntities_zh_Hans.properties\\E"
+ }, {
+
"condition":{"typeReachable":"org.apache.shardingsphere.infra.database.hive.metadata.data.loader.HiveMetaDataLoader"},
+
"pattern":"java.xml:\\Qcom/sun/org/apache/xml/internal/serializer/XMLEntities_zh_Hans_CN.properties\\E"
}]},
"bundles":[{
"name":"com.microsoft.sqlserver.jdbc.SQLServerResource",
"locales":["zh-CN"]
+ }, {
+ "name":"com.sun.org.apache.xml.internal.serializer.XMLEntities",
+ "locales":["zh-CN"]
}]
}
\ No newline at end of file
diff --git
a/infra/reachability-metadata/src/main/resources/META-INF/native-image/org.apache.shardingsphere/shardingsphere-infra-reachability-metadata/reflect-config.json
b/infra/reachability-metadata/src/main/resources/META-INF/native-image/org.apache.shardingsphere/shardingsphere-infra-reachability-metadata/reflect-config.json
index 9bb9fe20748..4597bf4a596 100644
---
a/infra/reachability-metadata/src/main/resources/META-INF/native-image/org.apache.shardingsphere/shardingsphere-infra-reachability-metadata/reflect-config.json
+++
b/infra/reachability-metadata/src/main/resources/META-INF/native-image/org.apache.shardingsphere/shardingsphere-infra-reachability-metadata/reflect-config.json
@@ -332,11 +332,6 @@
"name":"org.apache.shardingsphere.infra.yaml.data.pojo.YamlShardingSphereRowData",
"allPublicMethods": true
},
-{
-
"condition":{"typeReachable":"org.apache.shardingsphere.driver.jdbc.core.statement.ShardingSpherePreparedStatement"},
-
"name":"org.apache.shardingsphere.sql.parser.statement.hive.dml.HiveSelectStatement",
- "methods":[{"name":"<init>","parameterTypes":[] }]
-},
{
"condition":{"typeReachable":"sun.security.provider.SecureRandom"},
"name":"sun.security.provider.SecureRandom",
diff --git
a/infra/reachability-metadata/src/main/resources/META-INF/native-image/org.apache.shardingsphere/shardingsphere-infra-reachability-metadata/resource-config.json
b/infra/reachability-metadata/src/main/resources/META-INF/native-image/org.apache.shardingsphere/shardingsphere-infra-reachability-metadata/resource-config.json
index 8f35719b507..4b68f0d1a87 100644
---
a/infra/reachability-metadata/src/main/resources/META-INF/native-image/org.apache.shardingsphere/shardingsphere-infra-reachability-metadata/resource-config.json
+++
b/infra/reachability-metadata/src/main/resources/META-INF/native-image/org.apache.shardingsphere/shardingsphere-infra-reachability-metadata/resource-config.json
@@ -21,6 +21,6 @@
"locales":["en"]
}, {
"name":"org.opengauss.translation.messages",
- "locales":["zh"]
+ "locales":["en", "zh"]
}]
}
diff --git a/pom.xml b/pom.xml
index 01d49458c93..b5c5a366841 100644
--- a/pom.xml
+++ b/pom.xml
@@ -129,6 +129,7 @@
<clickhouse-jdbc.version>0.6.3</clickhouse-jdbc.version>
<hive.version>4.0.1</hive.version>
<hive-server2-jdbc-driver-thin.version>1.5.0</hive-server2-jdbc-driver-thin.version>
+ <hadoop.version>3.3.6</hadoop.version>
<presto.version>0.288.1</presto.version>
<hikari-cp.version>4.0.3</hikari-cp.version>
diff --git
a/test/e2e/operation/pipeline/src/test/java/org/apache/shardingsphere/test/e2e/data/pipeline/cases/PipelineContainerComposer.java
b/test/e2e/operation/pipeline/src/test/java/org/apache/shardingsphere/test/e2e/data/pipeline/cases/PipelineContainerComposer.java
index 2a9251c52d4..8b306e2f61a 100644
---
a/test/e2e/operation/pipeline/src/test/java/org/apache/shardingsphere/test/e2e/data/pipeline/cases/PipelineContainerComposer.java
+++
b/test/e2e/operation/pipeline/src/test/java/org/apache/shardingsphere/test/e2e/data/pipeline/cases/PipelineContainerComposer.java
@@ -61,6 +61,7 @@ import java.sql.ResultSet;
import java.sql.ResultSetMetaData;
import java.sql.SQLException;
import java.sql.Statement;
+import java.time.Duration;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
@@ -392,7 +393,7 @@ public final class PipelineContainerComposer implements
AutoCloseable {
connection.createStatement().execute(each);
}
}
- Awaitility.await().pollDelay(Math.max(sleepSeconds, 0L),
TimeUnit.SECONDS).until(() -> true);
+
Awaitility.await().timeout(Duration.ofMinutes(1L)).pollDelay(Math.max(sleepSeconds,
0L), TimeUnit.SECONDS).until(() -> true);
}
/**
diff --git a/test/native/native-image-filter/extra-filter.json
b/test/native/native-image-filter/extra-filter.json
index c4df37dbd77..24857b0391c 100644
--- a/test/native/native-image-filter/extra-filter.json
+++ b/test/native/native-image-filter/extra-filter.json
@@ -6,6 +6,7 @@
{"excludeClasses": "com.**"},
{"includeClasses": "com.oracle.svm.core.**"},
{"includeClasses": "com.sun.management.**"},
+ {"includeClasses": "com.sun.security.auth.UnixPrincipal"},
{"excludeClasses": "ch.qos.logback.**"},
{"excludeClasses": "groovy.**"},
{"excludeClasses": "io.**"},
diff --git a/test/native/pom.xml b/test/native/pom.xml
index 41a0d8be279..61b583aabeb 100644
--- a/test/native/pom.xml
+++ b/test/native/pom.xml
@@ -190,6 +190,17 @@
</exclusion>
</exclusions>
</dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-client-core</artifactId>
+ <version>${hadoop.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>*</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
<dependency>
<groupId>org.testcontainers</groupId>
<artifactId>junit-jupiter</artifactId>
diff --git
a/test/native/src/test/java/org/apache/shardingsphere/test/natived/jdbc/databases/hive/ZookeeperServiceDiscoveryTest.java
b/test/native/src/test/java/org/apache/shardingsphere/test/natived/jdbc/databases/hive/StandaloneMetastoreTest.java
similarity index 50%
copy from
test/native/src/test/java/org/apache/shardingsphere/test/natived/jdbc/databases/hive/ZookeeperServiceDiscoveryTest.java
copy to
test/native/src/test/java/org/apache/shardingsphere/test/natived/jdbc/databases/hive/StandaloneMetastoreTest.java
index ee9effbbc38..3b9afd28e15 100644
---
a/test/native/src/test/java/org/apache/shardingsphere/test/natived/jdbc/databases/hive/ZookeeperServiceDiscoveryTest.java
+++
b/test/native/src/test/java/org/apache/shardingsphere/test/natived/jdbc/databases/hive/StandaloneMetastoreTest.java
@@ -19,17 +19,12 @@ package
org.apache.shardingsphere.test.natived.jdbc.databases.hive;
import com.zaxxer.hikari.HikariConfig;
import com.zaxxer.hikari.HikariDataSource;
-import org.apache.curator.framework.CuratorFramework;
-import org.apache.curator.framework.CuratorFrameworkFactory;
-import org.apache.curator.retry.ExponentialBackoffRetry;
-import org.apache.curator.test.InstanceSpec;
import org.apache.shardingsphere.test.natived.commons.TestShardingService;
import org.awaitility.Awaitility;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.condition.EnabledInNativeImage;
-import org.testcontainers.containers.FixedHostPortGenericContainer;
import org.testcontainers.containers.GenericContainer;
import org.testcontainers.containers.Network;
import org.testcontainers.junit.jupiter.Container;
@@ -42,52 +37,38 @@ import java.sql.DriverManager;
import java.sql.SQLException;
import java.sql.Statement;
import java.time.Duration;
-import java.util.List;
import java.util.Properties;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.Matchers.is;
import static org.hamcrest.Matchers.nullValue;
-@SuppressWarnings({"SqlDialectInspection", "SqlNoDataSourceInspection",
"resource", "deprecation"})
+@SuppressWarnings({"SqlDialectInspection", "SqlNoDataSourceInspection",
"resource"})
@EnabledInNativeImage
@Testcontainers
-class ZookeeperServiceDiscoveryTest {
-
- private static final int RANDOM_PORT_FIRST = InstanceSpec.getRandomPort();
+class StandaloneMetastoreTest {
private static final Network NETWORK = Network.newNetwork();
@Container
- private static final GenericContainer<?> ZOOKEEPER_CONTAINER = new
GenericContainer<>("zookeeper:3.9.3-jre-17")
+ public static final GenericContainer<?> HMS_CONTAINER = new
GenericContainer<>("apache/hive:4.0.1")
+ .withEnv("SERVICE_NAME", "metastore")
.withNetwork(NETWORK)
- .withNetworkAliases("foo")
- .withExposedPorts(2181);
+ .withNetworkAliases("metastore");
- /**
- * TODO Maybe we should be able to find a better solution than {@link
InstanceSpec#getRandomPort()} to use a random available port on the host.
- * It is not a good practice to use {@link FixedHostPortGenericContainer}.
- * See <a
href="https://github.com/testcontainers/testcontainers-java/issues/9553";>testcontainers/testcontainers-java#9553</a>
.
- */
- @SuppressWarnings("unused")
@Container
- private static final GenericContainer<?> HIVE_SERVER2_1_CONTAINER = new
FixedHostPortGenericContainer<>("apache/hive:4.0.1")
- .withNetwork(NETWORK)
+ public static final GenericContainer<?> HS2_CONTAINER = new
GenericContainer<>("apache/hive:4.0.1")
.withEnv("SERVICE_NAME", "hiveserver2")
- .withEnv("SERVICE_OPTS",
"-Dhive.server2.support.dynamic.service.discovery=true" + " "
- + "-Dhive.zookeeper.quorum=" +
ZOOKEEPER_CONTAINER.getNetworkAliases().get(0) + ":2181" + " "
- + "-Dhive.server2.thrift.bind.host=0.0.0.0" + " "
- + "-Dhive.server2.thrift.port=" + RANDOM_PORT_FIRST)
- .withFixedExposedPort(RANDOM_PORT_FIRST, RANDOM_PORT_FIRST)
- .dependsOn(ZOOKEEPER_CONTAINER);
+ .withEnv("SERVICE_OPTS",
"-Dhive.metastore.uris=thrift://metastore:9083")
+ .withNetwork(NETWORK)
+ .withExposedPorts(10000)
+ .dependsOn(HMS_CONTAINER);
- private static final String SYSTEM_PROP_KEY_PREFIX =
"fixture.test-native.yaml.database.hive.zsd.";
+ private static final String SYSTEM_PROP_KEY_PREFIX =
"fixture.test-native.yaml.database.hive.hms.";
// Due to https://issues.apache.org/jira/browse/HIVE-28317 , the
`initFile` parameter of HiveServer2 JDBC Driver must be an absolute path.
private static final String ABSOLUTE_PATH =
Paths.get("src/test/resources/test-native/sql/test-native-databases-hive-iceberg.sql").toAbsolutePath().toString();
- private final String jdbcUrlSuffix =
";serviceDiscoveryMode=zooKeeper;zooKeeperNamespace=hiveserver2";
-
private String jdbcUrlPrefix;
@BeforeAll
@@ -105,63 +86,20 @@ class ZookeeperServiceDiscoveryTest {
System.clearProperty(SYSTEM_PROP_KEY_PREFIX + "ds2.jdbc-url");
}
- /**
- * TODO Same problem {@link InstanceSpec#getRandomPort()} as {@code
HIVE_SERVER2_1_CONTAINER}.
- *
- * @throws SQLException SQL exception.
- */
@Test
void assertShardingInLocalTransactions() throws SQLException {
- jdbcUrlPrefix = "jdbc:hive2://" + ZOOKEEPER_CONTAINER.getHost() + ":"
+ ZOOKEEPER_CONTAINER.getMappedPort(2181) + "/";
+ jdbcUrlPrefix = "jdbc:hive2://localhost:" +
HS2_CONTAINER.getMappedPort(10000) + "/";
DataSource dataSource = createDataSource();
TestShardingService testShardingService = new
TestShardingService(dataSource);
testShardingService.processSuccessInHive();
- HIVE_SERVER2_1_CONTAINER.stop();
- int randomPortSecond = InstanceSpec.getRandomPort();
- try (
- GenericContainer<?> hiveServer2SecondContainer = new
FixedHostPortGenericContainer<>("apache/hive:4.0.1")
- .withNetwork(NETWORK)
- .withEnv("SERVICE_NAME", "hiveserver2")
- .withEnv("SERVICE_OPTS",
"-Dhive.server2.support.dynamic.service.discovery=true" + " "
- + "-Dhive.zookeeper.quorum=" +
ZOOKEEPER_CONTAINER.getNetworkAliases().get(0) + ":2181" + " "
- + "-Dhive.server2.thrift.bind.host=0.0.0.0" +
" "
- + "-Dhive.server2.thrift.port=" +
randomPortSecond)
- .withFixedExposedPort(randomPortSecond,
randomPortSecond)
- .dependsOn(ZOOKEEPER_CONTAINER)) {
- hiveServer2SecondContainer.start();
-
extracted(hiveServer2SecondContainer.getMappedPort(randomPortSecond));
- testShardingService.processSuccessInHive();
- }
}
private Connection openConnection() throws SQLException {
Properties props = new Properties();
- return DriverManager.getConnection(jdbcUrlPrefix + jdbcUrlSuffix,
props);
+ return DriverManager.getConnection(jdbcUrlPrefix, props);
}
private DataSource createDataSource() throws SQLException {
- extracted(HIVE_SERVER2_1_CONTAINER.getMappedPort(RANDOM_PORT_FIRST));
- HikariConfig config = new HikariConfig();
-
config.setDriverClassName("org.apache.shardingsphere.driver.ShardingSphereDriver");
-
config.setJdbcUrl("jdbc:shardingsphere:classpath:test-native/yaml/jdbc/databases/hive/zsd.yaml?placeholder-type=system_props");
- System.setProperty(SYSTEM_PROP_KEY_PREFIX + "ds0.jdbc-url",
jdbcUrlPrefix + "demo_ds_0" + ";initFile=" + ABSOLUTE_PATH + jdbcUrlSuffix);
- System.setProperty(SYSTEM_PROP_KEY_PREFIX + "ds1.jdbc-url",
jdbcUrlPrefix + "demo_ds_1" + ";initFile=" + ABSOLUTE_PATH + jdbcUrlSuffix);
- System.setProperty(SYSTEM_PROP_KEY_PREFIX + "ds2.jdbc-url",
jdbcUrlPrefix + "demo_ds_2" + ";initFile=" + ABSOLUTE_PATH + jdbcUrlSuffix);
- return new HikariDataSource(config);
- }
-
- private void extracted(final int hiveServer2Port) throws SQLException {
- String connectionString = ZOOKEEPER_CONTAINER.getHost() + ":" +
ZOOKEEPER_CONTAINER.getMappedPort(2181);
-
Awaitility.await().atMost(Duration.ofMinutes(2L)).ignoreExceptions().until(()
-> {
- try (
- CuratorFramework client =
CuratorFrameworkFactory.builder().connectString(connectionString)
- .retryPolicy(new ExponentialBackoffRetry(1000,
3)).build()) {
- client.start();
- List<String> children =
client.getChildren().forPath("/hiveserver2");
- assertThat(children.size(), is(1));
- return children.get(0).startsWith("serverUri=0.0.0.0:" +
hiveServer2Port + ";version=4.0.1;sequence=");
- }
- });
Awaitility.await().atMost(Duration.ofMinutes(1L)).ignoreExceptions().until(()
-> {
openConnection().close();
return true;
@@ -173,5 +111,12 @@ class ZookeeperServiceDiscoveryTest {
statement.executeUpdate("CREATE DATABASE demo_ds_1");
statement.executeUpdate("CREATE DATABASE demo_ds_2");
}
+ HikariConfig config = new HikariConfig();
+
config.setDriverClassName("org.apache.shardingsphere.driver.ShardingSphereDriver");
+
config.setJdbcUrl("jdbc:shardingsphere:classpath:test-native/yaml/jdbc/databases/hive/standalone-hms.yaml?placeholder-type=system_props");
+ System.setProperty(SYSTEM_PROP_KEY_PREFIX + "ds0.jdbc-url",
jdbcUrlPrefix + "demo_ds_0" + ";initFile=" + ABSOLUTE_PATH);
+ System.setProperty(SYSTEM_PROP_KEY_PREFIX + "ds1.jdbc-url",
jdbcUrlPrefix + "demo_ds_1" + ";initFile=" + ABSOLUTE_PATH);
+ System.setProperty(SYSTEM_PROP_KEY_PREFIX + "ds2.jdbc-url",
jdbcUrlPrefix + "demo_ds_2" + ";initFile=" + ABSOLUTE_PATH);
+ return new HikariDataSource(config);
}
}
diff --git
a/test/native/src/test/java/org/apache/shardingsphere/test/natived/jdbc/databases/hive/ZookeeperServiceDiscoveryTest.java
b/test/native/src/test/java/org/apache/shardingsphere/test/natived/jdbc/databases/hive/ZookeeperServiceDiscoveryTest.java
index ee9effbbc38..9bcaf48dc3a 100644
---
a/test/native/src/test/java/org/apache/shardingsphere/test/natived/jdbc/databases/hive/ZookeeperServiceDiscoveryTest.java
+++
b/test/native/src/test/java/org/apache/shardingsphere/test/natived/jdbc/databases/hive/ZookeeperServiceDiscoveryTest.java
@@ -71,7 +71,7 @@ class ZookeeperServiceDiscoveryTest {
*/
@SuppressWarnings("unused")
@Container
- private static final GenericContainer<?> HIVE_SERVER2_1_CONTAINER = new
FixedHostPortGenericContainer<>("apache/hive:4.0.1")
+ private static final GenericContainer<?> HS2_1_CONTAINER = new
FixedHostPortGenericContainer<>("apache/hive:4.0.1")
.withNetwork(NETWORK)
.withEnv("SERVICE_NAME", "hiveserver2")
.withEnv("SERVICE_OPTS",
"-Dhive.server2.support.dynamic.service.discovery=true" + " "
@@ -116,10 +116,10 @@ class ZookeeperServiceDiscoveryTest {
DataSource dataSource = createDataSource();
TestShardingService testShardingService = new
TestShardingService(dataSource);
testShardingService.processSuccessInHive();
- HIVE_SERVER2_1_CONTAINER.stop();
+ HS2_1_CONTAINER.stop();
int randomPortSecond = InstanceSpec.getRandomPort();
try (
- GenericContainer<?> hiveServer2SecondContainer = new
FixedHostPortGenericContainer<>("apache/hive:4.0.1")
+ GenericContainer<?> hs2SecondContainer = new
FixedHostPortGenericContainer<>("apache/hive:4.0.1")
.withNetwork(NETWORK)
.withEnv("SERVICE_NAME", "hiveserver2")
.withEnv("SERVICE_OPTS",
"-Dhive.server2.support.dynamic.service.discovery=true" + " "
@@ -128,8 +128,8 @@ class ZookeeperServiceDiscoveryTest {
+ "-Dhive.server2.thrift.port=" +
randomPortSecond)
.withFixedExposedPort(randomPortSecond,
randomPortSecond)
.dependsOn(ZOOKEEPER_CONTAINER)) {
- hiveServer2SecondContainer.start();
-
extracted(hiveServer2SecondContainer.getMappedPort(randomPortSecond));
+ hs2SecondContainer.start();
+ extracted(hs2SecondContainer.getMappedPort(randomPortSecond));
testShardingService.processSuccessInHive();
}
}
@@ -140,7 +140,7 @@ class ZookeeperServiceDiscoveryTest {
}
private DataSource createDataSource() throws SQLException {
- extracted(HIVE_SERVER2_1_CONTAINER.getMappedPort(RANDOM_PORT_FIRST));
+ extracted(HS2_1_CONTAINER.getMappedPort(RANDOM_PORT_FIRST));
HikariConfig config = new HikariConfig();
config.setDriverClassName("org.apache.shardingsphere.driver.ShardingSphereDriver");
config.setJdbcUrl("jdbc:shardingsphere:classpath:test-native/yaml/jdbc/databases/hive/zsd.yaml?placeholder-type=system_props");
diff --git
a/test/native/src/test/resources/META-INF/native-image/shardingsphere-test-native-test-metadata/reflect-config.json
b/test/native/src/test/resources/META-INF/native-image/shardingsphere-test-native-test-metadata/reflect-config.json
index ca72894c15a..44c372fa4dd 100644
---
a/test/native/src/test/resources/META-INF/native-image/shardingsphere-test-native-test-metadata/reflect-config.json
+++
b/test/native/src/test/resources/META-INF/native-image/shardingsphere-test-native-test-metadata/reflect-config.json
@@ -168,5 +168,13 @@
"allDeclaredConstructors": true,
"allDeclaredMethods": true,
"allPublicMethods": true
+},
+{
+
"condition":{"typeReachable":"org.apache.shardingsphere.test.natived.jdbc.databases.hive.StandaloneMetastoreTest"},
+
"name":"org.apache.shardingsphere.test.natived.jdbc.databases.hive.StandaloneMetastoreTest",
+ "allDeclaredFields": true,
+ "allDeclaredConstructors": true,
+ "allDeclaredMethods": true,
+ "allPublicMethods": true
}
]
diff --git
a/test/native/src/test/resources/test-native/yaml/jdbc/databases/hive/standalone-hms.yaml
b/test/native/src/test/resources/test-native/yaml/jdbc/databases/hive/standalone-hms.yaml
new file mode 100644
index 00000000000..4ee01badefc
--- /dev/null
+++
b/test/native/src/test/resources/test-native/yaml/jdbc/databases/hive/standalone-hms.yaml
@@ -0,0 +1,64 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+dataSources:
+ ds_0:
+ dataSourceClassName: com.zaxxer.hikari.HikariDataSource
+ driverClassName: org.apache.hive.jdbc.HiveDriver
+ jdbcUrl: $${fixture.test-native.yaml.database.hive.hms.ds0.jdbc-url::}
+ ds_1:
+ dataSourceClassName: com.zaxxer.hikari.HikariDataSource
+ driverClassName: org.apache.hive.jdbc.HiveDriver
+ jdbcUrl: $${fixture.test-native.yaml.database.hive.hms.ds1.jdbc-url::}
+ ds_2:
+ dataSourceClassName: com.zaxxer.hikari.HikariDataSource
+ driverClassName: org.apache.hive.jdbc.HiveDriver
+ jdbcUrl: $${fixture.test-native.yaml.database.hive.hms.ds2.jdbc-url::}
+
+rules:
+- !SHARDING
+ tables:
+ t_order:
+ actualDataNodes: <LITERAL>ds_0.t_order, ds_1.t_order, ds_2.t_order
+ keyGenerateStrategy:
+ column: order_id
+ keyGeneratorName: snowflake
+ t_order_item:
+ actualDataNodes: <LITERAL>ds_0.t_order_item, ds_1.t_order_item,
ds_2.t_order_item
+ keyGenerateStrategy:
+ column: order_item_id
+ keyGeneratorName: snowflake
+ defaultDatabaseStrategy:
+ standard:
+ shardingColumn: user_id
+ shardingAlgorithmName: inline
+ shardingAlgorithms:
+ inline:
+ type: CLASS_BASED
+ props:
+ strategy: STANDARD
+ algorithmClassName:
org.apache.shardingsphere.test.natived.commons.algorithm.ClassBasedInlineShardingAlgorithmFixture
+ keyGenerators:
+ snowflake:
+ type: SNOWFLAKE
+ auditors:
+ sharding_key_required_auditor:
+ type: DML_SHARDING_CONDITIONS
+
+- !BROADCAST
+ tables:
+ - t_address
