This is an automated email from the ASF dual-hosted git repository.
yuqi4733 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gravitino.git
The following commit(s) were added to refs/heads/main by this push:
new b8245a1edf [#8933][#8959] fix(lance-rest-server): fix lance rest
server issues (#9186)
b8245a1edf is described below
commit b8245a1edfc0f0fb7d566b931aa2386afd4dfbcc
Author: Junda Yang <[email protected]>
AuthorDate: Thu Nov 20 03:01:02 2025 -0800
[#8933][#8959] fix(lance-rest-server): fix lance rest server issues (#9186)
### What changes were proposed in this pull request?
This PR combines two bug fixes for Lance REST server integration:
CI Configuration Fix:
- Renamed `ignoreIcebergRestService` to `ignoreAuxRestService` to
support both Iceberg and Lance auxiliary services
- Added Lance REST service to `gravitino.auxService.names` in config
template
- Added `:lance:lance-rest-server:jar` as test dependency in Iceberg
catalog, Spark connector (v3.3, v3.4, v3.5) build scripts
- Implemented `getLanceRestServiceConfigs()` in MiniGravitino to
auto-configure Lance REST service for integration tests
Configuration Key Fix:
- Removed excessive `LANCE_CONFIG_PREFIX` from `METALAKE_NAME` and
`NAMESPACE_BACKEND_URI` configuration entries
- Changed keys from `gravitino.lance-rest.gravitino.metalake-name` →
`gravitino.metalake-name` and `gravitino.lance-rest.gravitino.uri` →
`gravitino.uri`
### Why are the changes needed?
CI Fix: Fixing IT tests. The naming ignoreIcebergRestService was also
confusing since it should control all auxiliary services, not just
Iceberg.
Config Fix: The Lance REST server strips the `LANCE_CONFIG_PREFIX` when
starting, so having it in the configuration key definitions caused a
mismatch where configurations couldn't be properly loaded (e.g., the
server would look for `gravitino.metalake-name` but the config defined
`gravitino.lance-rest.gravitino.metalake-name`).
Fix: #8933
Fix: #8959
### Does this PR introduce _any_ user-facing change?
No to user-facing API changes, but yes to configuration changes
### How was this patch tested?
CI
---------
Co-authored-by: Jerry Shao <[email protected]>
Co-authored-by: Mini Yu <[email protected]>
---
.../catalog-lakehouse-iceberg/build.gradle.kts | 2 ++
.../integration/test/CatalogIcebergBaseIT.java | 2 +-
.../test/CatalogIcebergKerberosHiveIT.java | 2 +-
conf/gravitino.conf.template | 2 +-
.../connector/integration/test/FlinkEnvIT.java | 2 +-
.../gravitino/integration/test/MiniGravitino.java | 25 ++++++++++++++++++----
.../integration/test/MiniGravitinoContext.java | 6 +++---
.../gravitino/integration/test/util/BaseIT.java | 5 ++---
.../gravitino/lance/common/config/LanceConfig.java | 4 ++--
.../connector/integration/test/SparkEnvIT.java | 2 +-
spark-connector/v3.3/spark/build.gradle.kts | 1 +
spark-connector/v3.4/spark/build.gradle.kts | 1 +
spark-connector/v3.5/spark/build.gradle.kts | 1 +
13 files changed, 38 insertions(+), 17 deletions(-)
diff --git a/catalogs/catalog-lakehouse-iceberg/build.gradle.kts
b/catalogs/catalog-lakehouse-iceberg/build.gradle.kts
index e61649bb69..4a143321f9 100644
--- a/catalogs/catalog-lakehouse-iceberg/build.gradle.kts
+++ b/catalogs/catalog-lakehouse-iceberg/build.gradle.kts
@@ -148,6 +148,8 @@ tasks.test {
exclude("**/integration/test/**")
} else {
dependsOn(tasks.jar)
+ dependsOn(":iceberg:iceberg-rest-server:jar")
+ dependsOn(":lance:lance-rest-server:jar")
}
}
diff --git
a/catalogs/catalog-lakehouse-iceberg/src/test/java/org/apache/gravitino/catalog/lakehouse/iceberg/integration/test/CatalogIcebergBaseIT.java
b/catalogs/catalog-lakehouse-iceberg/src/test/java/org/apache/gravitino/catalog/lakehouse/iceberg/integration/test/CatalogIcebergBaseIT.java
index 1260d7d795..b0d4f3b5eb 100644
---
a/catalogs/catalog-lakehouse-iceberg/src/test/java/org/apache/gravitino/catalog/lakehouse/iceberg/integration/test/CatalogIcebergBaseIT.java
+++
b/catalogs/catalog-lakehouse-iceberg/src/test/java/org/apache/gravitino/catalog/lakehouse/iceberg/integration/test/CatalogIcebergBaseIT.java
@@ -124,7 +124,7 @@ public abstract class CatalogIcebergBaseIT extends BaseIT {
@BeforeAll
public void startup() throws Exception {
- ignoreIcebergRestService = false;
+ super.ignoreAuxRestService = false;
super.startIntegrationTest();
containerSuite.startHiveContainer();
initIcebergCatalogProperties();
diff --git
a/catalogs/catalog-lakehouse-iceberg/src/test/java/org/apache/gravitino/catalog/lakehouse/iceberg/integration/test/CatalogIcebergKerberosHiveIT.java
b/catalogs/catalog-lakehouse-iceberg/src/test/java/org/apache/gravitino/catalog/lakehouse/iceberg/integration/test/CatalogIcebergKerberosHiveIT.java
index 1017ccb4e1..cce787ec85 100644
---
a/catalogs/catalog-lakehouse-iceberg/src/test/java/org/apache/gravitino/catalog/lakehouse/iceberg/integration/test/CatalogIcebergKerberosHiveIT.java
+++
b/catalogs/catalog-lakehouse-iceberg/src/test/java/org/apache/gravitino/catalog/lakehouse/iceberg/integration/test/CatalogIcebergKerberosHiveIT.java
@@ -127,7 +127,7 @@ public class CatalogIcebergKerberosHiveIT extends BaseIT {
// Config kerberos configuration for Gravitino server
addKerberosConfig();
- ignoreIcebergRestService = false;
+ super.ignoreAuxRestService = false;
// Start Gravitino server
super.startIntegrationTest();
} catch (Exception e) {
diff --git a/conf/gravitino.conf.template b/conf/gravitino.conf.template
index 44a9e50f82..f3af89c12b 100644
--- a/conf/gravitino.conf.template
+++ b/conf/gravitino.conf.template
@@ -82,7 +82,7 @@ gravitino.authorization.serviceAdmins = anonymous
# THE CONFIGURATION FOR AUXILIARY SERVICE
# Auxiliary service names, separate by ',', currently support iceberg-rest and
lance-rest
-gravitino.auxService.names = iceberg-rest
+gravitino.auxService.names = iceberg-rest,lance-rest
# Iceberg REST service classpath
gravitino.iceberg-rest.classpath = iceberg-rest-server/libs,
iceberg-rest-server/conf
# Iceberg REST service host
diff --git
a/flink-connector/flink/src/test/java/org/apache/gravitino/flink/connector/integration/test/FlinkEnvIT.java
b/flink-connector/flink/src/test/java/org/apache/gravitino/flink/connector/integration/test/FlinkEnvIT.java
index d372eb92af..ca79144f5b 100644
---
a/flink-connector/flink/src/test/java/org/apache/gravitino/flink/connector/integration/test/FlinkEnvIT.java
+++
b/flink-connector/flink/src/test/java/org/apache/gravitino/flink/connector/integration/test/FlinkEnvIT.java
@@ -112,7 +112,7 @@ public abstract class FlinkEnvIT extends BaseIT {
protected abstract String getProvider();
private void initIcebergRestServiceEnv() {
- ignoreIcebergRestService = false;
+ super.ignoreAuxRestService = false;
Map<String, String> icebergRestServiceConfigs = new HashMap<>();
icebergRestServiceConfigs.put(
"gravitino."
diff --git
a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/MiniGravitino.java
b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/MiniGravitino.java
index d0ce1710d7..3cff6940c6 100644
---
a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/MiniGravitino.java
+++
b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/MiniGravitino.java
@@ -78,7 +78,7 @@ public class MiniGravitino {
mockConfDir.mkdirs();
}
- private void removeIcebergRestConfiguration(Properties properties) {
+ private void removeAuxRestConfiguration(Properties properties) {
// Disable Iceberg REST service
properties.remove(
AuxiliaryServiceManager.GRAVITINO_AUX_SERVICE_PREFIX
@@ -103,9 +103,9 @@ public class MiniGravitino {
serverConfig.loadPropertiesFromFile(
new File(ITUtils.joinPath(mockConfDir.getAbsolutePath(),
"gravitino.conf")));
- // Remove Iceberg rest service.
- if (context.ignoreIcebergRestService) {
- removeIcebergRestConfiguration(properties);
+ // Disable auxiliary rest service.
+ if (context.ignoreAuxRestService) {
+ removeAuxRestConfiguration(properties);
ITUtils.overwriteConfigFile(
ITUtils.joinPath(mockConfDir.getAbsolutePath(), "gravitino.conf"),
properties);
}
@@ -230,6 +230,22 @@ public class MiniGravitino {
return customConfigs;
}
+ private Map<String, String> getLanceRestServiceConfigs() throws IOException {
+ Map<String, String> customConfigs = new HashMap<>();
+
+ String lanceJarPath = Paths.get("lance", "lance-rest-server", "build",
"libs").toString();
+ String lanceConfigPath =
+ Paths.get("lance", "lance-rest-server", "src", "main",
"resources").toString();
+ customConfigs.put(
+ "gravitino.lance-rest." +
AuxiliaryServiceManager.AUX_SERVICE_CLASSPATH,
+ String.join(",", lanceJarPath, lanceConfigPath));
+
+ customConfigs.put(
+ "gravitino.lance-rest." +
JettyServerConfig.WEBSERVER_HTTP_PORT.getKey(),
+ String.valueOf(RESTUtils.findAvailablePort(4000, 5000)));
+ return customConfigs;
+ }
+
// Customize the config file
private void customizeConfigFile(String configTempFileName, String
configFileName)
throws IOException {
@@ -239,6 +255,7 @@ public class MiniGravitino {
String.valueOf(RESTUtils.findAvailablePort(2000, 3000)));
configMap.putAll(getIcebergRestServiceConfigs());
+ configMap.putAll(getLanceRestServiceConfigs());
configMap.putAll(context.customConfig);
ITUtils.rewriteConfigFile(configTempFileName, configFileName, configMap);
diff --git
a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/MiniGravitinoContext.java
b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/MiniGravitinoContext.java
index 5b2acfac33..601138ecfd 100644
---
a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/MiniGravitinoContext.java
+++
b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/MiniGravitinoContext.java
@@ -23,10 +23,10 @@ import java.util.Map;
public class MiniGravitinoContext {
Map<String, String> customConfig;
- final boolean ignoreIcebergRestService;
+ final boolean ignoreAuxRestService;
- public MiniGravitinoContext(Map<String, String> customConfig, boolean
ignoreIcebergRestService) {
+ public MiniGravitinoContext(Map<String, String> customConfig, boolean
ignoreAuxRestService) {
this.customConfig = customConfig;
- this.ignoreIcebergRestService = ignoreIcebergRestService;
+ this.ignoreAuxRestService = ignoreAuxRestService;
}
}
diff --git
a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/util/BaseIT.java
b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/util/BaseIT.java
index 72923341b9..1ae60a0fec 100644
---
a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/util/BaseIT.java
+++
b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/util/BaseIT.java
@@ -102,7 +102,7 @@ public class BaseIT {
protected Map<String, String> customConfigs = new HashMap<>();
- protected boolean ignoreIcebergRestService = true;
+ protected boolean ignoreAuxRestService = true;
public String DOWNLOAD_MYSQL_JDBC_DRIVER_URL =
"https://repo1.maven.org/maven2/mysql/mysql-connector-java/8.0.26/mysql-connector-java-8.0.26.jar";;
@@ -330,8 +330,7 @@ public class BaseIT {
serverConfig = new ServerConfig();
customConfigs.put(ENTITY_RELATIONAL_JDBC_BACKEND_PATH.getKey(),
file.getAbsolutePath());
if (testMode != null && testMode.equals(ITUtils.EMBEDDED_TEST_MODE)) {
- MiniGravitinoContext context =
- new MiniGravitinoContext(customConfigs, ignoreIcebergRestService);
+ MiniGravitinoContext context = new MiniGravitinoContext(customConfigs,
ignoreAuxRestService);
miniGravitino = new MiniGravitino(context);
miniGravitino.start();
serverConfig = miniGravitino.getServerConfig();
diff --git
a/lance/lance-common/src/main/java/org/apache/gravitino/lance/common/config/LanceConfig.java
b/lance/lance-common/src/main/java/org/apache/gravitino/lance/common/config/LanceConfig.java
index f517d1a349..f4556e43e4 100644
---
a/lance/lance-common/src/main/java/org/apache/gravitino/lance/common/config/LanceConfig.java
+++
b/lance/lance-common/src/main/java/org/apache/gravitino/lance/common/config/LanceConfig.java
@@ -47,14 +47,14 @@ public class LanceConfig extends Config implements
OverwriteDefaultConfig {
.createWithDefault(GRAVITINO_NAMESPACE_BACKEND);
public static final ConfigEntry<String> METALAKE_NAME =
- new ConfigBuilder(LANCE_CONFIG_PREFIX + GRAVITINO_NAMESPACE_BACKEND +
"." + CONFIG_METALAKE)
+ new ConfigBuilder(GRAVITINO_NAMESPACE_BACKEND + "." + CONFIG_METALAKE)
.doc("The Metalake name for Lance Gravitino namespace backend")
.version(ConfigConstants.VERSION_0_1_0)
.stringConf()
.create();
public static final ConfigEntry<String> NAMESPACE_BACKEND_URI =
- new ConfigBuilder(LANCE_CONFIG_PREFIX + GRAVITINO_NAMESPACE_BACKEND +
"." + CONFIG_URI)
+ new ConfigBuilder(GRAVITINO_NAMESPACE_BACKEND + "." + CONFIG_URI)
.doc("The URI of the namespace backend, e.g., Gravitino server URI")
.version(ConfigConstants.VERSION_0_1_0)
.stringConf()
diff --git
a/spark-connector/spark-common/src/test/java/org/apache/gravitino/spark/connector/integration/test/SparkEnvIT.java
b/spark-connector/spark-common/src/test/java/org/apache/gravitino/spark/connector/integration/test/SparkEnvIT.java
index 273d6d3bcd..9c2d2d5110 100644
---
a/spark-connector/spark-common/src/test/java/org/apache/gravitino/spark/connector/integration/test/SparkEnvIT.java
+++
b/spark-connector/spark-common/src/test/java/org/apache/gravitino/spark/connector/integration/test/SparkEnvIT.java
@@ -155,7 +155,7 @@ public abstract class SparkEnvIT extends SparkUtilIT {
protected void initCatalogEnv() throws Exception {}
private void initIcebergRestServiceEnv() {
- ignoreIcebergRestService = false;
+ super.ignoreAuxRestService = false;
Map<String, String> icebergRestServiceConfigs = new HashMap<>();
icebergRestServiceConfigs.put(
"gravitino."
diff --git a/spark-connector/v3.3/spark/build.gradle.kts
b/spark-connector/v3.3/spark/build.gradle.kts
index 9023404980..1cc7f6d1a1 100644
--- a/spark-connector/v3.3/spark/build.gradle.kts
+++ b/spark-connector/v3.3/spark/build.gradle.kts
@@ -175,6 +175,7 @@ tasks.test {
dependsOn(":catalogs:catalog-lakehouse-iceberg:jar")
dependsOn(":catalogs:catalog-hive:jar")
dependsOn(":iceberg:iceberg-rest-server:jar")
+ dependsOn(":lance:lance-rest-server:jar")
dependsOn(":catalogs:catalog-lakehouse-paimon:jar")
dependsOn(":catalogs:catalog-jdbc-mysql:jar")
dependsOn(":catalogs:catalog-jdbc-postgresql:jar")
diff --git a/spark-connector/v3.4/spark/build.gradle.kts
b/spark-connector/v3.4/spark/build.gradle.kts
index 8a9b527dce..519799d076 100644
--- a/spark-connector/v3.4/spark/build.gradle.kts
+++ b/spark-connector/v3.4/spark/build.gradle.kts
@@ -175,6 +175,7 @@ tasks.test {
dependsOn(":catalogs:catalog-lakehouse-iceberg:jar")
dependsOn(":catalogs:catalog-hive:jar")
dependsOn(":iceberg:iceberg-rest-server:jar")
+ dependsOn(":lance:lance-rest-server:jar")
dependsOn(":catalogs:catalog-lakehouse-paimon:jar")
dependsOn(":catalogs:catalog-jdbc-mysql:jar")
dependsOn(":catalogs:catalog-jdbc-postgresql:jar")
diff --git a/spark-connector/v3.5/spark/build.gradle.kts
b/spark-connector/v3.5/spark/build.gradle.kts
index edbed1ca2d..cc1d8c9f88 100644
--- a/spark-connector/v3.5/spark/build.gradle.kts
+++ b/spark-connector/v3.5/spark/build.gradle.kts
@@ -177,6 +177,7 @@ tasks.test {
dependsOn(":catalogs:catalog-lakehouse-iceberg:jar")
dependsOn(":catalogs:catalog-hive:jar")
dependsOn(":iceberg:iceberg-rest-server:jar")
+ dependsOn(":lance:lance-rest-server:jar")
dependsOn(":catalogs:catalog-lakehouse-paimon:jar")
dependsOn(":catalogs:catalog-jdbc-mysql:jar")
dependsOn(":catalogs:catalog-jdbc-postgresql:jar")
