This is an automated email from the ASF dual-hosted git repository.
czy006 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/amoro.git
The following commit(s) were added to refs/heads/master by this push:
new d1dca2749 [AMORO-3837] Integrate lance format (#4051)
d1dca2749 is described below
commit d1dca2749a5c36a37c450734b6e826f07f272929
Author: nathan.ma <[email protected]>
AuthorDate: Thu Jun 25 14:54:58 2026 +0800
[AMORO-3837] Integrate lance format (#4051)
* Add Lance format support
* Align Lance Arrow dependencies
* Fix Lance CI dependency checks
* Fix catalog refresh properties
* Restore catalog refresh behavior
* Fix Lance catalog exception handling
* Avoid exposing Lance catalog credentials
* Fix Lance catalog storage handling
* Fix Lance OSS endpoint catalog config
* Fix Lance database discovery semantics
* Fix Lance load table fallback semantics
---------
Co-authored-by: majin.nathan <majin.nathan@bytedance>
Co-authored-by: ConradJam <[email protected]>
---
amoro-ams/pom.xml | 36 +++
.../amoro/server/catalog/CatalogBuilder.java | 12 +-
.../dashboard/controller/CatalogController.java | 20 +-
.../org/apache/amoro/CommonUnifiedCatalog.java | 6 +-
.../main/java/org/apache/amoro/TableFormat.java | 1 +
amoro-format-lance/pom.xml | 84 ++++++
.../amoro/formats/lance/LanceCatalogFactory.java | 58 ++++
.../formats/lance/LanceDirectoryV1Catalog.java | 215 ++++++++++++++
.../org/apache/amoro/formats/lance/LanceTable.java | 71 +++++
.../amoro/formats/lance/LanceTableDescriptor.java | 329 +++++++++++++++++++++
.../services/org.apache.amoro.FormatCatalogFactory | 19 ++
...he.amoro.table.descriptor.FormatTableDescriptor | 18 ++
amoro-web/src/assets/icons/svg/lance.svg | 25 ++
amoro-web/src/types/common.type.ts | 1 +
dev/deps/dependencies-hadoop-2-spark-3.3 | 13 +-
dev/deps/dependencies-hadoop-3-spark-3.5 | 13 +-
pom.xml | 16 +
17 files changed, 917 insertions(+), 20 deletions(-)
diff --git a/amoro-ams/pom.xml b/amoro-ams/pom.xml
index 34f5591c7..af6fa6bb6 100644
--- a/amoro-ams/pom.xml
+++ b/amoro-ams/pom.xml
@@ -421,6 +421,13 @@
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>org.apache.amoro</groupId>
+ <artifactId>amoro-format-lance</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
+
<dependency>
<groupId>org.apache.curator</groupId>
<artifactId>curator-test</artifactId>
@@ -625,6 +632,31 @@
</dependency>
</dependencies>
</profile>
+ <profile>
+ <id>support-lance-format</id>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.amoro</groupId>
+ <artifactId>amoro-format-lance</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.arrow</groupId>
+ <artifactId>arrow-vector</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.arrow</groupId>
+ <artifactId>arrow-memory-netty</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.arrow</groupId>
+ <artifactId>arrow-c-data</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.arrow</groupId>
+ <artifactId>arrow-dataset</artifactId>
+ </dependency>
+ </dependencies>
+ </profile>
<profile>
<id>support-all-formats</id>
<dependencies>
@@ -636,6 +668,10 @@
<groupId>org.apache.amoro</groupId>
<artifactId>amoro-format-hudi</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.apache.amoro</groupId>
+ <artifactId>amoro-format-lance</artifactId>
+ </dependency>
</dependencies>
</profile>
<profile>
diff --git
a/amoro-ams/src/main/java/org/apache/amoro/server/catalog/CatalogBuilder.java
b/amoro-ams/src/main/java/org/apache/amoro/server/catalog/CatalogBuilder.java
index 31c8240ca..af39ba533 100644
---
a/amoro-ams/src/main/java/org/apache/amoro/server/catalog/CatalogBuilder.java
+++
b/amoro-ams/src/main/java/org/apache/amoro/server/catalog/CatalogBuilder.java
@@ -44,10 +44,18 @@ public class CatalogBuilder {
ImmutableMap.of(
CATALOG_TYPE_HADOOP,
Sets.newHashSet(
- TableFormat.ICEBERG, TableFormat.MIXED_ICEBERG,
TableFormat.PAIMON, TableFormat.HUDI),
+ TableFormat.ICEBERG,
+ TableFormat.MIXED_ICEBERG,
+ TableFormat.PAIMON,
+ TableFormat.HUDI,
+ TableFormat.LANCE),
CATALOG_TYPE_FILESYSTEM,
Sets.newHashSet(
- TableFormat.ICEBERG, TableFormat.MIXED_ICEBERG,
TableFormat.PAIMON, TableFormat.HUDI),
+ TableFormat.ICEBERG,
+ TableFormat.MIXED_ICEBERG,
+ TableFormat.PAIMON,
+ TableFormat.HUDI,
+ TableFormat.LANCE),
CATALOG_TYPE_GLUE,
Sets.newHashSet(TableFormat.ICEBERG, TableFormat.MIXED_ICEBERG),
CATALOG_TYPE_REST,
diff --git
a/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/controller/CatalogController.java
b/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/controller/CatalogController.java
index d688f9e29..172b2a0c8 100644
---
a/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/controller/CatalogController.java
+++
b/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/controller/CatalogController.java
@@ -20,6 +20,7 @@ package org.apache.amoro.server.dashboard.controller;
import static org.apache.amoro.TableFormat.HUDI;
import static org.apache.amoro.TableFormat.ICEBERG;
+import static org.apache.amoro.TableFormat.LANCE;
import static org.apache.amoro.TableFormat.MIXED_HIVE;
import static org.apache.amoro.TableFormat.MIXED_ICEBERG;
import static org.apache.amoro.TableFormat.PAIMON;
@@ -161,6 +162,18 @@ public class CatalogController {
CatalogDescriptor.of(CATALOG_TYPE_FILESYSTEM,
STORAGE_CONFIGS_VALUE_TYPE_HADOOP, PAIMON));
VALIDATE_CATALOGS.add(
CatalogDescriptor.of(CATALOG_TYPE_FILESYSTEM,
STORAGE_CONFIGS_VALUE_TYPE_S3, PAIMON));
+ VALIDATE_CATALOGS.add(
+ CatalogDescriptor.of(CATALOG_TYPE_FILESYSTEM,
STORAGE_CONFIGS_VALUE_TYPE_S3, LANCE));
+ VALIDATE_CATALOGS.add(
+ CatalogDescriptor.of(CATALOG_TYPE_FILESYSTEM,
STORAGE_CONFIGS_VALUE_TYPE_OSS, LANCE));
+ VALIDATE_CATALOGS.add(
+ CatalogDescriptor.of(CATALOG_TYPE_FILESYSTEM,
STORAGE_CONFIGS_VALUE_TYPE_LOCAL, LANCE));
+ VALIDATE_CATALOGS.add(
+ CatalogDescriptor.of(CATALOG_TYPE_HADOOP,
STORAGE_CONFIGS_VALUE_TYPE_LOCAL, LANCE));
+ VALIDATE_CATALOGS.add(
+ CatalogDescriptor.of(CATALOG_TYPE_HADOOP,
STORAGE_CONFIGS_VALUE_TYPE_S3, LANCE));
+ VALIDATE_CATALOGS.add(
+ CatalogDescriptor.of(CATALOG_TYPE_HADOOP,
STORAGE_CONFIGS_VALUE_TYPE_OSS, LANCE));
VALIDATE_CATALOGS.add(
CatalogDescriptor.of(CATALOG_TYPE_GLUE, STORAGE_CONFIGS_VALUE_TYPE_S3,
ICEBERG));
VALIDATE_CATALOGS.add(
@@ -427,7 +440,9 @@ public class CatalogController {
S3FileIOProperties.ENDPOINT,
STORAGE_CONFIGS_KEY_S3_ENDPOINT);
} else if (STORAGE_CONFIGS_VALUE_TYPE_OSS.equals(storageType)) {
- if (tableFormats.contains(ICEBERG.name()) ||
tableFormats.contains(MIXED_ICEBERG.name())) {
+ if (tableFormats.contains(ICEBERG.name())
+ || tableFormats.contains(MIXED_ICEBERG.name())
+ || tableFormats.contains(LANCE.name())) {
CatalogUtil.copyProperty(
catalogMeta.getCatalogProperties(),
storageConfig,
@@ -523,7 +538,8 @@ public class CatalogController {
S3FileIOProperties.ENDPOINT);
} else if (storageType.equals(STORAGE_CONFIGS_VALUE_TYPE_OSS)) {
if (info.getTableFormatList().contains(ICEBERG.name())
- || info.getTableFormatList().contains(MIXED_ICEBERG.name())) {
+ || info.getTableFormatList().contains(MIXED_ICEBERG.name())
+ || info.getTableFormatList().contains(LANCE.name())) {
CatalogUtil.copyProperty(
info.getStorageConfig(),
catalogMeta.getCatalogProperties(),
diff --git
a/amoro-common/src/main/java/org/apache/amoro/CommonUnifiedCatalog.java
b/amoro-common/src/main/java/org/apache/amoro/CommonUnifiedCatalog.java
index b631c595f..281911c9a 100644
--- a/amoro-common/src/main/java/org/apache/amoro/CommonUnifiedCatalog.java
+++ b/amoro-common/src/main/java/org/apache/amoro/CommonUnifiedCatalog.java
@@ -120,7 +120,8 @@ public class CommonUnifiedCatalog implements UnifiedCatalog
{
TableFormat.MIXED_ICEBERG,
TableFormat.ICEBERG,
TableFormat.PAIMON,
- TableFormat.HUDI)
+ TableFormat.HUDI,
+ TableFormat.LANCE)
.map(
formatCatalog -> {
try {
@@ -147,7 +148,8 @@ public class CommonUnifiedCatalog implements UnifiedCatalog
{
TableFormat.MIXED_ICEBERG,
TableFormat.ICEBERG,
TableFormat.PAIMON,
- TableFormat.HUDI
+ TableFormat.HUDI,
+ TableFormat.LANCE
};
Map<String, TableFormat> tableNameToFormat = Maps.newHashMap();
diff --git a/amoro-common/src/main/java/org/apache/amoro/TableFormat.java
b/amoro-common/src/main/java/org/apache/amoro/TableFormat.java
index 62400267a..c1179bdad 100644
--- a/amoro-common/src/main/java/org/apache/amoro/TableFormat.java
+++ b/amoro-common/src/main/java/org/apache/amoro/TableFormat.java
@@ -42,6 +42,7 @@ public final class TableFormat implements Serializable {
/** Open-source table formats */
public static final TableFormat ICEBERG = register("ICEBERG");
+ public static final TableFormat LANCE = register("LANCE");
public static final TableFormat MIXED_ICEBERG = register("MIXED_ICEBERG");
public static final TableFormat MIXED_HIVE = register("MIXED_HIVE");
public static final TableFormat PAIMON = register("PAIMON");
diff --git a/amoro-format-lance/pom.xml b/amoro-format-lance/pom.xml
new file mode 100755
index 000000000..4d546422c
--- /dev/null
+++ b/amoro-format-lance/pom.xml
@@ -0,0 +1,84 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~ with the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+<project xmlns="http://maven.apache.org/POM/4.0.0";
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
http://maven.apache.org/xsd/maven-4.0.0.xsd";>
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.amoro</groupId>
+ <artifactId>amoro-parent</artifactId>
+ <version>0.9-SNAPSHOT</version>
+ <relativePath>../pom.xml</relativePath>
+ </parent>
+
+ <artifactId>amoro-format-lance</artifactId>
+ <name>Amoro Project Lance Format</name>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.amoro</groupId>
+ <artifactId>amoro-common</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.arrow</groupId>
+ <artifactId>arrow-memory-core</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.arrow</groupId>
+ <artifactId>arrow-vector</artifactId>
+ </dependency>
+
+ <!-- Lance format Java API -->
+ <dependency>
+ <groupId>org.lance</groupId>
+ <artifactId>lance-core</artifactId>
+ <version>7.0.0</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.lance</groupId>
+ <artifactId>lance-namespace-core</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.lance</groupId>
+ <artifactId>lance-namespace-apache-client</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <!-- Lance namespace interfaces and models -->
+ <dependency>
+ <groupId>org.lance</groupId>
+ <artifactId>lance-namespace-core</artifactId>
+ <version>0.8.0</version>
+ </dependency>
+ <dependency>
+ <groupId>org.lance</groupId>
+ <artifactId>lance-namespace-apache-client</artifactId>
+ <version>0.8.0</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.iceberg</groupId>
+ <artifactId>iceberg-aws</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.iceberg</groupId>
+ <artifactId>iceberg-aliyun</artifactId>
+ </dependency>
+ </dependencies>
+
+</project>
diff --git
a/amoro-format-lance/src/main/java/org/apache/amoro/formats/lance/LanceCatalogFactory.java
b/amoro-format-lance/src/main/java/org/apache/amoro/formats/lance/LanceCatalogFactory.java
new file mode 100755
index 000000000..a78fa1ae3
--- /dev/null
+++
b/amoro-format-lance/src/main/java/org/apache/amoro/formats/lance/LanceCatalogFactory.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.amoro.formats.lance;
+
+import org.apache.amoro.FormatCatalog;
+import org.apache.amoro.FormatCatalogFactory;
+import org.apache.amoro.TableFormat;
+import org.apache.amoro.table.TableMetaStore;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/** Lance format catalog factory. */
+public class LanceCatalogFactory implements FormatCatalogFactory {
+
+ /** TableFormat instance for Lance. */
+ public static final TableFormat LANCE = TableFormat.register("LANCE");
+
+ @Override
+ public FormatCatalog create(
+ String catalogName,
+ String metastoreType,
+ Map<String, String> properties,
+ TableMetaStore metaStore) {
+ return new LanceDirectoryV1Catalog(catalogName, properties);
+ }
+
+ @Override
+ public TableFormat format() {
+ return LANCE;
+ }
+
+ @Override
+ public Map<String, String> convertCatalogProperties(
+ String catalogName, String metastoreType, Map<String, String>
unifiedCatalogProperties) {
+ if (unifiedCatalogProperties == null) {
+ return null;
+ }
+
+ return new HashMap<>(unifiedCatalogProperties);
+ }
+}
diff --git
a/amoro-format-lance/src/main/java/org/apache/amoro/formats/lance/LanceDirectoryV1Catalog.java
b/amoro-format-lance/src/main/java/org/apache/amoro/formats/lance/LanceDirectoryV1Catalog.java
new file mode 100755
index 000000000..e522c3711
--- /dev/null
+++
b/amoro-format-lance/src/main/java/org/apache/amoro/formats/lance/LanceDirectoryV1Catalog.java
@@ -0,0 +1,215 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.amoro.formats.lance;
+
+import org.apache.amoro.AmoroTable;
+import org.apache.amoro.FormatCatalog;
+import org.apache.amoro.NoSuchDatabaseException;
+import org.apache.amoro.NoSuchTableException;
+import org.apache.amoro.properties.CatalogMetaProperties;
+import org.apache.amoro.table.TableIdentifier;
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.memory.RootAllocator;
+import org.apache.arrow.util.Preconditions;
+import org.apache.iceberg.aliyun.AliyunProperties;
+import org.apache.iceberg.aws.s3.S3FileIOProperties;
+import org.lance.Dataset;
+import org.lance.namespace.LanceNamespace;
+import org.lance.namespace.errors.TableNotFoundException;
+import org.lance.namespace.model.DropTableRequest;
+import org.lance.namespace.model.ListTablesRequest;
+import org.lance.namespace.model.ListTablesResponse;
+import org.lance.namespace.model.TableExistsRequest;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Local catalog implementation for Lance.
+ *
+ * <p>This catalog treats a single root directory as the backing "metastore".
Under the configured
+ * root directory, each immediate subdirectory whose name ends with ".lance"
is treated as a Lance
+ * dataset. All tables live in a single logical database named "default".
+ */
+public class LanceDirectoryV1Catalog implements FormatCatalog {
+
+ private static final String DEFAULT_DATABASE = "default";
+ private static final String STORAGE_ACCESS_KEY_ID = "storage.access_key_id";
+ private static final String STORAGE_SECRET_ACCESS_KEY =
"storage.secret_access_key";
+ private static final String STORAGE_ENDPOINT = "storage.endpoint";
+ private final String catalogName;
+ private final Map<String, String> namespaceProperties;
+ private final LanceNamespace namespace;
+
+ public LanceDirectoryV1Catalog(String catalogName, Map<String, String>
catalogProperties) {
+ Preconditions.checkArgument(
+ catalogProperties != null && !catalogProperties.isEmpty(),
+ "Catalog properties must be set.");
+ this.catalogName = catalogName;
+ this.namespaceProperties = new HashMap<>(catalogProperties);
+ String root =
namespaceProperties.remove(CatalogMetaProperties.KEY_WAREHOUSE);
+ String storageAccessKey =
+ removeFirstNonNull(
+ namespaceProperties,
+ S3FileIOProperties.ACCESS_KEY_ID,
+ AliyunProperties.CLIENT_ACCESS_KEY_ID);
+ String storageSecretKey =
+ removeFirstNonNull(
+ namespaceProperties,
+ S3FileIOProperties.SECRET_ACCESS_KEY,
+ AliyunProperties.CLIENT_ACCESS_KEY_SECRET);
+
+ Preconditions.checkArgument(
+ root != null && !root.isEmpty(), "Warehouse must be set in
catalogProperties.");
+ this.namespaceProperties.put("manifest_enabled", "false");
+ this.namespaceProperties.put("root", root);
+ if (storageAccessKey != null) {
+ this.namespaceProperties.put(STORAGE_ACCESS_KEY_ID, storageAccessKey);
+ }
+ if (storageSecretKey != null) {
+ this.namespaceProperties.put(STORAGE_SECRET_ACCESS_KEY,
storageSecretKey);
+ }
+ putStorageEndpointIfPresent(namespaceProperties);
+ this.namespace = initializeNamespace(new RootAllocator(Long.MAX_VALUE));
+ }
+
+ @Override
+ public List<String> listDatabases() {
+ return Collections.singletonList(DEFAULT_DATABASE);
+ }
+
+ @Override
+ public boolean databaseExists(String database) {
+ return DEFAULT_DATABASE.equals(database);
+ }
+
+ @Override
+ public boolean tableExists(String database, String table) {
+ if (!databaseExists(database)) {
+ return false;
+ }
+
+ try {
+ TableExistsRequest request = new
TableExistsRequest().id(Collections.singletonList(table));
+ namespace.tableExists(request);
+ return true;
+ } catch (TableNotFoundException e) {
+ return false;
+ }
+ }
+
+ @Override
+ public void createDatabase(String database) {
+ throw new UnsupportedOperationException("Creating Lance databases is not
supported.");
+ }
+
+ @Override
+ public void dropDatabase(String database) {
+ throw new UnsupportedOperationException("Dropping Lance databases is not
supported.");
+ }
+
+ @Override
+ public AmoroTable<?> loadTable(String database, String tableName) {
+ if (!databaseExists(database) || !tableExists(database, tableName)) {
+ throw new NoSuchTableException("Table: " + database + "." + tableName +
" does not exist");
+ }
+
+ TableIdentifier identifier = TableIdentifier.of(catalogName, database,
tableName);
+ Dataset dataset =
+ Dataset.open()
+ .namespaceClient(namespace)
+ .tableId(Collections.singletonList(tableName))
+ .build();
+ return new LanceTable(identifier, dataset, Collections.emptyMap());
+ }
+
+ @Override
+ public boolean dropTable(String database, String table, boolean purge) {
+ validateDatabase(database);
+
+ try {
+ namespace.dropTable(new
DropTableRequest().id(Collections.singletonList(table)));
+ return true;
+ } catch (TableNotFoundException e) {
+ return false;
+ }
+ }
+
+ @Override
+ public List<String> listTables(String database) {
+ if (!databaseExists(database)) {
+ return Collections.emptyList();
+ }
+ return listTablesFromNamespace();
+ }
+
+ private List<String> listTablesFromNamespace() {
+ if (namespace == null) {
+ return Collections.emptyList();
+ }
+
+ ListTablesRequest request = new
ListTablesRequest().id(Collections.emptyList());
+ ListTablesResponse response = namespace.listTables(request);
+ if (response == null) {
+ return Collections.emptyList();
+ } else {
+ response.getTables();
+ }
+
+ return new ArrayList<>(response.getTables());
+ }
+
+ private LanceNamespace initializeNamespace(BufferAllocator allocator) {
+ return LanceNamespace.connect("dir", namespaceProperties, allocator);
+ }
+
+ private static String removeFirstNonNull(Map<String, String> properties,
String... keys) {
+ String value = null;
+ for (String key : keys) {
+ String removed = properties.remove(key);
+ if (value == null && removed != null) {
+ value = removed;
+ }
+ }
+ return value;
+ }
+
+ private static void putStorageEndpointIfPresent(Map<String, String>
properties) {
+ if (properties.containsKey(STORAGE_ENDPOINT)) {
+ return;
+ }
+
+ String endpoint = properties.get(S3FileIOProperties.ENDPOINT);
+ if (endpoint == null) {
+ endpoint = properties.get(AliyunProperties.OSS_ENDPOINT);
+ }
+ if (endpoint != null) {
+ properties.put(STORAGE_ENDPOINT, endpoint);
+ }
+ }
+
+ private void validateDatabase(String database) {
+ if (!databaseExists(database)) {
+ throw new NoSuchDatabaseException("Database: " + database + " does not
exist");
+ }
+ }
+}
diff --git
a/amoro-format-lance/src/main/java/org/apache/amoro/formats/lance/LanceTable.java
b/amoro-format-lance/src/main/java/org/apache/amoro/formats/lance/LanceTable.java
new file mode 100755
index 000000000..465d7ef28
--- /dev/null
+++
b/amoro-format-lance/src/main/java/org/apache/amoro/formats/lance/LanceTable.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.amoro.formats.lance;
+
+import org.apache.amoro.AmoroTable;
+import org.apache.amoro.TableFormat;
+import org.apache.amoro.TableSnapshot;
+import org.apache.amoro.table.TableIdentifier;
+import org.lance.Dataset;
+
+import java.util.Collections;
+import java.util.Map;
+
+/** AmoroTable wrapper for a Lance {@link Dataset}. */
+public class LanceTable implements AmoroTable<Dataset> {
+
+ private final TableIdentifier identifier;
+ private final Dataset dataset;
+ private final Map<String, String> tableProperties;
+
+ public LanceTable(
+ TableIdentifier identifier, Dataset dataset, Map<String, String>
tableProperties) {
+ this.identifier = identifier;
+ this.dataset = dataset;
+ this.tableProperties =
+ tableProperties == null
+ ? Collections.emptyMap()
+ : Collections.unmodifiableMap(tableProperties);
+ }
+
+ @Override
+ public TableIdentifier id() {
+ return identifier;
+ }
+
+ @Override
+ public TableFormat format() {
+ return LanceCatalogFactory.LANCE;
+ }
+
+ @Override
+ public Map<String, String> properties() {
+ return tableProperties;
+ }
+
+ @Override
+ public Dataset originalTable() {
+ return dataset;
+ }
+
+ @Override
+ public TableSnapshot currentSnapshot() {
+ throw new IllegalStateException("The method is not implemented.");
+ }
+}
diff --git
a/amoro-format-lance/src/main/java/org/apache/amoro/formats/lance/LanceTableDescriptor.java
b/amoro-format-lance/src/main/java/org/apache/amoro/formats/lance/LanceTableDescriptor.java
new file mode 100755
index 000000000..bdcbe3024
--- /dev/null
+++
b/amoro-format-lance/src/main/java/org/apache/amoro/formats/lance/LanceTableDescriptor.java
@@ -0,0 +1,329 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.amoro.formats.lance;
+
+import org.apache.amoro.AmoroTable;
+import org.apache.amoro.TableFormat;
+import org.apache.amoro.process.ProcessStatus;
+import org.apache.amoro.table.descriptor.AMSColumnInfo;
+import org.apache.amoro.table.descriptor.AmoroSnapshotsOfTable;
+import org.apache.amoro.table.descriptor.ConsumerInfo;
+import org.apache.amoro.table.descriptor.DDLInfo;
+import org.apache.amoro.table.descriptor.FormatTableDescriptor;
+import org.apache.amoro.table.descriptor.OperationType;
+import org.apache.amoro.table.descriptor.OptimizingProcessInfo;
+import org.apache.amoro.table.descriptor.OptimizingTaskInfo;
+import org.apache.amoro.table.descriptor.PartitionBaseInfo;
+import org.apache.amoro.table.descriptor.PartitionFileBaseInfo;
+import org.apache.amoro.table.descriptor.ServerTableMeta;
+import org.apache.amoro.table.descriptor.TableSummary;
+import org.apache.amoro.table.descriptor.TagOrBranchInfo;
+import org.apache.amoro.utils.CommonUtil;
+import org.apache.arrow.vector.types.pojo.Field;
+import org.apache.arrow.vector.types.pojo.Schema;
+import org.apache.commons.lang3.tuple.Pair;
+import org.lance.Branch;
+import org.lance.Dataset;
+import org.lance.Fragment;
+import org.lance.FragmentMetadata;
+import org.lance.ManifestSummary;
+import org.lance.Tag;
+import org.lance.Version;
+import org.lance.fragment.DataFile;
+import org.lance.fragment.DeletionFile;
+
+import java.time.ZonedDateTime;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+
+/** Table descriptor for Lance tables. */
+public class LanceTableDescriptor implements FormatTableDescriptor {
+
+ private ExecutorService ioExecutor;
+
+ @Override
+ public void withIoExecutor(ExecutorService ioExecutor) {
+ this.ioExecutor = ioExecutor;
+ }
+
+ @Override
+ public List<TableFormat> supportFormat() {
+ return Collections.singletonList(LanceCatalogFactory.LANCE);
+ }
+
+ @Override
+ public ServerTableMeta getTableDetail(AmoroTable<?> amoroTable) {
+ Dataset dataset = (Dataset) amoroTable.originalTable();
+
+ ServerTableMeta meta = new ServerTableMeta();
+ meta.setTableIdentifier(amoroTable.id());
+ meta.setTableType(LanceCatalogFactory.LANCE.name());
+ meta.setBaseLocation(dataset.uri());
+
+ // Schema
+ List<AMSColumnInfo> columns = new ArrayList<>();
+ Schema schema = dataset.getSchema();
+ for (Field field : schema.getFields()) {
+ AMSColumnInfo columnInfo = new AMSColumnInfo();
+ columnInfo.setField(field.getName());
+ columnInfo.setType(field.getType().toString());
+ columnInfo.setRequired(!field.isNullable());
+ columnInfo.setComment(null);
+ columns.add(columnInfo);
+ }
+ meta.setSchema(columns);
+ meta.setPkList(Collections.emptyList());
+ meta.setPartitionColumnList(Collections.emptyList());
+ meta.setProperties(amoroTable.properties());
+
+ Version version = dataset.getVersion();
+ ManifestSummary summary = version.getManifestSummary();
+
+ long totalFilesSize = summary.getTotalFilesSize();
+ long totalDataFiles = summary.getTotalDataFiles();
+ long totalDeletionFiles = summary.getTotalDeletionFiles();
+ long totalFileCount = totalDataFiles + totalDeletionFiles;
+ long totalRows = summary.getTotalRows();
+
+ String totalSizeHuman = CommonUtil.byteToXB(totalFilesSize);
+ String averageFileSizeHuman =
+ CommonUtil.byteToXB(totalFileCount == 0 ? 0 : totalFilesSize /
totalFileCount);
+
+ // Table summary
+ String tableFormatDisplay = "Lance";
+ TableSummary tableSummary =
+ new TableSummary(
+ totalFileCount, totalSizeHuman, averageFileSizeHuman, totalRows,
tableFormatDisplay);
+ meta.setTableSummary(tableSummary);
+
+ // Base metrics (data files only)
+ Map<String, Object> baseMetrics = new HashMap<>();
+ baseMetrics.put("totalSize", CommonUtil.byteToXB(totalFilesSize));
+ baseMetrics.put("fileCount", totalDataFiles);
+ baseMetrics.put(
+ "averageFileSize",
+ CommonUtil.byteToXB(totalDataFiles == 0 ? 0 : totalFilesSize /
totalDataFiles));
+ meta.setBaseMetrics(baseMetrics);
+
+ // Change metrics (deletion files if present)
+ Map<String, Object> changeMetrics = new HashMap<>();
+ if (totalDeletionFiles > 0) {
+ changeMetrics.put("deletionFileCount", totalDeletionFiles);
+ changeMetrics.put("deletedRows", summary.getTotalDeletionFileRows());
+ }
+ meta.setChangeMetrics(changeMetrics);
+
+ return meta;
+ }
+
+ @Override
+ public List<AmoroSnapshotsOfTable> getSnapshots(
+ AmoroTable<?> amoroTable, String ref, OperationType operationType) {
+ Dataset dataset = (Dataset) amoroTable.originalTable();
+ List<Version> versions = dataset.listVersions();
+
+ if (ioExecutor == null) {
+ List<AmoroSnapshotsOfTable> snapshots = new ArrayList<>();
+ for (Version version : versions) {
+ snapshots.add(buildSnapshot(version));
+ }
+ return snapshots;
+ }
+
+ List<CompletableFuture<AmoroSnapshotsOfTable>> futures = new ArrayList<>();
+ for (Version version : versions) {
+ futures.add(CompletableFuture.supplyAsync(() -> buildSnapshot(version),
ioExecutor));
+ }
+
+ List<AmoroSnapshotsOfTable> snapshots = new ArrayList<>();
+ for (CompletableFuture<AmoroSnapshotsOfTable> future : futures) {
+ try {
+ snapshots.add(future.get());
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new RuntimeException("Interrupted while loading Lance
snapshots", e);
+ } catch (ExecutionException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ return snapshots;
+ }
+
+ @Override
+ public List<PartitionFileBaseInfo> getSnapshotDetail(
+ AmoroTable<?> amoroTable, String snapshotId, String ref) {
+ Dataset dataset = (Dataset) amoroTable.originalTable();
+ long versionId;
+ try {
+ versionId = Long.parseLong(snapshotId);
+ } catch (NumberFormatException e) {
+ return Collections.emptyList();
+ }
+
+ try (Dataset versionDataset = dataset.checkoutVersion(versionId)) {
+ Version version = versionDataset.getVersion();
+ long commitTime = toMillis(version.getDataTime());
+ List<PartitionFileBaseInfo> files = new ArrayList<>();
+
+ for (Fragment fragment : versionDataset.getFragments()) {
+ FragmentMetadata metadata = fragment.metadata();
+ for (DataFile dataFile : metadata.getFiles()) {
+ String path = dataFile.getPath();
+ long size = dataFile.getFileSizeBytes() == null ? 0L :
dataFile.getFileSizeBytes();
+ files.add(
+ new PartitionFileBaseInfo(snapshotId, "DATA_FILE", commitTime,
"", 0, path, size));
+ }
+
+ DeletionFile deletionFile = metadata.getDeletionFile();
+ if (deletionFile != null) {
+ files.add(
+ new PartitionFileBaseInfo(snapshotId, "DELETION_FILE",
commitTime, "", 0, "", 0L));
+ }
+ }
+
+ return files;
+ }
+ }
+
+ @Override
+ public List<DDLInfo> getTableOperations(AmoroTable<?> amoroTable) {
+ return Collections.emptyList();
+ }
+
+ @Override
+ public List<PartitionBaseInfo> getTablePartitions(AmoroTable<?> amoroTable) {
+ // Lance tables are treated as non-partitioned for now.
+ return Collections.emptyList();
+ }
+
+ @Override
+ public List<PartitionFileBaseInfo> getTableFiles(
+ AmoroTable<?> amoroTable, String partition, Integer specId) {
+ Dataset dataset = (Dataset) amoroTable.originalTable();
+ Version version = dataset.getVersion();
+ long commitTime = toMillis(version.getDataTime());
+
+ List<PartitionFileBaseInfo> files = new ArrayList<>();
+ for (Fragment fragment : dataset.getFragments()) {
+ FragmentMetadata metadata = fragment.metadata();
+ for (DataFile dataFile : metadata.getFiles()) {
+ String path = dataFile.getPath();
+ long size = dataFile.getFileSizeBytes() == null ? 0L :
dataFile.getFileSizeBytes();
+ files.add(
+ new PartitionFileBaseInfo(
+ String.valueOf(version.getId()), "DATA_FILE", commitTime, "",
0, path, size));
+ }
+ }
+ return files;
+ }
+
+ @Override
+ public Pair<List<OptimizingProcessInfo>, Integer> getOptimizingProcessesInfo(
+ AmoroTable<?> amoroTable,
+ String type,
+ String processCategory,
+ ProcessStatus status,
+ int limit,
+ int offset) {
+ return Pair.of(Collections.emptyList(), 0);
+ }
+
+ @Override
+ public Map<String, String> getTableOptimizingTypes(AmoroTable<?> amoroTable)
{
+ return Collections.emptyMap();
+ }
+
+ @Override
+ public List<String> getProcessTypesByCategory(String processCategory) {
+ return Collections.emptyList();
+ }
+
+ @Override
+ public List<OptimizingTaskInfo> getOptimizingTaskInfos(
+ AmoroTable<?> amoroTable, String processId) {
+ return Collections.emptyList();
+ }
+
+ @Override
+ public List<TagOrBranchInfo> getTableTags(AmoroTable<?> amoroTable) {
+ Dataset dataset = (Dataset) amoroTable.originalTable();
+ List<Tag> tags = dataset.tags().list();
+
+ List<TagOrBranchInfo> infos = new ArrayList<>();
+ for (Tag tag : tags) {
+ TagOrBranchInfo info =
+ new TagOrBranchInfo(tag.getName(), tag.getVersion(), -1, 0L, 0L,
TagOrBranchInfo.TAG);
+ infos.add(info);
+ }
+ return infos;
+ }
+
+ @Override
+ public List<TagOrBranchInfo> getTableBranches(AmoroTable<?> amoroTable) {
+ Dataset dataset = (Dataset) amoroTable.originalTable();
+ List<Branch> branches = dataset.branches().list();
+
+ List<TagOrBranchInfo> branchInfos = new ArrayList<>();
+ for (Branch branch : branches) {
+ TagOrBranchInfo info =
+ new TagOrBranchInfo(
+ branch.getName(), branch.getParentVersion() + 1, -1, 0L, 0L,
TagOrBranchInfo.BRANCH);
+ branchInfos.add(info);
+ }
+ branchInfos.add(TagOrBranchInfo.MAIN_BRANCH);
+ return branchInfos;
+ }
+
+ @Override
+ public List<ConsumerInfo> getTableConsumerInfos(AmoroTable<?> amoroTable) {
+ return Collections.emptyList();
+ }
+
+ private AmoroSnapshotsOfTable buildSnapshot(Version version) {
+ ManifestSummary summary = version.getManifestSummary();
+ AmoroSnapshotsOfTable snapshot = new AmoroSnapshotsOfTable();
+ snapshot.setSnapshotId(String.valueOf(version.getId()));
+ snapshot.setCommitTime(toMillis(version.getDataTime()));
+ snapshot.setFileCount((int) summary.getTotalDataFiles());
+ snapshot.setFileSize(summary.getTotalFilesSize());
+ snapshot.setRecords(summary.getTotalRows());
+ snapshot.setOperation("WRITE");
+
+ Map<String, String> filesSummary = new HashMap<>();
+ filesSummary.put("data-files",
String.valueOf(summary.getTotalDataFiles()));
+ filesSummary.put("delta-files",
String.valueOf(summary.getTotalDeletionFiles()));
+ filesSummary.put("changelogs", "0");
+ snapshot.setFilesSummaryForChart(filesSummary);
+
+ return snapshot;
+ }
+
+ private long toMillis(ZonedDateTime time) {
+ if (time == null) {
+ return 0L;
+ }
+ return time.toInstant().toEpochMilli();
+ }
+}
diff --git
a/amoro-format-lance/src/main/resources/META-INF/services/org.apache.amoro.FormatCatalogFactory
b/amoro-format-lance/src/main/resources/META-INF/services/org.apache.amoro.FormatCatalogFactory
new file mode 100644
index 000000000..34b9298ab
--- /dev/null
+++
b/amoro-format-lance/src/main/resources/META-INF/services/org.apache.amoro.FormatCatalogFactory
@@ -0,0 +1,19 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+# *
+# http://www.apache.org/licenses/LICENSE-2.0
+# *
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+org.apache.amoro.formats.lance.LanceCatalogFactory
\ No newline at end of file
diff --git
a/amoro-format-lance/src/main/resources/META-INF/services/org.apache.amoro.table.descriptor.FormatTableDescriptor
b/amoro-format-lance/src/main/resources/META-INF/services/org.apache.amoro.table.descriptor.FormatTableDescriptor
new file mode 100644
index 000000000..14dc08db2
--- /dev/null
+++
b/amoro-format-lance/src/main/resources/META-INF/services/org.apache.amoro.table.descriptor.FormatTableDescriptor
@@ -0,0 +1,18 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+# *
+# http://www.apache.org/licenses/LICENSE-2.0
+# *
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+org.apache.amoro.formats.lance.LanceTableDescriptor
\ No newline at end of file
diff --git a/amoro-web/src/assets/icons/svg/lance.svg
b/amoro-web/src/assets/icons/svg/lance.svg
new file mode 100755
index 000000000..6fc725168
--- /dev/null
+++ b/amoro-web/src/assets/icons/svg/lance.svg
@@ -0,0 +1,25 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+/-->
+
+<svg xmlns="http://www.w3.org/2000/svg"; fill="none" viewBox="0 0 16 16"
width="16" height="16">
+ <rect x="1" y="1" width="14" height="14" rx="3" fill="#4A90E2" />
+ <path
+ d="M4.5 10.5L7 4.5L10 9L11.5 7L12.5 9.5L11 11.5H5.5L4.5 10.5Z"
+ fill="white"
+ />
+</svg>
diff --git a/amoro-web/src/types/common.type.ts
b/amoro-web/src/types/common.type.ts
index 2f5dff1c7..8f6a39d4e 100644
--- a/amoro-web/src/types/common.type.ts
+++ b/amoro-web/src/types/common.type.ts
@@ -380,6 +380,7 @@ export enum tableTypeIconMap {
HIVE = 'hive',
PAIMON = 'paimon',
HUDI = 'hudi',
+ LANCE = 'lance',
}
export type ILineChartOriginalData = Record<string, Record<string, number>>
diff --git a/dev/deps/dependencies-hadoop-2-spark-3.3
b/dev/deps/dependencies-hadoop-2-spark-3.3
index 4cfe8086e..59c955b2b 100644
--- a/dev/deps/dependencies-hadoop-2-spark-3.3
+++ b/dev/deps/dependencies-hadoop-2-spark-3.3
@@ -21,10 +21,11 @@ api-util/1.0.0-M20//api-util-1.0.0-M20.jar
apiguardian-api/1.1.2//apiguardian-api-1.1.2.jar
args4j/2.33//args4j-2.33.jar
arns/2.24.12//arns-2.24.12.jar
-arrow-format/15.0.2//arrow-format-15.0.2.jar
-arrow-memory-core/15.0.2//arrow-memory-core-15.0.2.jar
-arrow-memory-netty/15.0.2//arrow-memory-netty-15.0.2.jar
-arrow-vector/15.0.2//arrow-vector-15.0.2.jar
+arrow-format/18.3.0//arrow-format-18.3.0.jar
+arrow-memory-core/18.3.0//arrow-memory-core-18.3.0.jar
+arrow-memory-netty-buffer-patch/18.3.0//arrow-memory-netty-buffer-patch-18.3.0.jar
+arrow-memory-netty/18.3.0//arrow-memory-netty-18.3.0.jar
+arrow-vector/18.3.0//arrow-vector-18.3.0.jar
asm/5.0.4//asm-5.0.4.jar
async-profiler/2.9//async-profiler-2.9.jar
auth/2.24.12//auth-2.24.12.jar
@@ -76,15 +77,13 @@ datanucleus-rdbms/4.1.19//datanucleus-rdbms-4.1.19.jar
derby/10.14.2.0//derby-10.14.2.0.jar
dropwizard-metrics-hadoop-metrics2-reporter/0.1.2//dropwizard-metrics-hadoop-metrics2-reporter-0.1.2.jar
dynamodb/2.24.12//dynamodb-2.24.12.jar
-eclipse-collections-api/11.1.0//eclipse-collections-api-11.1.0.jar
-eclipse-collections/11.1.0//eclipse-collections-11.1.0.jar
ehcache/3.3.1//ehcache-3.3.1.jar
endpoints-spi/2.24.12//endpoints-spi-2.24.12.jar
error_prone_annotations/2.18.0//error_prone_annotations-2.18.0.jar
eventstream/1.0.1//eventstream-1.0.1.jar
failsafe/3.3.2//failsafe-3.3.2.jar
failureaccess/1.0.1//failureaccess-1.0.1.jar
-flatbuffers-java/23.5.26//flatbuffers-java-23.5.26.jar
+flatbuffers-java/25.2.10//flatbuffers-java-25.2.10.jar
flink-annotations/1.20.3//flink-annotations-1.20.3.jar
flink-clients/1.20.3//flink-clients-1.20.3.jar
flink-connector-datagen/1.20.3//flink-connector-datagen-1.20.3.jar
diff --git a/dev/deps/dependencies-hadoop-3-spark-3.5
b/dev/deps/dependencies-hadoop-3-spark-3.5
index a97b4a692..d99a4078a 100644
--- a/dev/deps/dependencies-hadoop-3-spark-3.5
+++ b/dev/deps/dependencies-hadoop-3-spark-3.5
@@ -16,10 +16,11 @@ apache-client/2.24.12//apache-client-2.24.12.jar
apiguardian-api/1.1.2//apiguardian-api-1.1.2.jar
args4j/2.33//args4j-2.33.jar
arns/2.24.12//arns-2.24.12.jar
-arrow-format/15.0.2//arrow-format-15.0.2.jar
-arrow-memory-core/15.0.2//arrow-memory-core-15.0.2.jar
-arrow-memory-netty/15.0.2//arrow-memory-netty-15.0.2.jar
-arrow-vector/15.0.2//arrow-vector-15.0.2.jar
+arrow-format/18.3.0//arrow-format-18.3.0.jar
+arrow-memory-core/18.3.0//arrow-memory-core-18.3.0.jar
+arrow-memory-netty-buffer-patch/18.3.0//arrow-memory-netty-buffer-patch-18.3.0.jar
+arrow-memory-netty/18.3.0//arrow-memory-netty-18.3.0.jar
+arrow-vector/18.3.0//arrow-vector-18.3.0.jar
asm/5.0.4//asm-5.0.4.jar
async-profiler/2.9//async-profiler-2.9.jar
auth/2.24.12//auth-2.24.12.jar
@@ -70,14 +71,12 @@ datasketches-memory/2.1.0//datasketches-memory-2.1.0.jar
derby/10.14.2.0//derby-10.14.2.0.jar
dropwizard-metrics-hadoop-metrics2-reporter/0.1.2//dropwizard-metrics-hadoop-metrics2-reporter-0.1.2.jar
dynamodb/2.24.12//dynamodb-2.24.12.jar
-eclipse-collections-api/11.1.0//eclipse-collections-api-11.1.0.jar
-eclipse-collections/11.1.0//eclipse-collections-11.1.0.jar
endpoints-spi/2.24.12//endpoints-spi-2.24.12.jar
error_prone_annotations/2.18.0//error_prone_annotations-2.18.0.jar
eventstream/1.0.1//eventstream-1.0.1.jar
failsafe/3.3.2//failsafe-3.3.2.jar
failureaccess/1.0.1//failureaccess-1.0.1.jar
-flatbuffers-java/23.5.26//flatbuffers-java-23.5.26.jar
+flatbuffers-java/25.2.10//flatbuffers-java-25.2.10.jar
flatbuffers/1.2.0-3f79e055//flatbuffers-1.2.0-3f79e055.jar
flink-annotations/1.20.3//flink-annotations-1.20.3.jar
flink-clients/1.20.3//flink-clients-1.20.3.jar
diff --git a/pom.xml b/pom.xml
index 78659d508..40cc34ecb 100644
--- a/pom.xml
+++ b/pom.xml
@@ -54,6 +54,7 @@
<module>amoro-format-mixed</module>
<module>amoro-format-hudi</module>
<module>amoro-format-paimon</module>
+ <module>amoro-format-lance</module>
<module>dist</module>
</modules>
@@ -166,6 +167,7 @@
<jsqlparser.version>4.7</jsqlparser.version>
<jcasbin.version>1.99.0</jcasbin.version>
<fasterxml.jackson.version>2.14.2</fasterxml.jackson.version>
+ <arrow.version>18.3.0</arrow.version>
<apache-directory-server.version>2.0.0-M15</apache-directory-server.version>
<apache-directory-api-all.version>1.0.0-M20</apache-directory-api-all.version>
<rocksdb-dependency-scope>compile</rocksdb-dependency-scope>
@@ -195,6 +197,14 @@
<dependencyManagement>
<dependencies>
+ <dependency>
+ <groupId>org.apache.arrow</groupId>
+ <artifactId>arrow-bom</artifactId>
+ <version>${arrow.version}</version>
+ <type>pom</type>
+ <scope>import</scope>
+ </dependency>
+
<dependency>
<groupId>org.apache.amoro</groupId>
<artifactId>amoro-common</artifactId>
@@ -219,6 +229,12 @@
<version>${project.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.amoro</groupId>
+ <artifactId>amoro-format-lance</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+
<dependency>
<groupId>org.apache.amoro</groupId>
<artifactId>amoro-mixed-hive</artifactId>
