This is an automated email from the ASF dual-hosted git repository.
jinsongzhou pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/amoro.git
The following commit(s) were added to refs/heads/master by this push:
new 499f6d3dc [AMORO-3564] Fix some issues in mixed format connector using
Iceberg 1.4 (#3505)
499f6d3dc is described below
commit 499f6d3dc334fb8cfcd483ca88b8014361e066ff
Author: ZhouJinsong <[email protected]>
AuthorDate: Fri Apr 11 14:22:19 2025 +0800
[AMORO-3564] Fix some issues in mixed format connector using Iceberg 1.4
(#3505)
* Rollback change in TestIcebergFindFiles
* Fix checkstyle errors
* Change Lists to guava shade
---
amoro-common/pom.xml | 6 ----
.../java/org/apache/amoro/client/PoolConfig.java | 36 +++++++++++++++++++---
amoro-format-iceberg/pom.xml | 6 ++++
.../org/apache/amoro/table/BasicUnkeyedTable.java | 1 -
.../org/apache/amoro/BasicTableTestHelper.java | 2 +-
.../v1.15/amoro-mixed-flink-runtime-1.15/pom.xml | 1 +
.../v1.16/amoro-mixed-flink-runtime-1.16/pom.xml | 1 +
.../v1.17/amoro-mixed-flink-1.17/pom.xml | 1 -
.../v3.2/amoro-mixed-spark-runtime-3.2/pom.xml | 4 +++
.../v3.3/amoro-mixed-spark-3.3/pom.xml | 1 -
amoro-format-mixed/amoro-mixed-trino/pom.xml | 1 -
11 files changed, 45 insertions(+), 15 deletions(-)
diff --git a/amoro-common/pom.xml b/amoro-common/pom.xml
index 0f50d55ca..802854146 100644
--- a/amoro-common/pom.xml
+++ b/amoro-common/pom.xml
@@ -112,12 +112,6 @@
<artifactId>hive-metastore</artifactId>
</dependency>
- <dependency>
- <groupId>org.apache.iceberg</groupId>
- <artifactId>iceberg-bundled-guava</artifactId>
- <version>${iceberg.version}</version>
- </dependency>
-
<dependency>
<groupId>org.rocksdb</groupId>
<artifactId>rocksdbjni</artifactId>
diff --git a/amoro-common/src/main/java/org/apache/amoro/client/PoolConfig.java
b/amoro-common/src/main/java/org/apache/amoro/client/PoolConfig.java
index c3574e612..ddb8c5609 100644
--- a/amoro-common/src/main/java/org/apache/amoro/client/PoolConfig.java
+++ b/amoro-common/src/main/java/org/apache/amoro/client/PoolConfig.java
@@ -18,15 +18,19 @@
package org.apache.amoro.client;
+import org.apache.amoro.shade.guava32.com.google.common.base.Preconditions;
+import org.apache.amoro.shade.guava32.com.google.common.collect.Lists;
+import org.apache.amoro.shade.thrift.org.apache.commons.lang3.tuple.Pair;
import org.apache.amoro.shade.thrift.org.apache.thrift.TServiceClient;
import org.apache.commons.beanutils.BeanUtils;
import org.apache.commons.pool2.impl.GenericObjectPoolConfig;
-import org.apache.http.client.utils.URLEncodedUtils;
+import java.io.UnsupportedEncodingException;
import java.lang.reflect.InvocationTargetException;
import java.net.URI;
-import java.nio.charset.Charset;
+import java.net.URLDecoder;
import java.time.Duration;
+import java.util.List;
public class PoolConfig<T extends TServiceClient> extends
GenericObjectPoolConfig<ThriftClient<T>> {
@@ -40,6 +44,10 @@ public class PoolConfig<T extends TServiceClient> extends
GenericObjectPoolConfi
private boolean autoReconnect = true;
private int maxReconnects = 5;
+ private static final String URL_ENCODING = "UTF-8";
+ private static final String URL_QUERY_DELIMITER = "&";
+ private static final String URL_QUERY_PARAMETER_DELIMITER = "=";
+
public PoolConfig() {
setMinIdle(MIN_IDLE_DEFAULT);
setMaxIdle(MAX_IDLE_DEFAULT);
@@ -88,15 +96,35 @@ public class PoolConfig<T extends TServiceClient> extends
GenericObjectPoolConfi
public static PoolConfig<?> forUrl(String url) {
PoolConfig<?> poolConfig = new PoolConfig<>();
- URLEncodedUtils.parse(URI.create(url),
String.valueOf(Charset.defaultCharset()))
+ parseQuery(URI.create(url))
.forEach(
pair -> {
try {
- BeanUtils.setProperty(poolConfig, pair.getName(),
pair.getValue());
+ BeanUtils.setProperty(poolConfig, pair.getKey(),
pair.getValue());
} catch (IllegalAccessException | InvocationTargetException e) {
throw new RuntimeException("Parse url parameters failed", e);
}
});
return poolConfig;
}
+
+ static List<Pair<String, String>> parseQuery(URI uri) {
+ Preconditions.checkNotNull(uri, "URI can not be null");
+ List<Pair<String, String>> queries = Lists.newArrayList();
+ String query = uri.getRawQuery();
+ if (query != null && !query.trim().isEmpty()) {
+ String[] pairs = query.trim().split(URL_QUERY_DELIMITER);
+ for (String pair : pairs) {
+ String[] kv = pair.trim().split(URL_QUERY_PARAMETER_DELIMITER, 2);
+ try {
+ String key = URLDecoder.decode(kv[0], URL_ENCODING);
+ String value = URLDecoder.decode(kv[1], URL_ENCODING);
+ queries.add(Pair.of(key, value));
+ } catch (UnsupportedEncodingException e) {
+ throw new RuntimeException("Unsupported encoding for uri", e);
+ }
+ }
+ }
+ return queries;
+ }
}
diff --git a/amoro-format-iceberg/pom.xml b/amoro-format-iceberg/pom.xml
index c76a879dd..7814644f8 100644
--- a/amoro-format-iceberg/pom.xml
+++ b/amoro-format-iceberg/pom.xml
@@ -97,6 +97,12 @@
<artifactId>iceberg-common</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.apache.iceberg</groupId>
+ <artifactId>iceberg-bundled-guava</artifactId>
+ <version>${iceberg.version}</version>
+ </dependency>
+
<dependency>
<groupId>org.apache.iceberg</groupId>
<artifactId>iceberg-parquet</artifactId>
diff --git
a/amoro-format-iceberg/src/main/java/org/apache/amoro/table/BasicUnkeyedTable.java
b/amoro-format-iceberg/src/main/java/org/apache/amoro/table/BasicUnkeyedTable.java
index 6c5cceee6..621f85c09 100644
---
a/amoro-format-iceberg/src/main/java/org/apache/amoro/table/BasicUnkeyedTable.java
+++
b/amoro-format-iceberg/src/main/java/org/apache/amoro/table/BasicUnkeyedTable.java
@@ -320,7 +320,6 @@ public class BasicUnkeyedTable implements UnkeyedTable,
HasTableOperations {
return new PartitionPropertiesUpdate(this, transaction);
}
- @Override
public UUID uuid() {
return UUID.fromString(this.operations().current().uuid());
}
diff --git
a/amoro-format-iceberg/src/test/java/org/apache/amoro/BasicTableTestHelper.java
b/amoro-format-iceberg/src/test/java/org/apache/amoro/BasicTableTestHelper.java
index d60d41450..105caf604 100644
---
a/amoro-format-iceberg/src/test/java/org/apache/amoro/BasicTableTestHelper.java
+++
b/amoro-format-iceberg/src/test/java/org/apache/amoro/BasicTableTestHelper.java
@@ -72,7 +72,7 @@ public class BasicTableTestHelper implements TableTestHelper {
if (tableProperties != null) {
this.tableProperties.putAll(tableProperties);
}
- this.tableProperties.put(TableProperties.FORMAT_VERSION, "2");
+ this.tableProperties.putIfAbsent(TableProperties.FORMAT_VERSION, "2");
}
public BasicTableTestHelper(
diff --git
a/amoro-format-mixed/amoro-mixed-flink/v1.15/amoro-mixed-flink-runtime-1.15/pom.xml
b/amoro-format-mixed/amoro-mixed-flink/v1.15/amoro-mixed-flink-runtime-1.15/pom.xml
index e218e978a..a19b52095 100644
---
a/amoro-format-mixed/amoro-mixed-flink/v1.15/amoro-mixed-flink-runtime-1.15/pom.xml
+++
b/amoro-format-mixed/amoro-mixed-flink/v1.15/amoro-mixed-flink-runtime-1.15/pom.xml
@@ -32,6 +32,7 @@
<url>https://amoro.apache.org</url>
<properties>
+ <iceberg.version>1.4.3</iceberg.version>
<flink.version>1.15.3</flink.version>
</properties>
diff --git
a/amoro-format-mixed/amoro-mixed-flink/v1.16/amoro-mixed-flink-runtime-1.16/pom.xml
b/amoro-format-mixed/amoro-mixed-flink/v1.16/amoro-mixed-flink-runtime-1.16/pom.xml
index 1bfcfb138..39efed2a6 100644
---
a/amoro-format-mixed/amoro-mixed-flink/v1.16/amoro-mixed-flink-runtime-1.16/pom.xml
+++
b/amoro-format-mixed/amoro-mixed-flink/v1.16/amoro-mixed-flink-runtime-1.16/pom.xml
@@ -32,6 +32,7 @@
<url>https://amoro.apache.org</url>
<properties>
+ <iceberg.version>1.4.3</iceberg.version>
<flink.version>1.16.2</flink.version>
</properties>
diff --git
a/amoro-format-mixed/amoro-mixed-flink/v1.17/amoro-mixed-flink-1.17/pom.xml
b/amoro-format-mixed/amoro-mixed-flink/v1.17/amoro-mixed-flink-1.17/pom.xml
index 669b76b5b..eb958c075 100644
--- a/amoro-format-mixed/amoro-mixed-flink/v1.17/amoro-mixed-flink-1.17/pom.xml
+++ b/amoro-format-mixed/amoro-mixed-flink/v1.17/amoro-mixed-flink-1.17/pom.xml
@@ -34,7 +34,6 @@
<packaging>jar</packaging>
<properties>
- <iceberg.version>1.4.3</iceberg.version>
<kafka.version>3.2.3</kafka.version>
<assertj.version>3.21.0</assertj.version>
<testcontainers.version>1.17.2</testcontainers.version>
diff --git
a/amoro-format-mixed/amoro-mixed-spark/v3.2/amoro-mixed-spark-runtime-3.2/pom.xml
b/amoro-format-mixed/amoro-mixed-spark/v3.2/amoro-mixed-spark-runtime-3.2/pom.xml
index f243bd1d4..0a63d31b2 100644
---
a/amoro-format-mixed/amoro-mixed-spark/v3.2/amoro-mixed-spark-runtime-3.2/pom.xml
+++
b/amoro-format-mixed/amoro-mixed-spark/v3.2/amoro-mixed-spark-runtime-3.2/pom.xml
@@ -33,6 +33,10 @@
<name>Amoro Project Mixed Format Spark 3.2 Runtime</name>
<url>https://amoro.apache.org</url>
+ <properties>
+ <iceberg.version>1.4.3</iceberg.version>
+ </properties>
+
<dependencies>
<dependency>
<groupId>org.apache.amoro</groupId>
diff --git
a/amoro-format-mixed/amoro-mixed-spark/v3.3/amoro-mixed-spark-3.3/pom.xml
b/amoro-format-mixed/amoro-mixed-spark/v3.3/amoro-mixed-spark-3.3/pom.xml
index 2dd91e9ba..8d6cbff8b 100644
--- a/amoro-format-mixed/amoro-mixed-spark/v3.3/amoro-mixed-spark-3.3/pom.xml
+++ b/amoro-format-mixed/amoro-mixed-spark/v3.3/amoro-mixed-spark-3.3/pom.xml
@@ -34,7 +34,6 @@
<url>https://amoro.apache.org</url>
<properties>
- <iceberg.version>1.4.3</iceberg.version>
<hive.version>2.3.9</hive.version>
<spark.version>3.3.2</spark.version>
<scala.version>2.12.15</scala.version>
diff --git a/amoro-format-mixed/amoro-mixed-trino/pom.xml
b/amoro-format-mixed/amoro-mixed-trino/pom.xml
index 936c8b4e1..4cf7cd341 100644
--- a/amoro-format-mixed/amoro-mixed-trino/pom.xml
+++ b/amoro-format-mixed/amoro-mixed-trino/pom.xml
@@ -53,7 +53,6 @@
</properties>
<dependencies>
-
<dependency>
<groupId>io.airlift</groupId>
<artifactId>concurrent</artifactId>
