This is an automated email from the ASF dual-hosted git repository.
zhoujinsong pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/amoro.git
The following commit(s) were added to refs/heads/master by this push:
new 18859a662 [AMORO-4337] Upgrade Iceberg dependency to 1.8.1 (#4338)
18859a662 is described below
commit 18859a662cddb9bb74066e6f27cf2821f15f705c
Author: Jiwon Park <[email protected]>
AuthorDate: Wed Aug 26 20:30:07 2026 +0900
[AMORO-4337] Upgrade Iceberg dependency to 1.8.1 (#4338)
* [AMORO-4337] Upgrade Iceberg dependency to 1.8.1
Bump iceberg.version from 1.7.2 to 1.8.1. The only API removal affecting
Amoro is FlinkAvroReader, removed in Iceberg 1.8.0:
RowDataFileScanTaskReader
in amoro-mixed-flink-common-iceberg-bridge is a copy of the Iceberg class of
the same name, so mirror the change Iceberg 1.8 made to its own copy
(FlinkPlannedAvroReader via createReaderFunc, and path() to location()).
Also refresh the Iceberg download links in the Flink CDC ingestion doc.
Signed-off-by: Jiwon Park <[email protected]>
* [AMORO-4337] Fix pos-delete read schema in IcebergRewriteExecutorTest
The test projected position-delete files with the virtual metadata columns
_file/_pos (MetadataColumns.FILE_PATH/ROW_POSITION), while the files
actually
store file_path/pos (DELETE_FILE_PATH/DELETE_FILE_POS) with different field
ids. Iceberg 1.7 tolerated the mismatch by synthesizing row positions and
ignoring the missing _file column, so the assertions passed by accident.
Iceberg 1.8 rejects projections of required fields that are absent from the
file (Missing required field: _file). Project with the delete-file schema so
the test reads the values actually written.
Signed-off-by: Jiwon Park <[email protected]>
* [AMORO-4337] Update dependency manifests for Iceberg 1.8.1
Regenerated with ./dev/dependencies.sh --replace: iceberg artifacts move to
1.8.1 and the transitive httpclient5/httpcore5 move to 5.4.1/5.3.1.
Signed-off-by: Jiwon Park <[email protected]>
---------
Signed-off-by: Jiwon Park <[email protected]>
---
.../optimizing/IcebergRewriteExecutorTest.java | 2 +-
.../flink/source/RowDataFileScanTaskReader.java | 6 ++--
dev/deps/dependencies-hadoop-2-spark-3.3 | 32 +++++++++++-----------
dev/deps/dependencies-hadoop-3-spark-3.5 | 32 +++++++++++-----------
docs/engines/flink/flink-cdc-ingestion.md | 4 +--
pom.xml | 2 +-
6 files changed, 39 insertions(+), 39 deletions(-)
diff --git
a/amoro-format-iceberg/src/test/java/org/apache/amoro/optimizing/IcebergRewriteExecutorTest.java
b/amoro-format-iceberg/src/test/java/org/apache/amoro/optimizing/IcebergRewriteExecutorTest.java
index 71b4b16d2..3cdfff266 100644
---
a/amoro-format-iceberg/src/test/java/org/apache/amoro/optimizing/IcebergRewriteExecutorTest.java
+++
b/amoro-format-iceberg/src/test/java/org/apache/amoro/optimizing/IcebergRewriteExecutorTest.java
@@ -80,7 +80,7 @@ public class IcebergRewriteExecutorTest extends TableTestBase
{
private RewriteFilesInput dataScanTask;
private final Schema posSchema =
- new Schema(MetadataColumns.FILE_PATH, MetadataColumns.ROW_POSITION);
+ new Schema(MetadataColumns.DELETE_FILE_PATH,
MetadataColumns.DELETE_FILE_POS);
public IcebergRewriteExecutorTest(boolean hasPartition, FileFormat
fileFormat) {
super(
diff --git
a/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common-iceberg-bridge/src/main/java/org/apache/iceberg/flink/source/RowDataFileScanTaskReader.java
b/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common-iceberg-bridge/src/main/java/org/apache/iceberg/flink/source/RowDataFileScanTaskReader.java
index 599d4cbb2..2d4a93617 100644
---
a/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common-iceberg-bridge/src/main/java/org/apache/iceberg/flink/source/RowDataFileScanTaskReader.java
+++
b/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common-iceberg-bridge/src/main/java/org/apache/iceberg/flink/source/RowDataFileScanTaskReader.java
@@ -36,8 +36,8 @@ import org.apache.iceberg.flink.FlinkSchemaUtil;
import org.apache.iceberg.flink.FlinkSourceFilter;
import org.apache.iceberg.flink.RowDataWrapper;
import org.apache.iceberg.flink.data.AdaptHiveFlinkParquetReaders;
-import org.apache.iceberg.flink.data.FlinkAvroReader;
import org.apache.iceberg.flink.data.FlinkOrcReader;
+import org.apache.iceberg.flink.data.FlinkPlannedAvroReader;
import org.apache.iceberg.flink.data.RowDataProjection;
import org.apache.iceberg.flink.data.RowDataUtil;
import org.apache.iceberg.io.CloseableIterable;
@@ -156,7 +156,7 @@ public class RowDataFileScanTaskReader implements
FileScanTaskReader<RowData> {
.reuseContainers()
.project(schema)
.split(task.start(), task.length())
- .createReaderFunc(readSchema -> new FlinkAvroReader(schema,
readSchema, idToConstant));
+ .createReaderFunc(readSchema ->
FlinkPlannedAvroReader.create(schema, idToConstant));
if (nameMapping != null) {
builder.withNameMapping(NameMappingParser.fromJson(nameMapping));
@@ -224,7 +224,7 @@ public class RowDataFileScanTaskReader implements
FileScanTaskReader<RowData> {
Schema tableSchema,
Schema requestedSchema,
InputFilesDecryptor inputFilesDecryptor) {
- super(task.file().path().toString(), task.deletes(), tableSchema,
requestedSchema);
+ super(task.file().location(), task.deletes(), tableSchema,
requestedSchema);
this.requiredRowType = FlinkSchemaUtil.convert(requiredSchema());
this.asStructLike = new RowDataWrapper(requiredRowType,
requiredSchema().asStruct());
this.inputFilesDecryptor = inputFilesDecryptor;
diff --git a/dev/deps/dependencies-hadoop-2-spark-3.3
b/dev/deps/dependencies-hadoop-2-spark-3.3
index 9c38f9fd8..d1fa96bf9 100644
--- a/dev/deps/dependencies-hadoop-2-spark-3.3
+++ b/dev/deps/dependencies-hadoop-2-spark-3.3
@@ -145,23 +145,23 @@ http-auth-spi/2.24.12//http-auth-spi-2.24.12.jar
http-auth/2.24.12//http-auth-2.24.12.jar
http-client-spi/2.24.12//http-client-spi-2.24.12.jar
httpclient/4.5.13//httpclient-4.5.13.jar
-httpclient5/5.4//httpclient5-5.4.jar
+httpclient5/5.4.1//httpclient5-5.4.1.jar
httpcore/4.4.13//httpcore-4.4.13.jar
-httpcore5-h2/5.3//httpcore5-h2-5.3.jar
-httpcore5/5.3//httpcore5-5.3.jar
-iceberg-aliyun/1.7.2//iceberg-aliyun-1.7.2.jar
-iceberg-api/1.7.2//iceberg-api-1.7.2.jar
-iceberg-arrow/1.7.2//iceberg-arrow-1.7.2.jar
-iceberg-aws/1.7.2//iceberg-aws-1.7.2.jar
-iceberg-bundled-guava/1.7.2//iceberg-bundled-guava-1.7.2.jar
-iceberg-common/1.7.2//iceberg-common-1.7.2.jar
-iceberg-core/1.7.2//iceberg-core-1.7.2.jar
-iceberg-data/1.7.2//iceberg-data-1.7.2.jar
-iceberg-hive-metastore/1.7.2//iceberg-hive-metastore-1.7.2.jar
-iceberg-orc/1.7.2//iceberg-orc-1.7.2.jar
-iceberg-parquet/1.7.2//iceberg-parquet-1.7.2.jar
-iceberg-spark-3.3_2.12/1.7.2//iceberg-spark-3.3_2.12-1.7.2.jar
-iceberg-spark-extensions-3.3_2.12/1.7.2//iceberg-spark-extensions-3.3_2.12-1.7.2.jar
+httpcore5-h2/5.3.1//httpcore5-h2-5.3.1.jar
+httpcore5/5.3.1//httpcore5-5.3.1.jar
+iceberg-aliyun/1.8.1//iceberg-aliyun-1.8.1.jar
+iceberg-api/1.8.1//iceberg-api-1.8.1.jar
+iceberg-arrow/1.8.1//iceberg-arrow-1.8.1.jar
+iceberg-aws/1.8.1//iceberg-aws-1.8.1.jar
+iceberg-bundled-guava/1.8.1//iceberg-bundled-guava-1.8.1.jar
+iceberg-common/1.8.1//iceberg-common-1.8.1.jar
+iceberg-core/1.8.1//iceberg-core-1.8.1.jar
+iceberg-data/1.8.1//iceberg-data-1.8.1.jar
+iceberg-hive-metastore/1.8.1//iceberg-hive-metastore-1.8.1.jar
+iceberg-orc/1.8.1//iceberg-orc-1.8.1.jar
+iceberg-parquet/1.8.1//iceberg-parquet-1.8.1.jar
+iceberg-spark-3.3_2.12/1.8.1//iceberg-spark-3.3_2.12-1.8.1.jar
+iceberg-spark-extensions-3.3_2.12/1.8.1//iceberg-spark-extensions-3.3_2.12-1.8.1.jar
icu4j/69.1//icu4j-69.1.jar
identity-spi/2.24.12//identity-spi-2.24.12.jar
ipaddress/5.5.1//ipaddress-5.5.1.jar
diff --git a/dev/deps/dependencies-hadoop-3-spark-3.5
b/dev/deps/dependencies-hadoop-3-spark-3.5
index 5a1400e08..c5f4fe97d 100644
--- a/dev/deps/dependencies-hadoop-3-spark-3.5
+++ b/dev/deps/dependencies-hadoop-3-spark-3.5
@@ -128,23 +128,23 @@ http-auth-spi/2.24.12//http-auth-spi-2.24.12.jar
http-auth/2.24.12//http-auth-2.24.12.jar
http-client-spi/2.24.12//http-client-spi-2.24.12.jar
httpclient/4.5.14//httpclient-4.5.14.jar
-httpclient5/5.4//httpclient5-5.4.jar
+httpclient5/5.4.1//httpclient5-5.4.1.jar
httpcore/4.4.16//httpcore-4.4.16.jar
-httpcore5-h2/5.3//httpcore5-h2-5.3.jar
-httpcore5/5.3//httpcore5-5.3.jar
-iceberg-aliyun/1.7.2//iceberg-aliyun-1.7.2.jar
-iceberg-api/1.7.2//iceberg-api-1.7.2.jar
-iceberg-arrow/1.7.2//iceberg-arrow-1.7.2.jar
-iceberg-aws/1.7.2//iceberg-aws-1.7.2.jar
-iceberg-bundled-guava/1.7.2//iceberg-bundled-guava-1.7.2.jar
-iceberg-common/1.7.2//iceberg-common-1.7.2.jar
-iceberg-core/1.7.2//iceberg-core-1.7.2.jar
-iceberg-data/1.7.2//iceberg-data-1.7.2.jar
-iceberg-hive-metastore/1.7.2//iceberg-hive-metastore-1.7.2.jar
-iceberg-orc/1.7.2//iceberg-orc-1.7.2.jar
-iceberg-parquet/1.7.2//iceberg-parquet-1.7.2.jar
-iceberg-spark-3.5_2.12/1.7.2//iceberg-spark-3.5_2.12-1.7.2.jar
-iceberg-spark-extensions-3.5_2.12/1.7.2//iceberg-spark-extensions-3.5_2.12-1.7.2.jar
+httpcore5-h2/5.3.1//httpcore5-h2-5.3.1.jar
+httpcore5/5.3.1//httpcore5-5.3.1.jar
+iceberg-aliyun/1.8.1//iceberg-aliyun-1.8.1.jar
+iceberg-api/1.8.1//iceberg-api-1.8.1.jar
+iceberg-arrow/1.8.1//iceberg-arrow-1.8.1.jar
+iceberg-aws/1.8.1//iceberg-aws-1.8.1.jar
+iceberg-bundled-guava/1.8.1//iceberg-bundled-guava-1.8.1.jar
+iceberg-common/1.8.1//iceberg-common-1.8.1.jar
+iceberg-core/1.8.1//iceberg-core-1.8.1.jar
+iceberg-data/1.8.1//iceberg-data-1.8.1.jar
+iceberg-hive-metastore/1.8.1//iceberg-hive-metastore-1.8.1.jar
+iceberg-orc/1.8.1//iceberg-orc-1.8.1.jar
+iceberg-parquet/1.8.1//iceberg-parquet-1.8.1.jar
+iceberg-spark-3.5_2.12/1.8.1//iceberg-spark-3.5_2.12-1.8.1.jar
+iceberg-spark-extensions-3.5_2.12/1.8.1//iceberg-spark-extensions-3.5_2.12-1.8.1.jar
icu4j/69.1//icu4j-69.1.jar
identity-spi/2.24.12//identity-spi-2.24.12.jar
ipaddress/5.5.1//ipaddress-5.5.1.jar
diff --git a/docs/engines/flink/flink-cdc-ingestion.md
b/docs/engines/flink/flink-cdc-ingestion.md
index 163fd5759..5241dd770 100644
--- a/docs/engines/flink/flink-cdc-ingestion.md
+++ b/docs/engines/flink/flink-cdc-ingestion.md
@@ -34,7 +34,7 @@ The following example will show how [MySQL
CDC](https://nightlies.apache.org/fli
**Requirements**
-Please add [Flink SQL Connector MySQL
CDC](https://mvnrepository.com/artifact/org.apache.flink/flink-connector-mysql-cdc)
and
[Iceberg](https://repo1.maven.org/maven2/org/apache/iceberg/iceberg-flink-1.18/1.7.2/iceberg-flink-1.18-1.7.2.jar)
Jars to the lib directory of the Flink engine package.
+Please add [Flink SQL Connector MySQL
CDC](https://mvnrepository.com/artifact/org.apache.flink/flink-connector-mysql-cdc)
and
[Iceberg](https://repo1.maven.org/maven2/org/apache/iceberg/iceberg-flink-1.18/1.8.1/iceberg-flink-1.18-1.8.1.jar)
Jars to the lib directory of the Flink engine package.
```sql
CREATE TABLE products (
@@ -114,7 +114,7 @@ The following example will show how to write CDC data from
multiple MySQL tables
**Requirements**
Please add [Flink Connector MySQL
CDC](https://mvnrepository.com/artifact/org.apache.flink/flink-connector-mysql-cdc/3.1.1)
-and
[Iceberg](https://mvnrepository.com/artifact/org.apache.iceberg/iceberg-flink-1.18/1.7.2)
dependencies to your
+and
[Iceberg](https://mvnrepository.com/artifact/org.apache.iceberg/iceberg-flink-1.18/1.8.1)
dependencies to your
Maven project's pom.xml file.
```java
diff --git a/pom.xml b/pom.xml
index 9ea3287f3..a0430402c 100644
--- a/pom.xml
+++ b/pom.xml
@@ -103,7 +103,7 @@
<maven-clean-plugin.version>3.3.2</maven-clean-plugin.version>
<maven-checkstyle-plugin.version>3.3.1</maven-checkstyle-plugin.version>
- <iceberg.version>1.7.2</iceberg.version>
+ <iceberg.version>1.8.1</iceberg.version>
<paimon.version>1.2.0</paimon.version>
<hive.version>3.1.3</hive.version>
<hadoop.version>3.4.3</hadoop.version>
