>From Michael Blow <[email protected]>: Michael Blow has uploaded this change for review. ( https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/21971?usp=email )
Change subject: [NO ISSUE][EXT] Advance iceberg to 1.12.0 ...................................................................... [NO ISSUE][EXT] Advance iceberg to 1.12.0 - user model changes: no - storage format changes: no - interface changes: no Details: Move iceberg from 1.11.0 to 1.12.0, and nessie with it from 0.107.5 to 0.108.8 (the version iceberg-nessie 1.12.0 declares). The forked org.apache.iceberg.gcp.gcs.PrefixedStorage is identical upstream at both tags, so iceberg-gcs-fork.version moves with no re-copy. 1.12.0 removes GenericAppenderFactory. The two tests that wrote Iceberg data files with it now use FormatModelRegistry.dataWriteBuilder, whose DataWriter yields the DataFile directly. Ext-ref: MB-72431 Generated-by: Claude Opus 5.5 <[email protected]> Change-Id: If2255f3d9bf6d45ef19f16b451c889d8abcff3dd --- M asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/aws/AwsS3ExternalDatasetTest.java M asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/SqlppHdfsExecutionTest.java M asterixdb/pom.xml 3 files changed, 23 insertions(+), 23 deletions(-) git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/71/21971/1 diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/aws/AwsS3ExternalDatasetTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/aws/AwsS3ExternalDatasetTest.java index d341800..300db67 100644 --- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/aws/AwsS3ExternalDatasetTest.java +++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/aws/AwsS3ExternalDatasetTest.java @@ -62,19 +62,18 @@ import org.apache.hadoop.fs.Path; import org.apache.hyracks.control.nc.NodeControllerService; import org.apache.iceberg.DataFile; -import org.apache.iceberg.DataFiles; import org.apache.iceberg.FileFormat; import org.apache.iceberg.PartitionSpec; import org.apache.iceberg.Schema; import org.apache.iceberg.Table; import org.apache.iceberg.TableProperties; import org.apache.iceberg.Tables; -import org.apache.iceberg.data.GenericAppenderFactory; import org.apache.iceberg.data.GenericRecord; import org.apache.iceberg.data.Record; -import org.apache.iceberg.hadoop.HadoopInputFile; +import org.apache.iceberg.encryption.EncryptedFiles; +import org.apache.iceberg.formats.FormatModelRegistry; import org.apache.iceberg.hadoop.HadoopTables; -import org.apache.iceberg.io.FileAppender; +import org.apache.iceberg.io.DataWriter; import org.apache.iceberg.relocated.com.google.common.base.Preconditions; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; @@ -227,14 +226,15 @@ FileFormat fileFormat = FileFormat.fromFileName(filename); Preconditions.checkNotNull(fileFormat, "Cannot determine format for file: %s", filename); - FileAppender<Record> fileAppender = new GenericAppenderFactory(AwsS3ExternalDatasetTest.SCHEMA) - .newAppender(fromPath(path, CONF), fileFormat); - try (FileAppender<Record> appender = fileAppender) { - appender.addAll(records); + DataWriter<Record> writer = FormatModelRegistry + .<Record, Object> dataWriteBuilder(fileFormat, Record.class, + EncryptedFiles.plainAsEncryptedOutput(fromPath(path, CONF))) + .schema(AwsS3ExternalDatasetTest.SCHEMA).spec(PartitionSpec.unpartitioned()).build(); + try (DataWriter<Record> w = writer) { + w.write(records); } - return DataFiles.builder(PartitionSpec.unpartitioned()).withInputFile(HadoopInputFile.fromPath(path, CONF)) - .withMetrics(fileAppender.metrics()).build(); + return writer.toDataFile(); } private static void prepareIcebergConfiguration() { diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/SqlppHdfsExecutionTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/SqlppHdfsExecutionTest.java index 412fbc1..d3b04c1 100644 --- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/SqlppHdfsExecutionTest.java +++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/SqlppHdfsExecutionTest.java @@ -50,19 +50,18 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.iceberg.DataFile; -import org.apache.iceberg.DataFiles; import org.apache.iceberg.FileFormat; import org.apache.iceberg.PartitionSpec; import org.apache.iceberg.Schema; import org.apache.iceberg.Table; import org.apache.iceberg.TableProperties; import org.apache.iceberg.Tables; -import org.apache.iceberg.data.GenericAppenderFactory; import org.apache.iceberg.data.GenericRecord; import org.apache.iceberg.data.Record; -import org.apache.iceberg.hadoop.HadoopInputFile; +import org.apache.iceberg.encryption.EncryptedFiles; +import org.apache.iceberg.formats.FormatModelRegistry; import org.apache.iceberg.hadoop.HadoopTables; -import org.apache.iceberg.io.FileAppender; +import org.apache.iceberg.io.DataWriter; import org.apache.iceberg.relocated.com.google.common.base.Preconditions; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; @@ -98,14 +97,15 @@ FileFormat fileFormat = FileFormat.fromFileName(filename); Preconditions.checkNotNull(fileFormat, "Cannot determine format for file: %s", filename); - FileAppender<Record> fileAppender = - new GenericAppenderFactory(schema).newAppender(fromPath(path, CONF), fileFormat); - try (FileAppender<Record> appender = fileAppender) { - appender.addAll(records); + DataWriter<Record> writer = FormatModelRegistry + .<Record, Object> dataWriteBuilder(fileFormat, Record.class, + EncryptedFiles.plainAsEncryptedOutput(fromPath(path, CONF))) + .schema(schema).spec(PartitionSpec.unpartitioned()).build(); + try (DataWriter<Record> w = writer) { + w.write(records); } - return DataFiles.builder(PartitionSpec.unpartitioned()).withInputFile(HadoopInputFile.fromPath(path, CONF)) - .withMetrics(fileAppender.metrics()).build(); + return writer.toDataFile(); } private static void setUpIcebergData() { diff --git a/asterixdb/pom.xml b/asterixdb/pom.xml index 4a30c50..f9371bc 100644 --- a/asterixdb/pom.xml +++ b/asterixdb/pom.xml @@ -123,7 +123,7 @@ <io.opencensus.version>0.31.1</io.opencensus.version> <protobuf-java.version>4.36.2</protobuf-java.version> - <icebergjavasdk.version>1.11.0</icebergjavasdk.version> + <icebergjavasdk.version>1.12.0</icebergjavasdk.version> <!-- The org.apache.iceberg.gcp.gcs package is forked into asterix-external-data, under util/google/iceberg/fileio. This records the release those files were @@ -134,10 +134,10 @@ The marker below is a one-line comment adjacent to the property on purpose: the updater looks only one line back for it. --> <!-- @pinned-with: ${icebergjavasdk.version} --> - <iceberg-gcs-fork.version>1.11.0</iceberg-gcs-fork.version> + <iceberg-gcs-fork.version>1.12.0</iceberg-gcs-fork.version> <!-- @derived-from: org.apache.iceberg:iceberg-nessie:${icebergjavasdk.version}#dep:nessie-client --> - <nessieproject.version>0.107.5</nessieproject.version> + <nessieproject.version>0.108.8</nessieproject.version> <implementation.title>Apache AsterixDB - ${project.name}</implementation.title> <implementation.url>https://asterixdb.apache.org/</implementation.url> -- To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/21971?usp=email To unsubscribe, or for help writing mail filters, visit https://asterix-gerrit.ics.uci.edu/settings?usp=email Gerrit-MessageType: newchange Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Change-Id: If2255f3d9bf6d45ef19f16b451c889d8abcff3dd Gerrit-Change-Number: 21971 Gerrit-PatchSet: 1 Gerrit-Owner: Michael Blow <[email protected]>
