This is an automated email from the ASF dual-hosted git repository.
sungwy pushed a change to branch sungwy-operation
in repository https://gitbox.apache.org/repos/asf/iceberg.git
from dde05cdf7d DOCS: Explicitly specify `operation` as a _required_ field
of `summary` field
add d4f0d7e803 Flink: Add IcebergSinkBuilder interface allowed unification
of most of operations on FlinkSink and IcebergSink Builders (#11305)
add b8c2b20237 Build: Bump parquet from 1.13.1 to 1.14.3 (#11264)
add de48a74263 Build: Bump com.palantir.baseline:gradle-baseline-java
(#11362)
add 181476471a Build: Bump com.google.errorprone:error_prone_annotations
(#11360)
add 0640a38b6b Build: Bump software.amazon.awssdk:bom from 2.28.21 to
2.28.26 (#11359)
add ce75f52719 Build: Bump com.google.cloud:libraries-bom from 26.48.0 to
26.49.0 (#11363)
add c16cefa501 Core: Move deleteRemovedMetadataFiles(..) to CatalogUtil
(#11352)
add d0a7ff915e Arrow: Fix indexing in Parquet dictionary encoded values
readers (#11247)
add a19896658c Spark 3.5: Update Spark to use planned Avro reads (#11299)
add a8ec43d975 Core, Spark 3.5: Remove dangling deletes as part of
RewriteDataFilesAction (#9724)
add 97946088a8 Spark: Randomize view/function names in testing (#11381)
add 60181f9a76 Spark 3.4: Randomize view/function names in testing (#11382)
add 9c0a806846 Spark 3.4: Action to remove dangling deletes (#11377)
add 043757c0a1 Spark 3.5: Reset Spark Conf for each test in
TestCompressionSettings (#11333)
add 02a988b09a Spec: Adds Row Lineage (#11130)
add 4850b622c7 AWS: Support S3 directory bucket listing (#11021)
add 1cb88a64f8 Core: Add LoadCredentialsResponse class/parser (#11339)
add 35a02d035e OpenAPI: Add endpoint for refreshing vended credentials
(#11281)
add 6ba1a1f578 AWS: Use testcontainers-minio instead of S3Mock (#11349)
add 12ac3ee5d3 Kafka Connect: Include third party licenses and notices in
distribution (#10829)
add fdc2c223ef Deprecate iceberg-pig (#11379)
add 6b04a6d000 Core: Track data files by spec id instead of full
PartitionSpec (#11323)
add 32e9f40468 Spark 3.5: Don't change table distribution when only
altering local order (#10774)
add 7738e1d722 Spec: Fix table of content generation (#11067)
add 9ecd97bf45 [KafkaConnect] Fix RecordConverter for UUID and Fixed Types
(#11346)
add 7ad11b2df1 Core: Snapshot `summary` map must have `operation` key
(#11354)
add 2b55fef7cc Core: Update TableMetadataParser to ensure all streams
closed (#11220)
add e3bbcac852 Build: Bump Spark 3.4 to 3.4.4 (#11366)
add b4d178fa05 Build: Bump junit from 5.11.1 to 5.11.3 (#11401)
add 9565b9c401 Build: Bump software.amazon.awssdk:bom from 2.28.26 to
2.29.1 (#11400)
add 681b09ddc9 Core: Move Javadoc about commit retries to SnapshotProducer
(#10995)
add 9fc9c052b5 Build: Bump junit-platform from 1.11.2 to 1.11.3 (#11402)
add 68a7102073 Build: Bump org.xerial:sqlite-jdbc from 3.46.1.3 to
3.47.0.0 (#11407)
add 48acaadcf9 Build: Bump
software.amazon.s3.accessgrants:aws-s3-accessgrants-java-plugin (#11405)
add c3191eecd9 Build: Bump net.snowflake:snowflake-jdbc from 3.19.0 to
3.19.1 (#11406)
add 6e911e0ea3 Build: Bump testcontainers from 1.20.2 to 1.20.3 (#11404)
add d10e67d422 Build: Bump com.google.errorprone:error_prone_annotations
(#11403)
add b4b0bdc543 Build: Bump mkdocs-macros-plugin from 1.2.0 to 1.3.7
(#11399)
add 696204392c Build: Bump mkdocs-material from 9.5.39 to 9.5.42 (#11398)
add 86a8560315 Flink: Fix disabling flaky range distribution bucketing
tests (#11410)
add a6503f573c Bump Azurite to the latest version (#11411)
add a28ebf748c Build: Bump datamodel-code-generator from 0.26.1 to 0.26.2
(#11356)
add 6c58f5bb7f Revert "Core: Snapshot `summary` map must have `operation`
key (#11354)" (#11409)
add e013c67f24 Aliyun: Remove spring-boot dependency (#11291)
add 7ac617a5a8 Build: Bump com.azure:azure-sdk-bom from 1.2.25 to 1.2.28
(#11267)
add 47eac5221b Spark: Flaky test due temp directory (#10811)
add 1e3ee1e4e8 Core: Add portable Roaring bitmap for row positions (#11372)
add 602c2b2dbe Flink 1.20: Update Flink to use planned Avro reads (#11386)
add 740d4e7b1c open-api: Fix `testFixtures` dependencies (#11422)
add 469c5560b2 Core: use ManifestFiles.open when possible (#11414)
add 5359bea71b GCS: Refresh vended credentials (#11282)
add bedc71167b Docs: Add 21 blogs / fix one broken link (#11424)
add 9e895cb6df AWS: Refresh vended credentials (#11389)
add 7c39086193 Build: Bump Hadoop to 3.4.1 (#11428)
add dec84c0549 Core: Remove credentials from LoadViewResponse (#11432)
add f4b36a5e55 OpenAPI: Remove credentials from LoadViewResult (#11433)
add 91e04c9c88 API: Add compatibility checks for Schemas with default
values (#11434)
add 57fb6d5658 Doc: Update rewrite data files spark procedure (#11396)
add ea61ee46db Docs: warn `parallelism > 1` doesn't work for migration
procedures (#11417)
add 1d4df34d7b Core: Log retry sleep time (#11413)
add caf424a373 Core: Use RoaringPositionBitmap in position index (#11441)
add fe23584fc3 Core: Add validation for table commit properties (#11437)
add 8b4ebc66e1 Core: Add cardinality to PositionDeleteIndex (#11442)
add b9ebc71fbc Puffin: Add deletion-vector-v1 blob type (#11238)
add d368a5f844 Spec: Add deletion vectors to the table spec (#11240)
add d9b9768766 API, Core: Add data file reference to DeleteFile (#11443)
add e47fa6a581 Build: Bump software.amazon.awssdk:bom from 2.29.1 to
2.29.6 (#11454)
add 29ee906496 Build: Bump com.gradleup.shadow:shadow-gradle-plugin from
8.3.3 to 8.3.5 (#11452)
add 7ffb6a3de7 Build: Bump com.google.cloud:libraries-bom from 26.49.0 to
26.50.0 (#11451)
add aa27d9057d Build: Bump org.apache.httpcomponents.client5:httpclient5
(#11450)
add c7f0f80e03 Build: Bump com.azure:azure-sdk-bom from 1.2.28 to 1.2.29
(#11453)
add 0669bcbad7 Flink: Maintenance - TableManager + ExpireSnapshots (#11144)
add ec0eef45eb Build: Bump mkdocs-material from 9.5.42 to 9.5.43 (#11455)
add 8357f65d23 Build: Bump net.snowflake:snowflake-jdbc from 3.19.1 to
3.20.0 (#11447)
add 329846875d Build: Bump kafka from 3.8.0 to 3.8.1 (#11449)
add 9dcf0d3d32 Build: Bump jackson-bom from 2.18.0 to 2.18.1 (#11448)
add af5be32fbe Core: Fix generated position delete file spec (#11458)
add ec269ee3ec API, Core: Add content offset and size to DeleteFile
(#11446)
add 7cc16fa94d Revert "Build: Bump parquet from 1.13.1 to 1.14.3 (#11264)"
(#11462)
add d0cca384a0 Spark 3.5: Preserve data file reference during manifest
rewrites (#11457)
add 43b2f7d007 Core: Make PositionDeleteIndex serializable (#11463)
add 592b3b1c51 Spark 3.5: Preserve content offset and size during manifest
rewrites (#11469)
add 20e0e3dae5 Spark 3.5: Fix flaky test due to temp directory not empty
during delete (#11470)
add 67ee082595 Core, Data, Flink, Spark: Improve tableDir initialization
for tests (#11460)
add 5bd314bdf6 Core: Support DVs in DeleteFileIndex (#11467)
add 549674b3fc Core: Adapt commit, scan, and snapshot stats for DVs
(#11464)
add ad24d4bf85 Spark: Synchronously merge new position deletes with old
deletes (#11273)
add 9be7f00dd6 Fix ADLSLocation file parsing (#11395)
add 2ffd3b02a2 open-api: Build runtime jar for test fixture (#11279)
add 7938403a43 Core, Puffin: Add DV file writer (#11476)
add 1e82c476df Flink: Fix config key typo in error message of
SplitComparators (#11482)
add 11e7230b67 API: Removes Explicit Parameterization of Schema Tests
(#11444)
add 5c8a5d68d4 Docs: Fix verifying release candidate with Spark and Flink
(#11461)
add 3da64d37d0 Flink: Port #11144 to v1.19 (#11473)
add fff9ec3bbc Docs: Fix format of verifying release candidate with Flink
(#11487)
add 166edc7298 Core: Support DVs in DeleteLoader (#11481)
add dda62154e6 Infra: Update DOAP.RDF for Apache Iceberg 1.7.0 (#11492)
add 6a16340947 Docs: Site Update for 1.7.0 Release (#11494)
add bbc0d9aad5 Infra: Add 1.7.0 to issue template (#11491)
add df0917d553 Build: Let revapi compare against 1.7.0 (#11490)
add cd25937dc7 Docs: Adds Release notes for 1.6.1 (#11500)
add dfee4cb35e Docs: Fixes Release Formatting for 1.7.0 Release Notes
(#11499)
add 978cf1a5ae Merge branch 'main' into sungwy-operation
No new revisions were added by this update.
Summary of changes:
.github/ISSUE_TEMPLATE/iceberg_bug_report.yml | 3 +-
LICENSE | 1 +
.../org/apache/iceberg/aliyun/TestUtility.java | 2 +-
.../iceberg/aliyun/oss/TestOSSOutputStream.java | 2 +-
.../iceberg/aliyun/oss/mock/AliyunOSSMock.java | 569 ++++++
.../iceberg/aliyun/oss/mock/AliyunOSSMockApp.java | 158 --
.../aliyun/oss/mock/AliyunOSSMockExtension.java | 23 +-
.../oss/mock/AliyunOSSMockLocalController.java | 522 ------
.../aliyun/oss/mock/AliyunOSSMockLocalStore.java | 14 +-
.../java/org/apache/iceberg/AddedRowsScanTask.java | 3 +-
api/src/main/java/org/apache/iceberg/DataFile.java | 22 +-
.../main/java/org/apache/iceberg/DeleteFile.java | 33 +
.../apache/iceberg/DeletedDataFileScanTask.java | 3 +-
.../org/apache/iceberg/DeletedRowsScanTask.java | 7 +-
.../main/java/org/apache/iceberg/FileFormat.java | 1 +
.../main/java/org/apache/iceberg/FileScanTask.java | 3 +-
api/src/main/java/org/apache/iceberg/Schema.java | 44 +-
.../apache/iceberg/actions/ActionsProvider.java | 6 +
.../iceberg/actions/RemoveDanglingDeleteFiles.java | 20 +-
.../apache/iceberg/actions/RewriteDataFiles.java | 16 +
.../org/apache/iceberg/util/DeleteFileSet.java | 7 +-
.../java/org/apache/iceberg/util/ScanTaskUtil.java | 50 +
.../test/java/org/apache/iceberg/TestHelpers.java | 3 +
.../test/java/org/apache/iceberg/TestSchema.java | 160 ++
.../org/apache/iceberg/util/TestScanTaskUtil.java | 56 +
...orizedDictionaryEncodedParquetValuesReader.java | 22 +-
.../org/apache/iceberg/aws/AwsIntegTestUtil.java | 47 +-
.../org/apache/iceberg/aws/glue/GlueTestBase.java | 2 +-
.../iceberg/aws/s3/TestS3FileIOIntegration.java | 73 +-
.../iceberg/aws/s3/TestS3MultipartUpload.java | 2 +-
.../apache/iceberg/aws/AwsClientProperties.java | 34 +-
.../java/org/apache/iceberg/aws/s3/S3FileIO.java | 8 +-
.../apache/iceberg/aws/s3/S3FileIOProperties.java | 44 +
.../main/java/org/apache/iceberg/aws/s3/S3URI.java | 33 +
.../iceberg/aws/s3/VendedCredentialsProvider.java | 138 ++
.../iceberg/aws/AwsClientPropertiesTest.java | 29 +
.../apache/iceberg/aws/TestS3FileIOProperties.java | 8 +
.../org/apache/iceberg/aws/s3/MinioContainer.java | 68 -
.../java/org/apache/iceberg/aws/s3/MinioUtil.java | 65 +
.../org/apache/iceberg/aws/s3/TestMinioUtil.java | 75 +
.../org/apache/iceberg/aws/s3/TestS3FileIO.java | 116 +-
.../apache/iceberg/aws/s3/TestS3InputStream.java | 13 +-
.../apache/iceberg/aws/s3/TestS3OutputStream.java | 13 +-
.../java/org/apache/iceberg/aws/s3/TestS3URI.java | 43 +
.../aws/s3/TestVendedCredentialsProvider.java | 323 ++++
.../iceberg/aws/s3/signer/S3SignerServlet.java | 6 +-
.../iceberg/aws/s3/signer/TestS3RestSigner.java | 18 +-
.../apache/iceberg/azure/adlsv2/ADLSLocation.java | 13 +-
.../iceberg/azure/adlsv2/ADLSLocationTest.java | 24 +-
.../iceberg/azure/adlsv2/AzuriteContainer.java | 2 +-
build.gradle | 69 +-
.../deletes/RoaringPositionBitmapBenchmark.java | 162 ++
.../src/main/java/org/apache/iceberg/BaseFile.java | 47 +-
.../java/org/apache/iceberg/BaseFileScanTask.java | 9 +-
.../java/org/apache/iceberg/BaseFilesTable.java | 10 +-
.../iceberg/BaseMetastoreTableOperations.java | 49 +-
.../src/main/java/org/apache/iceberg/BaseScan.java | 3 +
.../main/java/org/apache/iceberg/CatalogUtil.java | 45 +
.../java/org/apache/iceberg/ContentFileParser.java | 27 +-
.../java/org/apache/iceberg/DeleteFileIndex.java | 62 +-
.../main/java/org/apache/iceberg/FastAppend.java | 8 +-
.../main/java/org/apache/iceberg/FileMetadata.java | 36 +-
.../java/org/apache/iceberg/GenericDataFile.java | 5 +-
.../java/org/apache/iceberg/GenericDeleteFile.java | 10 +-
.../main/java/org/apache/iceberg/MergeAppend.java | 8 +-
.../apache/iceberg/MergingSnapshotProducer.java | 23 +-
.../java/org/apache/iceberg/PropertiesUpdate.java | 9 +-
.../main/java/org/apache/iceberg/ScanSummary.java | 3 +-
.../java/org/apache/iceberg/SnapshotProducer.java | 21 +
.../java/org/apache/iceberg/SnapshotSummary.java | 30 +-
.../java/org/apache/iceberg/StreamingDelete.java | 8 +-
.../java/org/apache/iceberg/TableMetadata.java | 6 +
.../org/apache/iceberg/TableMetadataParser.java | 12 +-
.../main/java/org/apache/iceberg/V2Metadata.java | 9 +-
.../main/java/org/apache/iceberg/V3Metadata.java | 23 +-
.../actions/BaseRemoveDanglingDeleteFiles.java | 20 +-
.../iceberg/actions/BaseRewriteDataFiles.java | 6 +
.../iceberg/avro/AvroWithPartnerVisitor.java | 2 +-
.../java/org/apache/iceberg/avro/ValueReaders.java | 4 +-
.../apache/iceberg/deletes/BaseDVFileWriter.java | 194 ++
.../iceberg/deletes/BitmapPositionDeleteIndex.java | 150 +-
.../org/apache/iceberg/deletes/DVFileWriter.java | 46 +
.../iceberg/deletes/PositionDeleteIndex.java | 26 +
.../iceberg/deletes/RoaringPositionBitmap.java | 318 ++++
.../deletes/SortingPositionOnlyDeleteWriter.java | 6 +-
.../iceberg/hadoop/HadoopTableOperations.java | 40 +-
.../java/org/apache/iceberg/io/BaseTaskWriter.java | 3 +-
.../org/apache/iceberg/io/ClusteredWriter.java | 5 +-
.../java/org/apache/iceberg/io/FanoutWriter.java | 3 +-
.../java/org/apache/iceberg/io/StructCopy.java | 8 +-
.../iceberg/metrics/CommitMetricsResult.java | 16 +
.../iceberg/metrics/CommitMetricsResultParser.java | 12 +
.../org/apache/iceberg/metrics/ScanMetrics.java | 6 +
.../apache/iceberg/metrics/ScanMetricsResult.java | 7 +
.../iceberg/metrics/ScanMetricsResultParser.java | 6 +
.../apache/iceberg/metrics/ScanMetricsUtil.java | 10 +-
.../java/org/apache/iceberg/puffin/Puffin.java | 5 +
.../org/apache/iceberg/puffin/PuffinWriter.java | 16 +-
.../apache/iceberg/puffin/StandardBlobTypes.java | 3 +
.../org/apache/iceberg/rest/RESTSerializers.java | 29 +-
.../rest/responses/LoadCredentialsResponse.java | 29 +-
.../responses/LoadCredentialsResponseParser.java | 77 +
.../rest/responses/LoadTableResponseParser.java | 8 +-
.../iceberg/rest/responses/LoadViewResponse.java | 8 -
.../rest/responses/LoadViewResponseParser.java | 22 -
.../org/apache/iceberg/util/ContentFileUtil.java | 27 +
.../java/org/apache/iceberg/util/PropertyUtil.java | 50 +
.../org/apache/iceberg/util/StructLikeUtil.java | 67 +
.../org/apache/iceberg/util/TableScanUtil.java | 4 +-
.../main/java/org/apache/iceberg/util/Tasks.java | 6 +-
.../apache/iceberg/DeleteFileIndexTestBase.java | 57 +-
.../org/apache/iceberg/FileGenerationUtil.java | 47 +-
.../org/apache/iceberg/FilterFilesTestBase.java | 10 +-
.../test/java/org/apache/iceberg/ScanTestBase.java | 15 +-
.../src/test/java/org/apache/iceberg/TestBase.java | 16 +
.../org/apache/iceberg/TestContentFileParser.java | 65 +-
.../org/apache/iceberg/TestCreateTransaction.java | 29 -
.../java/org/apache/iceberg/TestFastAppend.java | 25 +
.../org/apache/iceberg/TestManifestEncryption.java | 5 +-
.../org/apache/iceberg/TestManifestReader.java | 46 +-
.../apache/iceberg/TestManifestWriterVersions.java | 3 +
.../org/apache/iceberg/TestMetadataTableScans.java | 2 -
...stMetadataTableScansWithPartitionEvolution.java | 4 -
.../test/java/org/apache/iceberg/TestMetrics.java | 10 +-
.../java/org/apache/iceberg/TestMetricsModes.java | 11 +-
.../java/org/apache/iceberg/TestOverwrite.java | 5 -
.../iceberg/TestOverwriteWithValidation.java | 4 -
.../TestPartitionSpecBuilderCaseSensitivity.java | 14 -
.../org/apache/iceberg/TestPartitionSpecInfo.java | 12 +-
.../java/org/apache/iceberg/TestPartitioning.java | 12 +-
.../org/apache/iceberg/TestReplacePartitions.java | 11 -
.../org/apache/iceberg/TestReplaceTransaction.java | 7 -
.../org/apache/iceberg/TestSnapshotSummary.java | 72 +
.../java/org/apache/iceberg/TestSortOrder.java | 13 +-
.../java/org/apache/iceberg/TestSplitPlanning.java | 3 -
.../java/org/apache/iceberg/TestTableMetadata.java | 3 +-
.../apache/iceberg/TestTimestampPartitions.java | 5 -
.../java/org/apache/iceberg/TestTransaction.java | 18 +
.../deletes/TestBitmapPositionDeleteIndex.java | 105 ++
.../iceberg/deletes/TestRoaringPositionBitmap.java | 515 +++++
.../metrics/TestCommitMetricsResultParser.java | 12 +
.../metrics/TestScanMetricsResultParser.java | 7 +
.../iceberg/metrics/TestScanReportParser.java | 6 +
.../TestLoadCredentialsResponseParser.java | 112 ++
.../rest/responses/TestLoadViewResponseParser.java | 111 --
.../org/apache/iceberg/util/TestSnapshotUtil.java | 2 -
.../org/apache/iceberg/util/TestTableScanUtil.java | 16 +
.../org/apache/iceberg/deletes/64map32bitvals.bin | Bin 0 -> 48 bytes
.../org/apache/iceberg/deletes/64mapempty.bin | Bin 0 -> 8 bytes
.../org/apache/iceberg/deletes/64maphighvals.bin | Bin 0 -> 1086 bytes
.../org/apache/iceberg/deletes/64mapspreadvals.bin | Bin 0 -> 408 bytes
.../deletes/all-container-types-position-index.bin | Bin 0 -> 94 bytes
.../iceberg/deletes/empty-position-index.bin | Bin 0 -> 20 bytes
.../small-alternating-values-position-index.bin | Bin 0 -> 50 bytes
.../small-and-large-values-position-index.bin | Bin 0 -> 56 bytes
.../org/apache/iceberg/data/BaseDeleteLoader.java | 82 +
.../java/org/apache/iceberg/data/DeleteLoader.java | 5 +-
.../java/org/apache/iceberg/TestSplitScan.java | 6 +-
.../iceberg/data/TestGenericReaderDeletes.java | 8 +-
.../org/apache/iceberg/io/TestAppenderFactory.java | 4 -
.../org/apache/iceberg/io/TestBaseTaskWriter.java | 3 -
.../java/org/apache/iceberg/io/TestDVWriters.java | 342 ++++
.../apache/iceberg/io/TestFileWriterFactory.java | 4 -
.../apache/iceberg/io/TestPartitioningWriters.java | 4 -
.../iceberg/io/TestPositionDeltaWriters.java | 4 -
.../apache/iceberg/io/TestRollingFileWriters.java | 4 -
.../iceberg/io/TestTaskEqualityDeltaWriter.java | 3 -
.../org/apache/iceberg/io/TestWriterMetrics.java | 12 +-
deploy.gradle | 1 +
doap.rdf | 6 +-
docs/docs/spark-procedures.md | 20 +-
.../apache/iceberg/flink/sink/CommitSummary.java | 4 +-
.../flink/sink/IcebergStreamWriterMetrics.java | 3 +-
.../flink/source/split/SplitComparators.java | 4 +-
.../iceberg/flink/sink/TestDeltaTaskWriter.java | 3 -
.../flink/sink/TestIcebergFilesCommitter.java | 3 -
.../flink/source/TestStreamingMonitorFunction.java | 3 -
.../flink/source/TestStreamingReaderOperator.java | 3 -
.../TestFileSequenceNumberBasedSplitAssigner.java | 2 +-
.../flink/maintenance/api/ExpireSnapshots.java | 125 ++
.../flink/maintenance/api}/JdbcLockFactory.java | 7 +-
.../maintenance/api/MaintenanceTaskBuilder.java | 227 +++
.../flink/maintenance/api/TableMaintenance.java | 330 ++++
.../maintenance/{operator => api}/TaskResult.java | 7 +-
.../iceberg/flink/maintenance/api}/Trigger.java | 31 +-
.../{operator => api}/TriggerLockFactory.java | 9 +-
.../maintenance/operator/DeleteFilesProcessor.java | 130 ++
.../operator/ExpireSnapshotsProcessor.java | 129 ++
.../flink/maintenance/operator/LockRemover.java | 25 +-
.../flink/maintenance/operator/MonitorSource.java | 4 +-
.../flink/maintenance/operator/TableChange.java | 10 +-
.../operator/TableMaintenanceMetrics.java | 10 +-
.../maintenance/operator/TriggerEvaluator.java | 8 +-
.../flink/maintenance/operator/TriggerManager.java | 54 +-
.../apache/iceberg/flink/sink/CommitSummary.java | 4 +-
.../flink/sink/IcebergStreamWriterMetrics.java | 3 +-
.../flink/source/split/SplitComparators.java | 4 +-
.../api/MaintenanceTaskInfraExtension.java | 78 +
.../maintenance/api/MaintenanceTaskTestBase.java | 64 +
.../flink/maintenance/api/TestExpireSnapshots.java | 254 +++
.../{operator => api}/TestJdbcLockFactory.java | 4 +-
.../{operator => api}/TestLockFactoryBase.java | 2 +-
.../flink/maintenance/api/TestMaintenanceE2E.java | 67 +
.../maintenance/api/TestTableMaintenance.java | 460 +++++
.../flink/maintenance/operator/CollectingSink.java | 6 +-
.../maintenance/operator/FlinkSqlExtension.java | 135 --
.../operator/FlinkStreamingTestUtils.java | 73 -
.../flink/maintenance/operator/ManualSource.java | 12 +-
.../operator/MetricsReporterFactoryForTests.java | 65 +-
.../maintenance/operator/OperatorTestBase.java | 184 +-
.../operator/TestDeleteFilesProcessor.java | 116 ++
.../operator/TestExpireSnapshotsProcessor.java | 80 +
.../maintenance/operator/TestLockRemover.java | 120 +-
.../maintenance/operator/TestMonitorSource.java | 72 +-
.../maintenance/operator/TestTriggerManager.java | 152 +-
.../iceberg/flink/sink/TestDeltaTaskWriter.java | 3 -
...FlinkIcebergSinkRangeDistributionBucketing.java | 4 +-
.../flink/sink/TestIcebergFilesCommitter.java | 3 -
.../flink/source/TestStreamingMonitorFunction.java | 3 -
.../flink/source/TestStreamingReaderOperator.java | 3 -
.../TestFileSequenceNumberBasedSplitAssigner.java | 2 +-
.../apache/iceberg/flink/data/FlinkAvroReader.java | 12 +
...AvroReader.java => FlinkPlannedAvroReader.java} | 83 +-
.../iceberg/flink/data/FlinkValueReaders.java | 32 +
.../flink/maintenance/api/ExpireSnapshots.java | 125 ++
.../flink/maintenance/api}/JdbcLockFactory.java | 7 +-
.../maintenance/api/MaintenanceTaskBuilder.java | 227 +++
.../flink/maintenance/api/TableMaintenance.java | 330 ++++
.../maintenance/{operator => api}/TaskResult.java | 7 +-
.../iceberg/flink/maintenance/api}/Trigger.java | 31 +-
.../{operator => api}/TriggerLockFactory.java | 9 +-
.../maintenance/operator/DeleteFilesProcessor.java | 130 ++
.../operator/ExpireSnapshotsProcessor.java | 129 ++
.../flink/maintenance/operator/LockRemover.java | 25 +-
.../flink/maintenance/operator/MonitorSource.java | 4 +-
.../flink/maintenance/operator/TableChange.java | 10 +-
.../operator/TableMaintenanceMetrics.java | 10 +-
.../maintenance/operator/TriggerEvaluator.java | 8 +-
.../flink/maintenance/operator/TriggerManager.java | 54 +-
.../apache/iceberg/flink/sink/CommitSummary.java | 4 +-
.../org/apache/iceberg/flink/sink/FlinkSink.java | 14 +-
.../org/apache/iceberg/flink/sink/IcebergSink.java | 14 +-
.../iceberg/flink/sink/IcebergSinkBuilder.java | 83 +
.../flink/sink/IcebergStreamWriterMetrics.java | 3 +-
.../flink/source/RowDataFileScanTaskReader.java | 4 +-
.../flink/source/split/SplitComparators.java | 4 +-
.../flink/data/TestFlinkAvroReaderWriter.java | 2 +-
.../iceberg/flink/data/TestRowProjection.java | 2 +-
.../api/MaintenanceTaskInfraExtension.java | 78 +
.../maintenance/api/MaintenanceTaskTestBase.java | 64 +
.../flink/maintenance/api/TestExpireSnapshots.java | 254 +++
.../{operator => api}/TestJdbcLockFactory.java | 4 +-
.../{operator => api}/TestLockFactoryBase.java | 2 +-
.../flink/maintenance/api/TestMaintenanceE2E.java | 67 +
.../maintenance/api/TestTableMaintenance.java | 460 +++++
.../flink/maintenance/operator/CollectingSink.java | 6 +-
.../maintenance/operator/FlinkSqlExtension.java | 135 --
.../operator/FlinkStreamingTestUtils.java | 73 -
.../flink/maintenance/operator/ManualSource.java | 12 +-
.../operator/MetricsReporterFactoryForTests.java | 65 +-
.../maintenance/operator/OperatorTestBase.java | 184 +-
.../operator/TestDeleteFilesProcessor.java | 116 ++
.../operator/TestExpireSnapshotsProcessor.java | 80 +
.../maintenance/operator/TestLockRemover.java | 120 +-
.../maintenance/operator/TestMonitorSource.java | 72 +-
.../maintenance/operator/TestTriggerManager.java | 152 +-
.../iceberg/flink/sink/TestDeltaTaskWriter.java | 3 -
...FlinkIcebergSinkRangeDistributionBucketing.java | 4 +-
.../flink/sink/TestIcebergFilesCommitter.java | 3 -
.../flink/source/TestStreamingMonitorFunction.java | 3 -
.../flink/source/TestStreamingReaderOperator.java | 3 -
.../TestFileSequenceNumberBasedSplitAssigner.java | 2 +-
format/puffin-spec.md | 55 +
format/spec.md | 439 +++--
.../java/org/apache/iceberg/gcp/GCPProperties.java | 20 +
.../java/org/apache/iceberg/gcp/gcs/GCSFileIO.java | 12 +-
.../gcp/gcs/OAuth2RefreshCredentialsHandler.java | 99 +
.../org/apache/iceberg/gcp/GCPPropertiesTest.java | 30 +
.../org/apache/iceberg/gcp/gcs/GCSFileIOTest.java | 47 +
.../gcs/OAuth2RefreshCredentialsHandlerTest.java | 264 +++
gradle/libs.versions.toml | 39 +-
kafka-connect/build.gradle | 6 +-
kafka-connect/kafka-connect-runtime/LICENSE | 1970 ++++++++++++++++++++
kafka-connect/kafka-connect-runtime/NOTICE | 1723 +++++++++++++++++
.../iceberg/connect/data/RecordConverter.java | 25 +-
.../iceberg/connect/data/RecordConverterTest.java | 32 +-
LICENSE => open-api/LICENSE | 221 ++-
open-api/NOTICE | 226 +++
open-api/requirements.txt | 2 +-
open-api/rest-catalog-open-api.py | 26 +-
open-api/rest-catalog-open-api.yaml | 73 +-
.../apache/iceberg/pig/IcebergPigInputFormat.java | 5 +
.../org/apache/iceberg/pig/IcebergStorage.java | 8 +
.../org/apache/iceberg/pig/PigParquetReader.java | 13 +-
.../java/org/apache/iceberg/pig/SchemaUtil.java | 4 +
site/docs/blogs.md | 133 +-
site/docs/how-to-release.md | 11 +-
site/docs/releases.md | 94 +-
site/mkdocs.yml | 2 +-
site/nav.yml | 1 +
site/requirements.txt | 4 +-
.../apache/iceberg/spark/extensions/TestViews.java | 169 +-
.../org/apache/iceberg/spark/SparkContentFile.java | 7 +-
.../actions/RemoveDanglingDeletesSparkAction.java | 171 ++
.../spark/actions/RewriteDataFilesSparkAction.java | 33 +-
.../apache/iceberg/spark/actions/SparkActions.java | 6 +
.../actions/TestRemoveDanglingDeleteAction.java | 426 +++++
.../spark/actions/TestRewriteDataFilesAction.java | 193 +-
.../IcebergSqlExtensionsAstBuilder.scala | 10 +-
.../v2/SetWriteDistributionAndOrderingExec.scala | 10 +-
.../spark/extensions/TestMergeOnReadDelete.java | 83 +
.../spark/extensions/TestMergeOnReadUpdate.java | 99 +-
.../TestSetWriteDistributionAndOrdering.java | 29 +-
.../apache/iceberg/spark/extensions/TestViews.java | 169 +-
.../iceberg/spark/source/DVWriterBenchmark.java | 251 +++
.../org/apache/iceberg/spark/SparkContentFile.java | 34 +-
.../actions/RemoveDanglingDeletesSparkAction.java | 179 ++
.../spark/actions/RewriteDataFilesSparkAction.java | 34 +-
.../apache/iceberg/spark/actions/SparkActions.java | 6 +
.../apache/iceberg/spark/data/SparkAvroReader.java | 12 +
...AvroReader.java => SparkPlannedAvroReader.java} | 80 +-
.../iceberg/spark/data/SparkValueReaders.java | 38 +
.../apache/iceberg/spark/source/BaseRowReader.java | 4 +-
.../iceberg/spark/source/SparkBatchQueryScan.java | 21 +
.../spark/source/SparkPositionDeltaWrite.java | 111 +-
.../logical/SetWriteDistributionAndOrdering.scala | 2 +-
.../apache/iceberg/TestScanTaskSerialization.java | 2 +-
.../iceberg/spark/actions/TestCreateActions.java | 8 +-
.../actions/TestDeleteReachableFilesAction.java | 4 +-
.../spark/actions/TestExpireSnapshotsAction.java | 3 +-
.../actions/TestRemoveDanglingDeleteAction.java | 445 +++++
.../spark/actions/TestRemoveOrphanFilesAction.java | 4 +-
.../spark/actions/TestRewriteDataFilesAction.java | 217 ++-
.../spark/actions/TestRewriteManifestsAction.java | 101 +-
.../TestRewritePositionDeleteFilesAction.java | 3 +-
.../org/apache/iceberg/spark/data/TestHelpers.java | 15 +
.../iceberg/spark/data/TestSparkAvroEnums.java | 2 +-
.../iceberg/spark/data/TestSparkAvroReader.java | 2 +-
...estParquetDictionaryEncodedVectorizedReads.java | 92 +
.../spark/source/TestCompressionSettings.java | 19 +
.../iceberg/spark/source/TestDataFrameWrites.java | 61 +-
.../source/TestIcebergSourceHadoopTables.java | 7 +-
.../iceberg/spark/source/TestSparkDVWriters.java | 69 +
.../iceberg/spark/source/TestSparkDataFile.java | 4 +-
.../apache/iceberg/spark/sql/TestCreateTable.java | 42 +
.../decimal_dict_and_plain_encoding.parquet | Bin 0 -> 3685 bytes
346 files changed, 18927 insertions(+), 3161 deletions(-)
create mode 100644
aliyun/src/test/java/org/apache/iceberg/aliyun/oss/mock/AliyunOSSMock.java
delete mode 100644
aliyun/src/test/java/org/apache/iceberg/aliyun/oss/mock/AliyunOSSMockApp.java
delete mode 100644
aliyun/src/test/java/org/apache/iceberg/aliyun/oss/mock/AliyunOSSMockLocalController.java
rename
flink/v1.20/flink/src/test/java/org/apache/iceberg/flink/maintenance/operator/ConstantsForTests.java
=> api/src/main/java/org/apache/iceberg/actions/RemoveDanglingDeleteFiles.java
(61%)
create mode 100644 api/src/main/java/org/apache/iceberg/util/ScanTaskUtil.java
create mode 100644 api/src/test/java/org/apache/iceberg/TestSchema.java
create mode 100644
api/src/test/java/org/apache/iceberg/util/TestScanTaskUtil.java
create mode 100644
aws/src/main/java/org/apache/iceberg/aws/s3/VendedCredentialsProvider.java
delete mode 100644
aws/src/test/java/org/apache/iceberg/aws/s3/MinioContainer.java
create mode 100644 aws/src/test/java/org/apache/iceberg/aws/s3/MinioUtil.java
create mode 100644
aws/src/test/java/org/apache/iceberg/aws/s3/TestMinioUtil.java
create mode 100644
aws/src/test/java/org/apache/iceberg/aws/s3/TestVendedCredentialsProvider.java
create mode 100644
core/src/jmh/java/org/apache/iceberg/deletes/RoaringPositionBitmapBenchmark.java
rename
flink/v1.19/flink/src/test/java/org/apache/iceberg/flink/maintenance/operator/ConstantsForTests.java
=>
core/src/main/java/org/apache/iceberg/actions/BaseRemoveDanglingDeleteFiles.java
(64%)
create mode 100644
core/src/main/java/org/apache/iceberg/deletes/BaseDVFileWriter.java
create mode 100644
core/src/main/java/org/apache/iceberg/deletes/DVFileWriter.java
create mode 100644
core/src/main/java/org/apache/iceberg/deletes/RoaringPositionBitmap.java
rename aliyun/src/test/java/org/apache/iceberg/aliyun/oss/mock/Range.java =>
core/src/main/java/org/apache/iceberg/rest/responses/LoadCredentialsResponse.java
(68%)
create mode 100644
core/src/main/java/org/apache/iceberg/rest/responses/LoadCredentialsResponseParser.java
create mode 100644
core/src/main/java/org/apache/iceberg/util/StructLikeUtil.java
create mode 100644
core/src/test/java/org/apache/iceberg/deletes/TestRoaringPositionBitmap.java
create mode 100644
core/src/test/java/org/apache/iceberg/rest/responses/TestLoadCredentialsResponseParser.java
create mode 100644
core/src/test/resources/org/apache/iceberg/deletes/64map32bitvals.bin
create mode 100644
core/src/test/resources/org/apache/iceberg/deletes/64mapempty.bin
create mode 100644
core/src/test/resources/org/apache/iceberg/deletes/64maphighvals.bin
create mode 100644
core/src/test/resources/org/apache/iceberg/deletes/64mapspreadvals.bin
create mode 100644
core/src/test/resources/org/apache/iceberg/deletes/all-container-types-position-index.bin
create mode 100644
core/src/test/resources/org/apache/iceberg/deletes/empty-position-index.bin
create mode 100644
core/src/test/resources/org/apache/iceberg/deletes/small-alternating-values-position-index.bin
create mode 100644
core/src/test/resources/org/apache/iceberg/deletes/small-and-large-values-position-index.bin
create mode 100644 data/src/test/java/org/apache/iceberg/io/TestDVWriters.java
create mode 100644
flink/v1.19/flink/src/main/java/org/apache/iceberg/flink/maintenance/api/ExpireSnapshots.java
rename
flink/{v1.20/flink/src/main/java/org/apache/iceberg/flink/maintenance/operator
=>
v1.19/flink/src/main/java/org/apache/iceberg/flink/maintenance/api}/JdbcLockFactory.java
(98%)
create mode 100644
flink/v1.19/flink/src/main/java/org/apache/iceberg/flink/maintenance/api/MaintenanceTaskBuilder.java
create mode 100644
flink/v1.19/flink/src/main/java/org/apache/iceberg/flink/maintenance/api/TableMaintenance.java
rename
flink/v1.19/flink/src/main/java/org/apache/iceberg/flink/maintenance/{operator
=> api}/TaskResult.java (93%)
rename
flink/{v1.20/flink/src/main/java/org/apache/iceberg/flink/maintenance/operator
=>
v1.19/flink/src/main/java/org/apache/iceberg/flink/maintenance/api}/Trigger.java
(67%)
rename
flink/v1.19/flink/src/main/java/org/apache/iceberg/flink/maintenance/{operator
=> api}/TriggerLockFactory.java (86%)
create mode 100644
flink/v1.19/flink/src/main/java/org/apache/iceberg/flink/maintenance/operator/DeleteFilesProcessor.java
create mode 100644
flink/v1.19/flink/src/main/java/org/apache/iceberg/flink/maintenance/operator/ExpireSnapshotsProcessor.java
create mode 100644
flink/v1.19/flink/src/test/java/org/apache/iceberg/flink/maintenance/api/MaintenanceTaskInfraExtension.java
create mode 100644
flink/v1.19/flink/src/test/java/org/apache/iceberg/flink/maintenance/api/MaintenanceTaskTestBase.java
create mode 100644
flink/v1.19/flink/src/test/java/org/apache/iceberg/flink/maintenance/api/TestExpireSnapshots.java
rename
flink/v1.19/flink/src/test/java/org/apache/iceberg/flink/maintenance/{operator
=> api}/TestJdbcLockFactory.java (92%)
rename
flink/v1.19/flink/src/test/java/org/apache/iceberg/flink/maintenance/{operator
=> api}/TestLockFactoryBase.java (97%)
create mode 100644
flink/v1.19/flink/src/test/java/org/apache/iceberg/flink/maintenance/api/TestMaintenanceE2E.java
create mode 100644
flink/v1.19/flink/src/test/java/org/apache/iceberg/flink/maintenance/api/TestTableMaintenance.java
delete mode 100644
flink/v1.19/flink/src/test/java/org/apache/iceberg/flink/maintenance/operator/FlinkSqlExtension.java
delete mode 100644
flink/v1.19/flink/src/test/java/org/apache/iceberg/flink/maintenance/operator/FlinkStreamingTestUtils.java
create mode 100644
flink/v1.19/flink/src/test/java/org/apache/iceberg/flink/maintenance/operator/TestDeleteFilesProcessor.java
create mode 100644
flink/v1.19/flink/src/test/java/org/apache/iceberg/flink/maintenance/operator/TestExpireSnapshotsProcessor.java
copy
flink/v1.20/flink/src/main/java/org/apache/iceberg/flink/data/{FlinkAvroReader.java
=> FlinkPlannedAvroReader.java} (61%)
create mode 100644
flink/v1.20/flink/src/main/java/org/apache/iceberg/flink/maintenance/api/ExpireSnapshots.java
rename
flink/{v1.19/flink/src/main/java/org/apache/iceberg/flink/maintenance/operator
=>
v1.20/flink/src/main/java/org/apache/iceberg/flink/maintenance/api}/JdbcLockFactory.java
(98%)
create mode 100644
flink/v1.20/flink/src/main/java/org/apache/iceberg/flink/maintenance/api/MaintenanceTaskBuilder.java
create mode 100644
flink/v1.20/flink/src/main/java/org/apache/iceberg/flink/maintenance/api/TableMaintenance.java
rename
flink/v1.20/flink/src/main/java/org/apache/iceberg/flink/maintenance/{operator
=> api}/TaskResult.java (93%)
rename
flink/{v1.19/flink/src/main/java/org/apache/iceberg/flink/maintenance/operator
=>
v1.20/flink/src/main/java/org/apache/iceberg/flink/maintenance/api}/Trigger.java
(67%)
rename
flink/v1.20/flink/src/main/java/org/apache/iceberg/flink/maintenance/{operator
=> api}/TriggerLockFactory.java (86%)
create mode 100644
flink/v1.20/flink/src/main/java/org/apache/iceberg/flink/maintenance/operator/DeleteFilesProcessor.java
create mode 100644
flink/v1.20/flink/src/main/java/org/apache/iceberg/flink/maintenance/operator/ExpireSnapshotsProcessor.java
create mode 100644
flink/v1.20/flink/src/main/java/org/apache/iceberg/flink/sink/IcebergSinkBuilder.java
create mode 100644
flink/v1.20/flink/src/test/java/org/apache/iceberg/flink/maintenance/api/MaintenanceTaskInfraExtension.java
create mode 100644
flink/v1.20/flink/src/test/java/org/apache/iceberg/flink/maintenance/api/MaintenanceTaskTestBase.java
create mode 100644
flink/v1.20/flink/src/test/java/org/apache/iceberg/flink/maintenance/api/TestExpireSnapshots.java
rename
flink/v1.20/flink/src/test/java/org/apache/iceberg/flink/maintenance/{operator
=> api}/TestJdbcLockFactory.java (92%)
rename
flink/v1.20/flink/src/test/java/org/apache/iceberg/flink/maintenance/{operator
=> api}/TestLockFactoryBase.java (97%)
create mode 100644
flink/v1.20/flink/src/test/java/org/apache/iceberg/flink/maintenance/api/TestMaintenanceE2E.java
create mode 100644
flink/v1.20/flink/src/test/java/org/apache/iceberg/flink/maintenance/api/TestTableMaintenance.java
delete mode 100644
flink/v1.20/flink/src/test/java/org/apache/iceberg/flink/maintenance/operator/FlinkSqlExtension.java
delete mode 100644
flink/v1.20/flink/src/test/java/org/apache/iceberg/flink/maintenance/operator/FlinkStreamingTestUtils.java
create mode 100644
flink/v1.20/flink/src/test/java/org/apache/iceberg/flink/maintenance/operator/TestDeleteFilesProcessor.java
create mode 100644
flink/v1.20/flink/src/test/java/org/apache/iceberg/flink/maintenance/operator/TestExpireSnapshotsProcessor.java
create mode 100644
gcp/src/main/java/org/apache/iceberg/gcp/gcs/OAuth2RefreshCredentialsHandler.java
create mode 100644
gcp/src/test/java/org/apache/iceberg/gcp/gcs/OAuth2RefreshCredentialsHandlerTest.java
create mode 100644 kafka-connect/kafka-connect-runtime/LICENSE
create mode 100644 kafka-connect/kafka-connect-runtime/NOTICE
copy LICENSE => open-api/LICENSE (58%)
create mode 100644 open-api/NOTICE
create mode 100644
spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/actions/RemoveDanglingDeletesSparkAction.java
create mode 100644
spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/actions/TestRemoveDanglingDeleteAction.java
create mode 100644
spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/source/DVWriterBenchmark.java
create mode 100644
spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/actions/RemoveDanglingDeletesSparkAction.java
copy
spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/data/{SparkAvroReader.java
=> SparkPlannedAvroReader.java} (62%)
create mode 100644
spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/actions/TestRemoveDanglingDeleteAction.java
create mode 100644
spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDVWriters.java
create mode 100644
spark/v3.5/spark/src/test/resources/decimal_dict_and_plain_encoding.parquet
