This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a change to branch
dependabot/gradle/net.ltgt.gradle-gradle-errorprone-plugin-3.1.0
in repository https://gitbox.apache.org/repos/asf/iceberg.git
omit 28a1ff9777 Build: Bump net.ltgt.gradle:gradle-errorprone-plugin from
3.0.1 to 3.1.0
add bbd24684fe Build: Bump adlfs from 2023.1.0 to 2023.4.0 in /python
(#7543)
add 23419b4e75 API, Core: Make RewriteFiles flexible (#7501)
add f7b1b72621 API, Core: Make RewriteFiles flexible (#7501)
add 39bbb98286 API, Core: Make RewriteFiles flexible (#7501)
add 174c4ca52f API, Core: Make RewriteFiles flexible (#7501)
add bbe7a64d1c AWS: add missing line to assign param S3FileIOProperties
inside constructor (#7559)
add ef64c0af9c Build: Update RoaringBitmap to 0.9.44 (#7563)
add 217efffe03 Python: Move to strictyaml (#7529)
add 615ab116a6 Core: Refactor naming in MergingSnapshotProducer (#7564)
add b60bf8b3a6 Fix typos and improve wording in aws.md (#7548)
add 08dace77d5 Spark 3.3: Uniquess validation when computing updates of
changelogs (#7388)
add 66bd2782ba Core: Add finalizer to ResolvingFileIO (#7536)
add 46fe807df3 Core, AWS: Add flag to suppress duplicate FileIO stack
trace (#7552)
add 5808344390 Spark 3.2, 3.4: Uniqueness validation when computing
updates of changelogs (#7573)
add 46fa4e2a21 AWS: create HttpClientProperties, move s3 related methods
into S3FileIOProperties (#7562)
add 2a06bb5aab Doc: Updates Writing to Partitioned Table Spark Docs (#7499)
add 4fe9e593a8 Update slack invite link (#7583)
add 8865be48bc Nessie: Bump Nessie dependencies from 0.57.0 to 0.58.1
(#7579)
add cf082af91d Docs: Add identifier to each Markdown file under docs
(#7575)
add 852c1bf011 Python: Bump PyArrow to 12.0.0 (#7588)
add dbaeadfc4f Core: Check all specs to determine schema of partitions
table (#7551)
add b88951bbef API, Flink: StructProjection returns null projection object
for null nested struct value (#7517)
add e01bc223b6 Build: Bump mkdocstrings-python from 0.9.0 to 1.0.0 in
/python (#7606)
add eea7bb1cc1 Build: Bump pre-commit from 3.2.2 to 3.3.1 in /python
(#7605)
add 83f0c7d9d8 Build: Bump griffe from 0.27.3 to 0.27.5 in /python (#7604)
add 1cdc871107 Build: Bump mkdocs-material from 9.1.9 to 9.1.12 in /python
(#7603)
add aafb3cee40 Build: Bump mkdocs from 1.4.2 to 1.4.3 in /python (#7602)
add 7c60be7e5b Build: Upgrade Gradle to 8.1.1 (#7610)
add 95be02c912 Core: Remove deprecated AssertHelpers usage in catalog
(#7596)
add e3592e8a55 Build: Bump Arrow from 11.0.0 to 12.0.0 (#7595)
add 00f7babfdc Core: Remove deprecated AssertHelpers usage (#7597)
add 81af3f76ab Spark: Fix Parquet read benchmarks for Spark 3.3 + 3.4
(#7587)
add 1144241009 Docs: Improve readability on page Branching and Tagging
(#7592)
add 203b8db4dc Flink: change sink shuffle to use RowData as data type and
statistics key type (#7494)
add 198fb723dd Flink: add toString, equals, hashCode overrides for
RowDataProjection. (#7493)
add ce01bec3d4 Core: Add ReadableMetrics in Entries table (#7539)
add adbc4ff3e6 Add unique JDBC application identifier and user agent
header (#7580)
add 04a79ec9d7 Spark: Remove deprecated methods in
VectorizedSparkParquetReaders (#7591)
add f551673589 Views: Update spec with expectations on versions,
representations, and dialects (#7500)
add 36ad90b876 Core: Allow deleting old partition spec columns in V1
(#7398)
add 42cf1579ab API, Core, Spark: Add file groups failure in rewrite result
(#7361)
add f2c065b445 Docs: Update link to contributing section (#7117)
add 29eebdd89e AWS: Add S3FileIOAwsClientFactory for S3FileIO (#7590)
add d23b36c31d Core: Add FileIO tracker/closer to REST catalog (#7487)
add 9907a97351 API, Core: Expose file and data sequence numbers through
ContentFile (#7555)
add abac62f857 Spark 3.4: Avoid local sort for MERGE cardinality check
(#7558)
add 890ca0ef73 Spark 3.4: Fixup for RewritePositionDeleteFilesSparkAction
(#7389) (#7565)
add 7cbde142f7 Flink: Add config for max allowed consecutive planning
failures in IcebergSource before failing the job (#7571)
add f07bedc86f Build: Bump com.fasterxml.jackson.core:jackson-annotations
(#7601)
add 9d7ef2e9dd Spark: Disable aggregate pushdown for incremental scan
(#7626)
add d65fa091b9 Spark 3.4: Add RewritePositionDeleteFilesProcedure (#7572)
add 70386f92ab Build: Remove Kyle and add bitsondatadev to collaborators
.asf.yaml (#7634)
add 202af0335b Improve Error Handling to map Snowflake Exceptions into
Iceberg Exceptions (#6952)
add 477da36e3f Flink: backport Add config for max allowed consecutive
planning failures in IcebergSource before failing the job (#7571) to 1.16 and
1.15 (#7629)
add 67decf90d9 Flink: backport PR #7494. change sink shuffle to use
RowData as data type and statistics key type (#7632)
add 39a5f503c1 Flink: backport PR #7493. add toString, equals, hashCode
overrides for RowDataProjection (#7631)
add 4e87cff70d Flink: sink support of spec evaluation (#7171)
add 4554938912 Core: Allow one data writer in BasePositionDeltaWriter
(#7648)
add 24490b0da3 Spark 3.4: Minor refactoring for SparkPositionDeltaWrite
(#7650)
add bfe3a25b68 Spark-3.4: Fix errorprone warning (#7654)
add d3fcd9890a GCP, Pig: Switch tests to JUnit5 (#7647)
add 22d19c2847 Spark 3.4: Fix NPE when create branch and tag on table
without snapshot (#7652)
add 2f61a08274 Spark 3.4: Split update into delete and insert for position
deltas (#7646)
add 14a30c015b Parquet: Update parquet to 1.13.1 (#7301)
add 8ace681824 Spark 3.4: Harmonize RewriteDataFilesSparkAction code with
RewritePositionDeleteFilesSparkAction (#7630)
add 08ae725e1a Spark 3.3, 3.4: Fix always true/false condition in
rewrite_data_files procedure (#6760)
add d8fc5b3c0a Spark 3.2: backport Spark SQL extension on
create/update/drop tags (#7662)
add 950daa5702 Spark: Backport fix NPE when create branch and tag on table
without snapshot (#7659)
add b5e2f3608e Core: Compacted position delete files should use the max
data sequence number of source files (#7651)
add 8c2a924a8d Docs: RewritePositionDeleteFiles procedure (#7589)
add f5f2bb3b9b Build: Bump mkdocs-material from 9.1.12 to 9.1.14 in
/python (#7670)
add c03f1268cb Build: Bump fastavro from 1.7.3 to 1.7.4 in /python (#7671)
add 98b92a4e6a Build: Bump duckdb from 0.7.1 to 0.8.0 in /python (#7673)
add 41c510fdb1 Build: Bump griffe from 0.27.5 to 0.28.0 in /python (#7674)
add 17cf01581c Spec: OpenAPI responses should reference schemas (#6699)
add fa9d0dad7d Core, Parquet: Remove Parquet dictionary encoding table
property (#7665)
add 81a9f6a5f7 Build: Bump pre-commit from 3.3.1 to 3.3.2 in /python
(#7672)
add be6235cf75 Build: Bump com.esotericsoftware:kryo-shaded from 4.0.2 to
4.0.3 (#7669)
add 65004c15f0 Infra: Use the standard shadow plugin (#7681)
add b2e99b5b35 Spark 3.4: Add TimestampNTZ (#7553)
add 7d13b48896 Spark 3.3: Add RewritePositionDeleteFilesSparkAction (#7684)
add 2502a23663 Spark 3.4: Distribution and ordering enhancements (#7637)
add 9464bc49f5 Flink: Port #7171 to flink 1.17
add 53683f87fb Flink: Port #7171 to 1.15 (#7679)
add 9585525e92 Spark 3.3: Avoid local sort for MERGE cardinality check
(#7686)
add 711780b6cc Spark 3.3: Add RewritePositionDeleteFilesProcedure (#7687)
add a2dee506ab Build: Bump requests from 2.30.0 to 2.31.0 in /python
(#7689)
add d898de52f0 Nessie: Bump Nessie version from 0.58.1 to 0.59.0 (#7642)
add 11eed7180b Spark 3.3: Harmonize RewriteDataFilesSparkAction (#7676)
add 7dbdfd33a6 Core: Metadata table code harmonization for
readable_metrics (#7613)
add 8fe6248736 Nessie: Disable table metadata files cleanup during commits
(#7641)
add 4b8941e031 Core: Support set system level properties with
environmental variables (#5659)
add 82880be396 Core: Fix docstring (#7697)
add 5eb45117fa Core: Add writer for unordered position deletes (#7692)
add 3a584a2835 Python: Refactor integration tests (#7698)
add 6872b7cd27 Spec: TableRequirement definition and parser mismatch
(#7700)
add aa1c1ef49e Parquet: Add encryption config to read and write builders
(#2639)
add b69e84b6cf Core: Make all of the BaseFile stat maps immutable (#7643)
add 893af4a198 Enable extra commit properties with metadata delete (#7649)
add b430775ea2 Spark 3.4: Support fanout writers in
SparkPositionDeltaWrite (#7703)
add 06ce8b2f18 Spark 3.3: Fix bucket expressions on binary columns (#7693)
add 601c5af9b6 Core: Remove deprecated AssertHelpers (#7711)
add 9b6397f31f Nessie: Bump Nessie to 0.60.1 (#7701)
add 7d3fa50fc2 Build: Add openapi label (#7712)
add 8858f1cfd3 Revert "Python: Refactor integration tests (#7698)" (#7729)
add ee1040e504 Parquet: skip writing bloom filters for deletes (#7617)
add 54119bd0f1 AWS: Fix Tests3RestSigner on OSX (#7742)
add 1ea185290a API: Improve Javadoc of listNamespaces() (#7196)
add ce23dbdec1 Python: Loosen dependency on typing-extensions (#7747)
add 08f3ca767d Flink: Bump version to 1.16.2 and 1.17.1 (#7745)
add aff1c4767e Core: Switch tests to Junit5 in avro package (#7655)
add 7d46f9b2e8 Build: Bump griffe from 0.28.0 to 0.29.0 in /python (#7728)
add 7a1ebc1cdf Build: Bump mkdocstrings from 0.21.2 to 0.22.0 in /python
(#7727)
add 479c1c096f Build: Bump mkdocstrings-python from 1.0.0 to 1.1.0 in
/python (#7724)
add 9829b44a6d Python: Fix integration-test path (#7705)
add 3ca8aabfe3 Core: Switch tests to JUnit5 in deletes,expressions,auth
packages (#7755)
add 5f39174fc4 Python: Add support for initial default (#7699)
add 96f0f22bd8 API: Switch tests to JUnit5 (#7730)
add 9d42eb44a3 API: Switch tests to JUnit5 (io,metrics,transforms
packages) (#7757)
add 677105c2b0 Build: Bump typing-extensions from 4.6.2 to 4.6.3 in
/python (#7765)
add f5e393d09a Build: Bump mkdocs-material from 9.1.14 to 9.1.15 in
/python (#7764)
add 9d17cdd35f Python: Remove integration-test caching (#7704)
add 2fe8e13835 Python: Refactor integration tests (#7698) (#7768)
add 26af4c7387 Spec: Add missing `last-column-id` (#7445)
add c591c26e5e Python: Add support for pydocstyle (#7772)
add 66b0631b3f API: Switch tests to JUnit5 (types, util packages) (#7758)
add 20f0d0cc01 Core: Remove deprecated AssertHelpers in hadoop package
(#7722)
add 23e0d10754 Build: Bump com.fasterxml.jackson.core:jackson-annotations
(#7763)
add 54648263d6 Open-API: Add Python code as an example (#7751)
add cfa090531e Python: Pin `mkdocs-admon` (#7779)
add e3edb77b98 Python: Fix TypeError when partition tuples have nulls
(#7783)
add 04b3e6bda0 Spark 3.4: Add a bucket binary unit test (#7720)
add fbfe584f84 Tests: More permissive error message assertion in catalog
tests (#7784)
add c4db21784f Python: Add support for sequence number inheritance (#7778)
add 18c343beae Core: Fix paths for unpartitioned specs in writers (#7685)
add 814a22f7bc Tests: Update catalog test assertion to be more permissive
(#7789)
add e674cca20f Core: Deprecate `newOutputFile` from
`{ClusteredWriter,FanoutWriter}` (#7788)
add 5a2eba3d79 Python: Fix optionality of Manifest fields (#7796)
add 3221b1c4e7 Fix D403 pydocstyle issues (#7801)
add 2a83d9e58f Core: Key metadata in Avro format (#6450)
add cdc1a27ade Core: Add REST spec and request for commits to multiple
tables (#7741)
add 13addef17d Python: D414 and D200 docstring issues (#7806)
add 39fb3d1630 Python: Resolve pydocstring D202 violations (#7805)
add 791460646b Open-API: Update code (#7807)
add 94c07f6735 Build: Bump pytest from 7.3.1 to 7.3.2 in /python (#7817)
add 35ad710dd3 Build: Bump mkdocstrings-python from 1.1.0 to 1.1.2 in
/python (#7816)
add c2bf539bb4 Build: Bump ray from 2.4.0 to 2.5.0 in /python (#7814)
add 264376773a Build: Bump requests-mock from 1.10.0 to 1.11.0 in /python
(#7815)
add 8aefc6925f Build: Bump moto from 4.1.10 to 4.1.11 in /python (#7813)
add 386f9b8cee Build: skip scheduled workflows on forks (#7818)
add dd70b7e8f7 Core: Switch tests to JUnit5 in metrics,puffin,rest
packages (#7803)
add f309dd098d Core: Fix NPE in SnapshotUtil.ancestorsOf (#7718)
add 81a5b337e9 Spark: Update antlr4 to match Spark 3.4 (#7824)
add fc381a81a1 Spark: Use constants instead of plain strings (#7825)
add c256604b34 Docs: Update Nessie docs (#7826)
add a63d18f1cf Python: Make Glue catalog configurable (#7781)
add 50fefe4036 Docs: Fix SQL formatting in Flink docs (#7761)
add 9130acf6c2 Build: Bump com.github.alisiikh:gradle-scalastyle-plugin
(#7811)
add 4263135256 Parquet: Revert workaround for resource usage with zstd
(#7834)
add 461f8d0f8f Python: Fix D400 Docstring voilations (#7829)
add c36c7bad3c Python: Add pickle support (#7645)
add 54062ac619 Python: Fixes for D205 issues (#7823)
add 3fb7dab4f3 Build: Update ORC to 1.8.4 (#7839)
add 84b8338ed1 Docs: Add docs for CommitMetadata (#7743)
add c115f499f0 Core: Switch tests to JUnit5 in
catalog,encryption,inmemory,io,view packages (#7767)
add d29f392e09 Nessie: Support ApiV2 for Nessie client (#6712)
add 6f212e22da Add 1.3.0 to Bug Template (#7845)
add 5584426b7d Build: Let Rev API compare against 1.3.0 (#7848)
add c85ba939ab Core: Move UpdateRequirement out of rest package (#7750)
add f2b01f8ab5 Core: View metadata implementation (#7759)
add b897cd132e Flink: Add possibilit of ordering the splits based on the
file sequence number (#7661)
add 8cabd87c83 Build: Bump pre-commit from 3.3.2 to 3.3.3 in /python
(#7855)
add 563236f62f Build: Bump pyarrow from 12.0.0 to 12.0.1 in /python (#7856)
add aeafd2246c Build: Bump pydantic from 1.10.8 to 1.10.9 in /python
(#7857)
add 82157460d9 Build: Bump rich from 13.4.1 to 13.4.2 in /python (#7858)
add 944ec4977e Build: Bump duckdb from 0.8.0 to 0.8.1 in /python (#7859)
add 53a9c8c773 Build: Bump mkdocs-material from 9.1.15 to 9.1.16 in
/python (#7860)
add 7c3c34897a Core: Remove usage of AssertHelpers (#7862)
add 6586788e99 Core: Include all reachable snapshots with v1 format and
REF snapshot mode (#7621)
add d8f2daf967 Core: Add REST API for committing changes against multiple
tables (#7569)
add 717b3d7faf Flink: Improve naming of Tests (#7864)
add 9ffb762153 Python: Add positional deletes (#6775)
add 5c64100752 Core: Remove usage of AssertHelpers (#7868)
add 80aee85383 Python: Alter table plumbing and REST support (#6323)
add 806580e36e Python: Fix D105 docstring violations (#7852)
add 97d00f9eb3 Python: Fix docstrings (#7877)
add 3a9d6508a6 Core: Switch tests to JUnit5 in rest/hadoop pakages (#7861)
add 6d517236fb Python: Remove future annotations (#7878)
add 6b23d2555d Python: Log to logging instead of warn (#7876)
add ec7da7aafb Core: Register JSON Parser for UpdateTableRequest (#7891)
add a29c6cebfd Core: Add last updated timestamp and snapshotId for
Partitions table (#7581)
add 3ac871a9e3 ORC: Fix to avoid test failure on Hadoop 2.7.3 (#7894)
add c6d9b92110 Docs: Add missing ; in Flink examples (#7908)
add d5884f37ce Core: Migrate jdbc package to JUnit5 (#7874)
add fb929437af Core: Add create(..) method to UpdateTableRequest (#7867)
add 006591c876 Flink: Backport to 1.15 and 1.16: Add possibilit of
ordering the splits based on the file sequence number (#7661) (#7889)
add 607c268268 Spark: Remove usage of AssertHelpers (#7899)
add 5e0e5e39fa Flink: Fix serialization in TableSink with anonymous object
(#7866)
add 6a9cf4f0c0 Build: Bump pytest from 7.3.2 to 7.4.0 in /python (#7902)
add 47a930b0d5 Build: Bump mkdocs-material from 9.1.16 to 9.1.17 in
/python (#7903)
add 70ab49ed77 Build: Bump griffe from 0.29.0 to 0.29.1 in /python (#7904)
add 18a42e43c6 Build: Bump moto from 4.1.11 to 4.1.12 in /python (#7905)
add e85ec9447c Python: Bump version to 0.4.0 (#7912)
add 9f12b85667 Core: Fixes OOM caused by Avro decoder caching (#7791)
add b8db3f0042 Core: add JSON parser for ContentFile and FileScanTask
(#6934)
add fbed421697 Flink: Key projection should be based on the requested
Flink table schema (#7836)
add f5bb0c0da0 Flink: remove the creation of default database in
FlinkCatalog open method (#7795)
add d98e7a1152 Spark 3.4: Multiple shuffle partitions per file in
compaction (#7897)
add 96c87dd49f Docs: Add `rewritten_bytes_count` to `rewrite_data_files`
(#7916)
add 15a9af7228 Dell,Arrow : Migrate test cases in dell,arrow packages to
JUnit5 (#7872)
add 4e6c7bad35 Docs: Add Spark `timestamp_ntz` type to docs (#7917)
add c96edfc096 Spark: Bump to 3.4.1 (#7881)
add dd0ee24860 Nessie: Minor Refactoring of NessieTableOperations (#7893)
add 2fea2dd70a Nessie: Test Nessie's new storage model (#7702)
add 51eaf68063 Python: Pin pyparsing and update tests (#7927)
add 37900b3706 Flink: port PR #7836 to 1.15/1.16 (#7923)
add 13295ce4d3 Docs: Update Python Release verification steps (#7940)
add 582ec1cfb8 Bump Nessie to 0.63.0 (#7828)
add 94eed43d5e Build: Bump net.ltgt.gradle:gradle-errorprone-plugin from
3.0.1 to 3.1.0
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (28a1ff9777)
\
N -- N -- N
refs/heads/dependabot/gradle/net.ltgt.gradle-gradle-errorprone-plugin-3.1.0
(94eed43d5e)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
No new revisions were added by this update.
Summary of changes:
.asf.yaml | 3 +-
.github/ISSUE_TEMPLATE/iceberg_bug_report.yml | 3 +-
.github/ISSUE_TEMPLATE/iceberg_question.yml | 2 +-
.github/labeler.yml | 2 +
.github/workflows/open-api.yml | 10 +-
.github/workflows/python-integration.yml | 40 +-
.github/workflows/recurring-jmh-benchmarks.yml | 1 +
.github/workflows/stale.yml | 1 +
.gitignore | 2 +
.palantir/revapi.yml | 10 +
CONTRIBUTING.md | 2 +-
.../main/java/org/apache/iceberg/ContentFile.java | 34 +
.../main/java/org/apache/iceberg/FileScanTask.java | 5 +
.../main/java/org/apache/iceberg/RewriteFiles.java | 102 ++
api/src/main/java/org/apache/iceberg/Snapshot.java | 6 +-
.../apache/iceberg/actions/RewriteDataFiles.java | 19 +
.../org/apache/iceberg/catalog/SessionCatalog.java | 29 +-
.../apache/iceberg/catalog/SupportsNamespaces.java | 29 +-
.../org/apache/iceberg/util/StructProjection.java | 12 +-
.../java/org/apache/iceberg/TestAccessors.java | 18 +-
.../test/java/org/apache/iceberg/TestHelpers.java | 83 +-
.../java/org/apache/iceberg/TestIcebergBuild.java | 34 +-
.../apache/iceberg/TestMetricsSerialization.java | 19 +-
.../org/apache/iceberg/TestPartitionPaths.java | 19 +-
.../iceberg/TestPartitionSpecValidation.java | 102 +-
.../java/org/apache/iceberg/TestSnapshotRef.java | 39 +-
.../apache/iceberg/TestTransformSerialization.java | 12 +-
.../org/apache/iceberg/catalog/TestNamespace.java | 6 +-
.../iceberg/catalog/TestTableIdentifier.java | 34 +-
.../org/apache/iceberg/events/TestListeners.java | 13 +-
.../iceberg/expressions/TestAggregateBinding.java | 23 +-
.../expressions/TestAggregateEvaluator.java | 14 +-
.../apache/iceberg/expressions/TestEvaluator.java | 917 ++++++-------
.../iceberg/expressions/TestExpressionBinding.java | 60 +-
.../iceberg/expressions/TestExpressionHelpers.java | 105 +-
.../expressions/TestExpressionSerialization.java | 23 +-
.../iceberg/expressions/TestExpressionUtil.java | 542 ++++----
.../TestInclusiveManifestEvaluator.java | 271 ++--
.../expressions/TestInclusiveMetricsEvaluator.java | 295 +++--
.../expressions/TestLiteralSerialization.java | 19 +-
.../TestMetricsEvaluatorsNaNHandling.java | 140 +-
.../expressions/TestMiscLiteralConversions.java | 46 +-
.../expressions/TestNumericLiteralConversions.java | 124 +-
.../iceberg/expressions/TestPredicateBinding.java | 467 ++++---
.../expressions/TestStrictMetricsEvaluator.java | 208 +--
.../expressions/TestStringLiteralConversions.java | 59 +-
.../org/apache/iceberg/io/TestCloseableGroup.java | 2 +-
.../apache/iceberg/io/TestCloseableIterable.java | 55 +-
.../org/apache/iceberg/io/TestClosingIterator.java | 22 +-
.../apache/iceberg/metrics/TestDefaultCounter.java | 2 +-
.../iceberg/metrics/TestDefaultMetricsContext.java | 2 +-
.../apache/iceberg/metrics/TestDefaultTimer.java | 2 +-
.../metrics/TestFixedReservoirHistogram.java | 2 +-
.../apache/iceberg/transforms/TestBucketing.java | 258 ++--
.../transforms/TestBucketingProjection.java | 31 +-
.../org/apache/iceberg/transforms/TestDates.java | 163 +--
.../iceberg/transforms/TestDatesProjection.java | 32 +-
.../apache/iceberg/transforms/TestIdentity.java | 70 +-
.../iceberg/transforms/TestNotStartsWith.java | 109 +-
.../apache/iceberg/transforms/TestProjection.java | 118 +-
.../apache/iceberg/transforms/TestResiduals.java | 73 +-
.../apache/iceberg/transforms/TestStartsWith.java | 14 +-
.../apache/iceberg/transforms/TestTimestamps.java | 219 ++--
.../transforms/TestTimestampsProjection.java | 32 +-
.../transforms/TestTransformSerialization.java | 9 +-
.../apache/iceberg/transforms/TestTruncate.java | 102 +-
.../transforms/TestTruncatesProjection.java | 33 +-
.../iceberg/transforms/TestTruncatesResiduals.java | 12 +-
.../apache/iceberg/types/TestBinaryComparator.java | 19 +-
.../iceberg/types/TestCharSeqComparator.java | 49 +-
.../iceberg/types/TestComparableComparator.java | 18 +-
.../org/apache/iceberg/types/TestComparators.java | 13 +-
.../org/apache/iceberg/types/TestConversions.java | 87 +-
.../iceberg/types/TestReadabilityChecks.java | 184 +--
.../iceberg/types/TestSerializableTypes.java | 55 +-
.../org/apache/iceberg/types/TestTypeUtil.java | 98 +-
.../java/org/apache/iceberg/types/TestTypes.java | 2 +-
.../apache/iceberg/util/TestCharSequenceSet.java | 2 +-
.../org/apache/iceberg/util/TestDateTimeUtil.java | 2 +-
.../org/apache/iceberg/util/TestExceptionUtil.java | 2 +-
.../apache/iceberg/arrow/ArrowSchemaUtilTest.java | 77 +-
.../iceberg/arrow/vectorized/ArrowReaderTest.java | 46 +-
.../vectorized/parquet/DecimalVectorUtilTest.java | 20 +-
.../iceberg/aws/s3/TestS3FileIOIntegration.java | 29 +
.../apache/iceberg/aws/AwsClientProperties.java | 184 +++
.../java/org/apache/iceberg/aws/AwsProperties.java | 106 +-
.../apache/iceberg/aws/HttpClientProperties.java | 237 ++++
.../iceberg/aws/S3FileIOAwsClientFactories.java | 80 ++
.../aws/s3/DefaultS3FileIOAwsClientFactory.java | 58 +
.../java/org/apache/iceberg/aws/s3/S3FileIO.java | 23 +-
.../iceberg/aws/s3/S3FileIOAwsClientFactory.java | 30 +-
.../apache/iceberg/aws/s3/S3FileIOProperties.java | 82 ++
.../iceberg/aws/AwsClientPropertiesTest.java | 103 ++
.../iceberg/aws/HttpClientPropertiesTest.java | 82 ++
.../aws/TestS3FileIOAwsClientFactories.java | 53 +
.../iceberg/aws/s3/TestS3FileIOProperties.java | 79 ++
.../iceberg/aws/s3/signer/TestS3RestSigner.java | 1 +
baseline.gradle | 1 +
build.gradle | 71 +-
.../org/apache/iceberg/ManifestReadBenchmark.java | 163 +++
.../org/apache/iceberg/BaseContentScanTask.java | 15 +-
.../java/org/apache/iceberg/BaseEntriesTable.java | 124 +-
.../src/main/java/org/apache/iceberg/BaseFile.java | 45 +-
.../java/org/apache/iceberg/BaseFileScanTask.java | 5 +
.../java/org/apache/iceberg/BaseFilesTable.java | 112 +-
.../org/apache/iceberg/BaseOverwriteFiles.java | 2 +-
.../java/org/apache/iceberg/BaseRewriteFiles.java | 82 +-
.../src/main/java/org/apache/iceberg/BaseScan.java | 2 +-
.../java/org/apache/iceberg/BaseTransaction.java | 4 +
.../java/org/apache/iceberg/ContentFileParser.java | 270 ++++
.../main/java/org/apache/iceberg/DataFiles.java | 13 +-
.../org/apache/iceberg/FileScanTaskParser.java | 126 ++
.../java/org/apache/iceberg/GenericDataFile.java | 3 +-
.../apache/iceberg/InheritableMetadataFactory.java | 12 +-
.../java/org/apache/iceberg/ManifestFiles.java | 15 +-
.../org/apache/iceberg/ManifestFilterManager.java | 6 +
.../apache/iceberg/MergingSnapshotProducer.java | 160 ++-
.../org/apache/iceberg/MetadataUpdateParser.java | 7 +-
.../main/java/org/apache/iceberg/MetricsUtil.java | 51 +
.../java/org/apache/iceberg/PartitionsTable.java | 85 +-
.../main/java/org/apache/iceberg/ScanSummary.java | 2 +-
.../apache/iceberg/SerializableByteBufferMap.java | 14 +
.../java/org/apache/iceberg/SystemConfigs.java | 123 ++
.../java/org/apache/iceberg/SystemProperties.java | 8 +-
.../java/org/apache/iceberg/TableMetadata.java | 20 +-
.../java/org/apache/iceberg/UpdateRequirement.java | 202 +++
.../requests => }/UpdateRequirementParser.java | 7 +-
.../org/apache/iceberg/UpdateRequirements.java | 169 +++
.../main/java/org/apache/iceberg/V1Metadata.java | 10 +
.../main/java/org/apache/iceberg/V2Metadata.java | 10 +
.../actions/BaseRewriteDataFilesAction.java | 15 +-
.../apache/iceberg/actions/RewriteFileGroup.java | 17 +
.../RewritePositionDeletesCommitManager.java | 22 +-
.../actions/RewritePositionDeletesGroup.java | 8 +
.../org/apache/iceberg/avro/GenericAvroReader.java | 6 +-
.../org/apache/iceberg/avro/GenericAvroWriter.java | 6 +-
.../org/apache/iceberg/catalog/TableCommit.java | 69 +
.../apache/iceberg/data/avro/DecoderResolver.java | 11 +-
.../apache/iceberg/data/avro/IcebergDecoder.java | 53 +-
.../org/apache/iceberg/data/avro/RawDecoder.java | 64 +
.../iceberg/deletes/PositionDeleteWriter.java | 7 +
.../deletes/SortingPositionOnlyDeleteWriter.java | 111 ++
.../org/apache/iceberg/encryption/KeyMetadata.java | 132 ++
.../iceberg/encryption/KeyMetadataDecoder.java | 76 ++
.../iceberg/encryption/KeyMetadataEncoder.java | 110 ++
.../apache/iceberg/io/BasePositionDeltaWriter.java | 26 +-
.../org/apache/iceberg/io/ClusteredWriter.java | 10 +-
.../iceberg/io/FanoutPositionOnlyDeleteWriter.java | 79 ++
.../java/org/apache/iceberg/io/FanoutWriter.java | 10 +-
.../org/apache/iceberg/io/ResolvingFileIO.java | 47 +-
.../org/apache/iceberg/io/RollingFileWriter.java | 2 +-
.../org/apache/iceberg/rest/CatalogHandlers.java | 13 +-
.../java/org/apache/iceberg/rest/RESTCatalog.java | 13 +
.../org/apache/iceberg/rest/RESTSerializers.java | 75 +-
.../apache/iceberg/rest/RESTSessionCatalog.java | 75 +-
.../apache/iceberg/rest/RESTTableOperations.java | 30 +-
.../org/apache/iceberg/rest/ResourcePaths.java | 4 +
.../rest/requests/CommitTransactionRequest.java | 47 +
.../requests/CommitTransactionRequestParser.java | 78 ++
.../rest/requests/UpdateRequirementParser.java | 5 +
.../iceberg/rest/requests/UpdateTableRequest.java | 58 +-
.../rest/requests/UpdateTableRequestParser.java | 110 ++
.../java/org/apache/iceberg/util/JsonUtil.java | 35 +-
.../java/org/apache/iceberg/util/SnapshotUtil.java | 4 +
.../org/apache/iceberg/util/StructLikeMap.java | 7 +
.../java/org/apache/iceberg/util/ThreadPools.java | 25 +-
.../java/org/apache/iceberg/view/ViewMetadata.java | 129 ++
.../apache/iceberg/view/ViewMetadataParser.java | 176 +++
.../org/apache/iceberg/view/ViewProperties.java | 18 +-
.../apache/iceberg/MetadataTableScanTestBase.java | 12 +-
.../test/java/org/apache/iceberg/ScanTestBase.java | 17 +-
.../java/org/apache/iceberg/TableTestBase.java | 92 +-
.../iceberg/TestBaseIncrementalAppendScan.java | 27 +-
.../iceberg/TestBaseIncrementalChangelogScan.java | 9 +-
.../java/org/apache/iceberg/TestCatalogUtil.java | 77 +-
.../org/apache/iceberg/TestContentFileParser.java | 338 +++++
.../org/apache/iceberg/TestCreateTransaction.java | 25 +-
.../java/org/apache/iceberg/TestDataTableScan.java | 102 +-
.../java/org/apache/iceberg/TestDeleteFiles.java | 46 +-
.../apache/iceberg/TestEntriesMetadataTable.java | 15 +-
.../java/org/apache/iceberg/TestFastAppend.java | 39 +-
.../org/apache/iceberg/TestFileScanTaskParser.java | 108 ++
.../org/apache/iceberg/TestFormatVersions.java | 25 +-
.../iceberg/TestIncrementalDataTableScan.java | 34 +-
.../org/apache/iceberg/TestLocationProvider.java | 44 +-
.../org/apache/iceberg/TestManifestCaching.java | 2 +-
.../apache/iceberg/TestManifestListVersions.java | 29 +-
.../org/apache/iceberg/TestManifestReader.java | 9 +-
.../apache/iceberg/TestManifestReaderStats.java | 42 +-
.../apache/iceberg/TestManifestWriterVersions.java | 11 +-
.../org/apache/iceberg/TestMetadataTableScans.java | 252 ++--
...stMetadataTableScansWithPartitionEvolution.java | 54 +-
.../apache/iceberg/TestMetadataUpdateParser.java | 27 +-
.../org/apache/iceberg/TestMetricsTruncation.java | 9 +-
.../java/org/apache/iceberg/TestOverwrite.java | 33 +-
.../java/org/apache/iceberg/TestPartitioning.java | 39 +-
.../java/org/apache/iceberg/TestRewriteFiles.java | 8 +-
.../java/org/apache/iceberg/TestScanSummary.java | 22 +-
.../apache/iceberg/TestSchemaAndMappingUpdate.java | 21 +-
.../iceberg/TestSequenceNumberForV2Table.java | 29 +-
.../test/java/org/apache/iceberg/TestSnapshot.java | 123 ++
.../org/apache/iceberg/TestSnapshotLoading.java | 5 +
.../java/org/apache/iceberg/TestSortOrder.java | 26 +-
.../java/org/apache/iceberg/TestSplitPlanning.java | 50 +-
.../java/org/apache/iceberg/TestTransaction.java | 61 +-
.../requests => }/TestUpdateRequirementParser.java | 80 +-
.../org/apache/iceberg/TestUpdateRequirements.java | 577 +++++++++
.../java/org/apache/iceberg/TestWapWorkflow.java | 79 +-
.../iceberg/actions/TestBinPackStrategy.java | 87 +-
.../apache/iceberg/actions/TestSortStrategy.java | 38 +-
.../java/org/apache/iceberg/avro/AvroDataTest.java | 6 +-
.../apache/iceberg/avro/TestAvroDataWriter.java | 35 +-
.../apache/iceberg/avro/TestAvroDeleteWriters.java | 74 +-
.../apache/iceberg/avro/TestAvroEncoderUtil.java | 9 +-
.../org/apache/iceberg/avro/TestAvroEnums.java | 17 +-
.../org/apache/iceberg/avro/TestAvroFileSplit.java | 106 +-
.../apache/iceberg/avro/TestAvroNameMapping.java | 13 +-
.../avro/TestAvroOptionsWithNonNullDefaults.java | 18 +-
.../iceberg/avro/TestAvroSchemaProjection.java | 16 +-
.../java/org/apache/iceberg/avro/TestHasIds.java | 12 +-
.../apache/iceberg/avro/TestReadProjection.java | 119 +-
.../apache/iceberg/avro/TestSchemaConversions.java | 55 +-
.../org/apache/iceberg/catalog/CatalogTests.java | 1346 ++++++++++---------
.../iceberg/catalog/TestTableIdentifierParser.java | 97 +-
.../iceberg/data/avro/TestDecoderResolver.java | 167 +++
.../apache/iceberg/deletes/TestEqualityFilter.java | 38 +-
.../apache/iceberg/deletes/TestPositionFilter.java | 94 +-
.../org/apache/iceberg/encryption/TestCiphers.java | 7 +-
.../iceberg/encryption/TestKeyMetadataParser.java | 48 +
.../iceberg/expressions/TestExpressionParser.java | 14 +-
.../apache/iceberg/hadoop/HadoopFileIOTest.java | 53 +-
.../apache/iceberg/hadoop/HadoopTableTestBase.java | 16 +-
.../apache/iceberg/hadoop/TestCachingCatalog.java | 83 +-
.../iceberg/hadoop/TestCatalogUtilDropTable.java | 50 +-
.../apache/iceberg/hadoop/TestHadoopCatalog.java | 351 +++--
.../apache/iceberg/hadoop/TestHadoopCommits.java | 273 ++--
.../apache/iceberg/hadoop/TestHadoopTables.java | 95 +-
.../org/apache/iceberg/hadoop/TestStaticTable.java | 69 +-
.../iceberg/hadoop/TestTableSerialization.java | 12 +-
.../iceberg/inmemory/TestInMemoryFileIO.java | 2 +-
.../iceberg/inmemory/TestInMemoryInputFile.java | 5 +-
.../iceberg/inmemory/TestInMemoryOutputFile.java | 2 +-
.../iceberg/io/TestByteBufferInputStreams.java | 320 ++---
.../java/org/apache/iceberg/io/TestIOUtil.java | 102 +-
.../iceberg/io/TestMultiBufferInputStream.java | 77 +-
.../org/apache/iceberg/io/TestResolvingIO.java | 11 +-
.../iceberg/io/TestSingleBufferInputStream.java | 81 +-
.../org/apache/iceberg/jdbc/TestJdbcCatalog.java | 288 ++---
.../iceberg/jdbc/TestJdbcTableConcurrency.java | 19 +-
.../java/org/apache/iceberg/jdbc/TestJdbcUtil.java | 2 +-
.../apache/iceberg/mapping/TestNameMapping.java | 26 +-
.../metrics/TestCommitMetricsResultParser.java | 2 +-
.../iceberg/metrics/TestCommitReportParser.java | 2 +-
.../iceberg/metrics/TestCounterResultParser.java | 2 +-
.../metrics/TestScanMetricsResultParser.java | 2 +-
.../org/apache/iceberg/metrics/TestScanReport.java | 2 +-
.../iceberg/metrics/TestScanReportParser.java | 2 +-
.../iceberg/metrics/TestTimerResultParser.java | 2 +-
.../iceberg/puffin/TestFileMetadataParser.java | 2 +-
.../apache/iceberg/puffin/TestPuffinFormat.java | 2 +-
.../apache/iceberg/puffin/TestPuffinReader.java | 2 +-
.../apache/iceberg/puffin/TestPuffinWriter.java | 2 +-
.../apache/iceberg/rest/RESTCatalogAdapter.java | 48 +-
.../iceberg/rest/RequestResponseTestBase.java | 9 +-
.../org/apache/iceberg/rest/TestHTTPClient.java | 31 +-
.../org/apache/iceberg/rest/TestRESTCatalog.java | 337 ++++-
.../java/org/apache/iceberg/rest/TestRESTUtil.java | 2 +-
.../org/apache/iceberg/rest/TestResourcePaths.java | 71 +-
.../apache/iceberg/rest/auth/TestOAuth2Util.java | 64 +-
.../TestCommitTransactionRequestParser.java | 175 +++
.../rest/requests/TestCreateNamespaceRequest.java | 89 +-
.../rest/requests/TestCreateTableRequest.java | 148 +--
.../rest/requests/TestRenameTableRequest.java | 94 +-
.../requests/TestReportMetricsRequestParser.java | 2 +-
.../TestUpdateNamespacePropertiesRequest.java | 133 +-
.../requests/TestUpdateTableRequestParser.java | 222 ++++
.../responses/TestCatalogErrorResponseParser.java | 17 +-
.../iceberg/rest/responses/TestConfigResponse.java | 118 +-
.../responses/TestCreateNamespaceResponse.java | 89 +-
.../rest/responses/TestGetNamespaceResponse.java | 85 +-
.../rest/responses/TestListNamespacesResponse.java | 72 +-
.../rest/responses/TestListTablesResponse.java | 94 +-
.../rest/responses/TestLoadTableResponse.java | 173 +--
.../responses/TestOAuthErrorResponseParser.java | 2 +-
.../rest/responses/TestOAuthTokenResponse.java | 57 +-
.../TestUpdateNamespacePropertiesResponse.java | 155 ++-
.../iceberg/util/TestInMemoryLockManager.java | 12 +-
.../java/org/apache/iceberg/util/TestJsonUtil.java | 80 +-
.../org/apache/iceberg/util/TestLocationUtil.java | 10 +-
.../org/apache/iceberg/util/TestSnapshotUtil.java | 110 +-
.../org/apache/iceberg/util/TestTableScanUtil.java | 11 +-
.../view/TestSQLViewRepresentationParser.java | 31 +-
.../iceberg/view/TestViewHistoryEntryParser.java | 17 +-
.../org/apache/iceberg/view/TestViewMetadata.java | 294 +++++
.../iceberg/view/TestViewMetadataParser.java | 184 +++
.../iceberg/view/TestViewRepresentationParser.java | 9 +-
.../apache/iceberg/view/TestViewVersionParser.java | 17 +-
.../org/apache/iceberg/view/ValidViewMetadata.json | 80 ++
.../view/ViewMetadataInvalidCurrentSchema.json | 80 ++
.../view/ViewMetadataInvalidCurrentVersion.json | 80 ++
.../iceberg/view/ViewMetadataLimitedVersions.json | 80 ++
.../view/ViewMetadataMissingCurrentSchema.json | 79 ++
.../view/ViewMetadataMissingCurrentVersion.json | 79 ++
.../iceberg/view/ViewMetadataMissingLocation.json | 79 ++
.../data/avro/TestSingleMessageEncoding.java | 2 +-
.../TestParquetEncryptionWithWriteSupport.java | 181 +++
.../apache/iceberg/io/TestPartitioningWriters.java | 95 ++
.../iceberg/io/TestPositionDeltaWriters.java | 25 +
.../dell/ecs/TestEcsAppendOutputStream.java | 17 +-
.../apache/iceberg/dell/ecs/TestEcsCatalog.java | 32 +-
.../apache/iceberg/dell/ecs/TestEcsInputFile.java | 16 +-
.../apache/iceberg/dell/ecs/TestEcsOutputFile.java | 17 +-
.../dell/ecs/TestEcsSeekableInputStream.java | 16 +-
.../org/apache/iceberg/dell/ecs/TestEcsURI.java | 10 +-
.../iceberg/dell/ecs/TestPropertiesSerDesUtil.java | 7 +-
.../iceberg/dell/mock/ecs/EcsS3MockRule.java | 5 +-
.../apache/iceberg/dell/mock/ecs/MockS3Client.java | 7 +-
docs/aws.md | 72 +-
docs/branching-and-tagging.md | 3 +-
docs/configuration.md | 91 +-
docs/dell.md | 1 +
docs/delta-lake-migration.md | 1 +
docs/evolution.md | 1 +
docs/flink-actions.md | 1 +
docs/flink-configuration.md | 46 +-
docs/flink-connector.md | 1 +
docs/flink-ddl.md | 9 +-
docs/flink-getting-started.md | 9 +-
docs/flink-queries.md | 3 +-
docs/flink-writes.md | 1 +
docs/hive-migration.md | 1 +
docs/java-api-quickstart.md | 1 +
docs/java-api.md | 1 +
docs/java-custom-catalog.md | 1 +
docs/jdbc.md | 1 +
docs/maintenance.md | 1 +
docs/nessie.md | 6 +-
docs/partitioning.md | 1 +
docs/performance.md | 1 +
docs/reliability.md | 1 +
docs/schemas.md | 1 +
docs/spark-configuration.md | 16 +
docs/spark-ddl.md | 1 +
docs/spark-getting-started.md | 1 +
docs/spark-procedures.md | 49 +-
docs/spark-queries.md | 1 +
docs/spark-structured-streaming.md | 1 +
docs/spark-writes.md | 169 ++-
docs/table-migration.md | 1 +
flink/v1.15/build.gradle | 2 +
.../org/apache/iceberg/flink/FlinkCatalog.java | 9 +-
.../iceberg/flink/FlinkDynamicTableFactory.java | 9 +-
.../org/apache/iceberg/flink/FlinkReadConf.java | 9 +
.../org/apache/iceberg/flink/FlinkReadOptions.java | 4 +
.../iceberg/flink/data/RowDataProjection.java | 90 +-
.../iceberg/flink/sink/BaseDeltaTaskWriter.java | 3 +-
.../iceberg/flink/sink/FlinkManifestUtil.java | 20 +-
.../org/apache/iceberg/flink/sink/FlinkSink.java | 3 +-
.../iceberg/flink/sink/IcebergFilesCommitter.java | 15 +-
.../iceberg/flink/sink/shuffle/DataStatistics.java | 18 +-
.../flink/sink/shuffle/DataStatisticsEvent.java | 8 +-
.../flink/sink/shuffle/DataStatisticsOperator.java | 51 +-
.../flink/sink/shuffle/DataStatisticsOrRecord.java | 53 +-
.../shuffle/DataStatisticsOrRecordSerializer.java | 219 ++++
.../flink/sink/shuffle/MapDataStatistics.java | 27 +-
.../sink/shuffle/MapDataStatisticsSerializer.java | 183 +++
.../apache/iceberg/flink/source/IcebergSource.java | 32 +-
.../apache/iceberg/flink/source/ScanContext.java | 24 +-
.../source/assigner/DefaultSplitAssigner.java} | 20 +-
...ctory.java => OrderedSplitAssignerFactory.java} | 22 +-
.../assigner/SimpleSplitAssignerFactory.java | 9 +-
.../enumerator/ContinuousIcebergEnumerator.java | 12 +-
.../flink/source/reader/IcebergSourceReader.java | 4 +-
.../source/reader/IcebergSourceSplitReader.java | 17 +-
.../split/SerializableComparator.java} | 17 +-
.../flink/source/split/SplitComparators.java | 59 +
.../org/apache/iceberg/flink/DataGenerator.java | 4 +-
.../org/apache/iceberg/flink/DataGenerators.java | 198 +--
.../org/apache/iceberg/flink/SimpleDataUtil.java | 39 +-
.../iceberg/flink/TestFlinkAnonymousTable.java | 64 +
.../iceberg/flink/TestFlinkCatalogDatabase.java | 51 +-
.../iceberg/flink/TestFlinkCatalogFactory.java | 8 +-
.../iceberg/flink/data/TestRowDataProjection.java | 507 ++++++--
.../iceberg/flink/sink/TestDeltaTaskWriter.java | 128 +-
.../iceberg/flink/sink/TestFlinkManifest.java | 9 +-
.../flink/sink/TestIcebergFilesCommitter.java | 148 ++-
.../sink/shuffle/TestDataStatisticsOperator.java | 186 +--
.../apache/iceberg/flink/source/SplitHelpers.java | 45 +-
.../source/assigner/SplitAssignerTestBase.java} | 20 +-
.../source/assigner/TestDefaultSplitAssigner.java} | 27 +-
.../TestFileSequenceNumberBasedSplitAssigner.java | 84 ++
.../enumerator/ManualContinuousSplitPlanner.java | 9 +-
.../TestContinuousIcebergEnumerator.java | 119 +-
.../source/reader/TestIcebergSourceReader.java | 71 +-
flink/v1.16/build.gradle | 4 +-
.../org/apache/iceberg/flink/FlinkCatalog.java | 9 +-
.../iceberg/flink/FlinkDynamicTableFactory.java | 9 +-
.../org/apache/iceberg/flink/FlinkReadConf.java | 9 +
.../org/apache/iceberg/flink/FlinkReadOptions.java | 4 +
.../iceberg/flink/data/RowDataProjection.java | 90 +-
.../iceberg/flink/sink/BaseDeltaTaskWriter.java | 3 +-
.../iceberg/flink/sink/FlinkManifestUtil.java | 20 +-
.../org/apache/iceberg/flink/sink/FlinkSink.java | 3 +-
.../iceberg/flink/sink/IcebergFilesCommitter.java | 15 +-
.../iceberg/flink/sink/shuffle/DataStatistics.java | 18 +-
.../flink/sink/shuffle/DataStatisticsEvent.java | 8 +-
.../flink/sink/shuffle/DataStatisticsOperator.java | 51 +-
.../flink/sink/shuffle/DataStatisticsOrRecord.java | 53 +-
.../shuffle/DataStatisticsOrRecordSerializer.java | 219 ++++
.../flink/sink/shuffle/MapDataStatistics.java | 27 +-
.../sink/shuffle/MapDataStatisticsFactory.java | 34 -
.../sink/shuffle/MapDataStatisticsSerializer.java | 183 +++
.../apache/iceberg/flink/source/IcebergSource.java | 32 +-
.../apache/iceberg/flink/source/ScanContext.java | 24 +-
.../source/assigner/DefaultSplitAssigner.java} | 20 +-
.../assigner/OrderedSplitAssignerFactory.java} | 22 +-
.../assigner/SimpleSplitAssignerFactory.java | 9 +-
.../enumerator/ContinuousIcebergEnumerator.java | 12 +-
.../flink/source/reader/IcebergSourceReader.java | 4 +-
.../source/reader/IcebergSourceSplitReader.java | 17 +-
.../source/split/SerializableComparator.java} | 15 +-
.../flink/source/split/SplitComparators.java | 59 +
.../org/apache/iceberg/flink/DataGenerator.java | 4 +-
.../org/apache/iceberg/flink/DataGenerators.java | 198 +--
.../org/apache/iceberg/flink/SimpleDataUtil.java | 39 +-
.../iceberg/flink/TestFlinkAnonymousTable.java | 64 +
.../iceberg/flink/TestFlinkCatalogDatabase.java | 51 +-
.../iceberg/flink/TestFlinkCatalogFactory.java | 8 +-
.../iceberg/flink/data/TestRowDataProjection.java | 507 ++++++--
.../iceberg/flink/sink/TestDeltaTaskWriter.java | 128 +-
.../iceberg/flink/sink/TestFlinkManifest.java | 9 +-
.../flink/sink/TestIcebergFilesCommitter.java | 148 ++-
.../sink/shuffle/TestDataStatisticsOperator.java | 186 +--
.../apache/iceberg/flink/source/SplitHelpers.java | 45 +-
...litAssigner.java => SplitAssignerTestBase.java} | 20 +-
.../source/assigner/TestDefaultSplitAssigner.java} | 27 +-
.../TestFileSequenceNumberBasedSplitAssigner.java | 84 ++
.../enumerator/ManualContinuousSplitPlanner.java | 9 +-
.../TestContinuousIcebergEnumerator.java | 119 +-
.../source/reader/TestIcebergSourceReader.java | 71 +-
.../iceberg/flink/util/TestFlinkPackage.java | 2 +-
flink/v1.17/build.gradle | 4 +-
.../org/apache/iceberg/flink/FlinkCatalog.java | 9 +-
.../iceberg/flink/FlinkDynamicTableFactory.java | 9 +-
.../org/apache/iceberg/flink/FlinkReadConf.java | 9 +
.../org/apache/iceberg/flink/FlinkReadOptions.java | 4 +
.../iceberg/flink/data/RowDataProjection.java | 90 +-
.../iceberg/flink/sink/BaseDeltaTaskWriter.java | 3 +-
.../iceberg/flink/sink/FlinkManifestUtil.java | 20 +-
.../org/apache/iceberg/flink/sink/FlinkSink.java | 3 +-
.../iceberg/flink/sink/IcebergFilesCommitter.java | 15 +-
.../iceberg/flink/sink/shuffle/DataStatistics.java | 18 +-
.../flink/sink/shuffle/DataStatisticsEvent.java | 8 +-
.../flink/sink/shuffle/DataStatisticsFactory.java | 33 -
.../flink/sink/shuffle/DataStatisticsOperator.java | 51 +-
.../flink/sink/shuffle/DataStatisticsOrRecord.java | 53 +-
.../shuffle/DataStatisticsOrRecordSerializer.java | 219 ++++
.../flink/sink/shuffle/MapDataStatistics.java | 27 +-
.../sink/shuffle/MapDataStatisticsFactory.java | 34 -
.../sink/shuffle/MapDataStatisticsSerializer.java | 183 +++
.../apache/iceberg/flink/source/IcebergSource.java | 32 +-
.../apache/iceberg/flink/source/ScanContext.java | 24 +-
...plitAssigner.java => DefaultSplitAssigner.java} | 20 +-
.../assigner/OrderedSplitAssignerFactory.java} | 22 +-
.../assigner/SimpleSplitAssignerFactory.java | 9 +-
.../enumerator/ContinuousIcebergEnumerator.java | 12 +-
.../flink/source/reader/IcebergSourceReader.java | 4 +-
.../source/reader/IcebergSourceSplitReader.java | 17 +-
.../source/split/SerializableComparator.java} | 15 +-
.../flink/source/split/SplitComparators.java | 59 +
.../org/apache/iceberg/flink/DataGenerator.java | 4 +-
.../org/apache/iceberg/flink/DataGenerators.java | 198 +--
.../org/apache/iceberg/flink/SimpleDataUtil.java | 39 +-
.../iceberg/flink/TestFlinkAnonymousTable.java | 64 +
.../iceberg/flink/TestFlinkCatalogDatabase.java | 27 +-
.../iceberg/flink/TestFlinkCatalogFactory.java | 8 +-
.../iceberg/flink/data/TestRowDataProjection.java | 488 +++++--
.../iceberg/flink/sink/TestDeltaTaskWriter.java | 128 +-
.../iceberg/flink/sink/TestFlinkManifest.java | 9 +-
.../flink/sink/TestIcebergFilesCommitter.java | 148 ++-
.../sink/shuffle/TestDataStatisticsOperator.java | 186 +--
.../apache/iceberg/flink/source/SplitHelpers.java | 45 +-
.../source/assigner/SplitAssignerTestBase.java} | 20 +-
.../source/assigner/TestDefaultSplitAssigner.java} | 27 +-
.../TestFileSequenceNumberBasedSplitAssigner.java | 84 ++
.../enumerator/ManualContinuousSplitPlanner.java | 9 +-
.../TestContinuousIcebergEnumerator.java | 119 +-
.../source/reader/TestIcebergSourceReader.java | 71 +-
.../iceberg/flink/util/TestFlinkPackage.java | 2 +-
format/spec.md | 35 +
format/view-spec.md | 4 +
.../org/apache/iceberg/gcp/gcs/GCSFileIOTest.java | 14 +-
.../apache/iceberg/gcp/gcs/GCSInputStreamTest.java | 17 +-
.../iceberg/gcp/gcs/GCSOutputStreamTest.java | 11 +-
gradle/wrapper/gradle-wrapper.properties | 4 +-
gradlew | 9 +-
hive3/build.gradle | 4 +-
jmh.gradle | 2 +
mr/build.gradle | 4 +-
.../org/apache/iceberg/nessie/NessieCatalog.java | 24 +-
.../apache/iceberg/nessie/NessieIcebergClient.java | 10 +-
.../iceberg/nessie/NessieTableOperations.java | 45 +-
.../java/org/apache/iceberg/nessie/NessieUtil.java | 60 +
.../org/apache/iceberg/nessie/BaseTestIceberg.java | 29 +-
.../iceberg/nessie/TestCustomNessieClient.java | 8 +-
.../apache/iceberg/nessie/TestNessieCatalog.java | 38 +-
.../iceberg/nessie/TestNessieIcebergClient.java | 13 +
.../org/apache/iceberg/nessie/TestNessieTable.java | 59 +-
python/pyiceberg/files.py => open-api/Makefile | 29 +-
open-api/README.md | 42 +
.../pyiceberg/__init__.py => open-api/header.txt | 4 +-
.../__init__.py => open-api/requirements.txt | 3 +-
open-api/rest-catalog-open-api.py | 811 ++++++++++++
open-api/rest-catalog-open-api.yaml | 433 +++++--
.../org/apache/iceberg/orc/TestOrcDataWriter.java | 8 +-
.../java/org/apache/iceberg/parquet/Parquet.java | 135 +-
.../iceberg/parquet/ParquetCodecFactory.java | 6 +-
.../org/apache/iceberg/parquet/ParquetWriter.java | 63 +-
.../java/org/apache/iceberg/parquet/ReadConf.java | 9 +-
.../iceberg/parquet/TestBloomRowGroupFilter.java | 2 +
.../iceberg/parquet/TestParquetEncryption.java | 127 ++
.../org/apache/iceberg/pig/SchemaUtilTest.java | 18 +-
python/.pre-commit-config.yaml | 28 +-
python/Makefile | 12 +-
python/dev/Dockerfile | 4 +-
python/dev/entrypoint.sh | 2 +-
python/dev/provision.py | 140 +-
python/mkdocs/docs/api.md | 168 +--
python/mkdocs/docs/configuration.md | 16 +-
python/mkdocs/docs/contributing.md | 16 +
python/mkdocs/docs/verify-release.md | 8 +-
python/mkdocs/requirements.txt | 10 +-
python/poetry.lock | 1327 ++++++++++---------
python/pyiceberg/__init__.py | 2 +-
python/pyiceberg/avro/codecs/snappy_codec.py | 2 +-
python/pyiceberg/avro/decoder.py | 61 +-
python/pyiceberg/avro/file.py | 21 +-
python/pyiceberg/avro/reader.py | 32 +-
python/pyiceberg/avro/resolver.py | 71 +-
python/pyiceberg/catalog/__init__.py | 134 +-
python/pyiceberg/catalog/dynamodb.py | 90 +-
python/pyiceberg/catalog/glue.py | 91 +-
python/pyiceberg/catalog/hive.py | 76 +-
python/pyiceberg/catalog/noop.py | 79 ++
python/pyiceberg/catalog/rest.py | 93 +-
python/pyiceberg/cli/console.py | 46 +-
python/pyiceberg/cli/output.py | 10 +-
python/pyiceberg/conversions.py | 48 +-
python/pyiceberg/exceptions.py | 44 +-
python/pyiceberg/expressions/__init__.py | 141 +-
python/pyiceberg/expressions/literals.py | 28 +-
python/pyiceberg/expressions/parser.py | 2 +-
python/pyiceberg/expressions/visitors.py | 157 +--
python/pyiceberg/files.py | 4 +-
python/pyiceberg/io/__init__.py | 88 +-
python/pyiceberg/io/fsspec.py | 60 +-
python/pyiceberg/io/memory.py | 4 +-
python/pyiceberg/io/pyarrow.py | 206 ++-
python/pyiceberg/manifest.py | 128 +-
python/pyiceberg/partitioning.py | 40 +-
python/pyiceberg/schema.py | 325 ++---
python/pyiceberg/serializers.py | 24 +-
python/pyiceberg/table/__init__.py | 481 ++++++-
python/pyiceberg/table/metadata.py | 46 +-
python/pyiceberg/table/refs.py | 1 +
python/pyiceberg/table/snapshots.py | 8 +-
python/pyiceberg/table/sorting.py | 17 +-
python/pyiceberg/transforms.py | 37 +-
python/pyiceberg/typedef.py | 12 +-
python/pyiceberg/types.py | 108 +-
python/pyiceberg/utils/bin_packing.py | 2 +
python/pyiceberg/utils/config.py | 29 +-
python/pyiceberg/utils/datetime.py | 41 +-
python/pyiceberg/utils/decimal.py | 38 +-
python/pyiceberg/utils/deprecated.py | 6 +-
python/pyiceberg/utils/parsing.py | 2 +-
python/pyiceberg/utils/schema_conversion.py | 108 +-
python/pyiceberg/utils/singleton.py | 2 +-
python/pyproject.toml | 33 +-
python/tests/avro/test_resolver.py | 21 +
python/tests/catalog/integration_test_dynamodb.py | 2 +-
python/tests/catalog/integration_test_glue.py | 2 +-
python/tests/catalog/test_base.py | 7 +-
python/tests/catalog/test_glue.py | 20 +
python/tests/catalog/test_rest.py | 8 +-
python/tests/cli/test_console.py | 9 +-
python/tests/conftest.py | 172 ++-
python/tests/expressions/test_expressions.py | 38 +-
python/tests/expressions/test_parser.py | 92 +-
python/tests/expressions/test_visitors.py | 21 +
python/tests/io/test_pyarrow.py | 117 +-
python/tests/table/test_init.py | 145 ++-
python/tests/test_integration.py | 142 ++-
python/tests/test_types.py | 8 +
python/tests/utils/test_config.py | 5 +-
python/tests/utils/test_manifest.py | 91 +-
python/tests/utils/test_schema_conversion.py | 4 +-
.../iceberg/snowflake/JdbcSnowflakeClient.java | 57 +-
.../apache/iceberg/snowflake/SnowflakeCatalog.java | 18 +-
.../iceberg/snowflake/JdbcSnowflakeClientTest.java | 363 +++++-
.../datasources/v2/CreateOrReplaceBranchExec.scala | 14 +-
.../iceberg/spark/extensions/TestBranchDDL.java | 10 +
.../spark/source/TestIcebergSourceTablesBase.java | 184 ++-
.../IcebergSqlExtensions.g4 | 20 +-
.../IcebergSparkSqlExtensionsParser.scala | 5 +-
.../IcebergSqlExtensionsAstBuilder.scala | 39 +
.../plans/logical/CreateOrReplaceTag.scala | 0
.../spark/sql/catalyst/plans/logical/DropTag.scala | 0
.../sql/catalyst/plans/logical/TagOptions.scala | 0
.../datasources/v2/CreateOrReplaceBranchExec.scala | 14 +-
.../datasources/v2/CreateOrReplaceTagExec.scala | 14 +-
.../execution/datasources/v2/DropTagExec.scala} | 40 +-
.../v2/ExtendedDataSourceV2Strategy.scala | 10 +-
.../iceberg/spark/extensions/TestBranchDDL.java | 10 +
.../iceberg/spark/extensions/TestTagDDL.java | 10 +
.../apache/iceberg/spark/ChangelogIterator.java | 191 +--
.../iceberg/spark/ComputeUpdateIterator.java | 142 +++
.../iceberg/spark/RemoveCarryoverIterator.java | 163 +++
.../apache/iceberg/spark/SparkSessionCatalog.java | 6 +-
.../vectorized/VectorizedSparkParquetReaders.java | 57 -
.../procedures/CreateChangelogViewProcedure.java | 17 +-
.../apache/iceberg/spark/source/SparkTable.java | 18 +-
.../iceberg/spark/TestChangelogIterator.java | 141 +-
.../org/apache/iceberg/spark/data/TestHelpers.java | 17 +-
.../data/TestSparkParquetReadMetadataColumns.java | 9 +-
...estParquetDictionaryEncodedVectorizedReads.java | 1 -
.../vectorized/TestParquetVectorizedReads.java | 48 +-
.../spark/source/TestDataSourceOptions.java | 55 +-
.../spark/source/TestIcebergSourceTablesBase.java | 196 ++-
spark/v3.3/build.gradle | 2 +
.../spark/MergeCardinalityCheckBenchmark.java | 230 ++++
.../catalyst/analysis/RewriteMergeIntoTable.scala | 32 +-
.../sql/catalyst/plans/logical/MergeRows.scala | 1 -
.../datasources/v2/CreateOrReplaceBranchExec.scala | 14 +-
.../datasources/v2/CreateOrReplaceTagExec.scala | 14 +-
.../v2/ExtendedDataSourceV2Strategy.scala | 4 +-
.../execution/datasources/v2/MergeRowsExec.scala | 30 +-
.../iceberg/spark/extensions/TestBranchDDL.java | 10 +
.../spark/extensions/TestIcebergExpressions.java | 31 +
.../TestRequiredDistributionAndOrdering.java | 17 +
.../extensions/TestRewriteDataFilesProcedure.java | 43 +
.../TestRewritePositionDeleteFilesProcedure.java | 152 +++
.../iceberg/spark/extensions/TestTagDDL.java | 10 +
.../SparkParquetReadersFlatDataBenchmark.java | 2 +
.../SparkParquetReadersNestedDataBenchmark.java | 2 +
.../apache/iceberg/spark/ChangelogIterator.java | 191 +--
.../iceberg/spark/ComputeUpdateIterator.java | 142 +++
.../iceberg/spark/RemoveCarryoverIterator.java | 163 +++
.../apache/iceberg/spark/SparkSessionCatalog.java | 6 +-
.../spark/actions/RewriteDataFilesSparkAction.java | 163 ++-
.../RewritePositionDeleteFilesSparkAction.java} | 168 +--
.../apache/iceberg/spark/actions/SparkActions.java | 5 +
.../SparkBinPackPositionDeletesRewriter.java | 0
.../vectorized/VectorizedSparkParquetReaders.java | 57 -
.../procedures/CreateChangelogViewProcedure.java | 17 +-
.../RewritePositionDeleteFilesProcedure.java | 109 ++
.../iceberg/spark/procedures/SparkProcedures.java | 1 +
.../iceberg/spark/source/SparkScanBuilder.java | 5 +
.../apache/iceberg/spark/source/SparkTable.java | 5 +-
.../expressions/TransformExpressions.scala | 3 +
.../datasources/SparkExpressionConverter.scala | 4 +
.../iceberg/spark/TestChangelogIterator.java | 141 +-
.../spark/actions/TestRewriteDataFilesAction.java | 3 +-
.../TestRewritePositionDeleteFilesAction.java | 145 ++-
.../org/apache/iceberg/spark/data/TestHelpers.java | 17 +-
.../data/TestSparkParquetReadMetadataColumns.java | 9 +-
...estParquetDictionaryEncodedVectorizedReads.java | 6 +-
...naryFallbackToPlainEncodingVectorizedReads.java | 3 +-
.../vectorized/TestParquetVectorizedReads.java | 128 +-
.../iceberg/spark/source/FourColumnRecord.java | 0
.../spark/source/TestDataSourceOptions.java | 65 +-
.../spark/source/TestIcebergSourceTablesBase.java | 196 ++-
spark/v3.4/build.gradle | 8 +-
.../spark/MergeCardinalityCheckBenchmark.java | 230 ++++
.../iceberg/spark/UpdateProjectionBenchmark.java | 213 ++++
.../catalyst/analysis/RewriteMergeIntoTable.scala | 180 ++-
.../analysis/RewriteRowLevelIcebergCommand.scala | 96 ++
.../sql/catalyst/analysis/RewriteUpdateTable.scala | 43 +-
.../sql/catalyst/plans/logical/MergeRows.scala | 3 +-
.../sql/catalyst/plans/logical/UpdateRows.scala} | 20 +-
.../datasources/v2/CreateOrReplaceBranchExec.scala | 14 +-
.../datasources/v2/CreateOrReplaceTagExec.scala | 14 +-
.../v2/ExtendedDataSourceV2Strategy.scala | 13 +-
.../execution/datasources/v2/MergeRowsExec.scala | 288 +++--
.../execution/datasources/v2/UpdateRowsExec.scala | 86 ++
.../SparkRowLevelOperationsTestBase.java | 21 +-
.../spark/extensions/TestAddFilesProcedure.java | 110 +-
.../spark/extensions/TestAlterTableSchema.java | 52 +-
.../spark/extensions/TestAncestorsOfProcedure.java | 31 +-
.../iceberg/spark/extensions/TestBranchDDL.java | 104 +-
.../spark/extensions/TestCallStatementParser.java | 10 +-
.../TestCherrypickSnapshotProcedure.java | 63 +-
.../spark/extensions/TestConflictValidation.java | 230 ++--
.../spark/extensions/TestCopyOnWriteDelete.java | 11 +-
.../spark/extensions/TestCopyOnWriteMerge.java | 11 +-
.../spark/extensions/TestCopyOnWriteUpdate.java | 11 +-
.../iceberg/spark/extensions/TestDelete.java | 29 +-
.../apache/iceberg/spark/extensions/TestMerge.java | 45 +-
.../spark/extensions/TestMergeOnReadDelete.java | 22 +-
.../spark/extensions/TestMergeOnReadMerge.java | 11 +-
.../spark/extensions/TestMergeOnReadUpdate.java | 11 +-
.../extensions/TestRemoveOrphanFilesProcedure.java | 158 ++-
.../spark/extensions/TestReplaceBranch.java | 29 +-
.../TestRequiredDistributionAndOrdering.java | 17 +
.../extensions/TestRewriteDataFilesProcedure.java | 140 +-
.../extensions/TestRewriteManifestsProcedure.java | 64 +-
.../TestRewritePositionDeleteFilesProcedure.java | 152 +++
.../iceberg/spark/extensions/TestTagDDL.java | 10 +
.../iceberg/spark/extensions/TestUpdate.java | 11 +-
.../iceberg/spark/extensions/TestWriteAborts.java | 31 +-
.../SparkParquetReadersFlatDataBenchmark.java | 4 +
.../SparkParquetReadersNestedDataBenchmark.java | 4 +
.../iceberg/spark/source/WritersBenchmark.java | 86 +-
.../apache/iceberg/spark/ChangelogIterator.java | 191 +--
.../iceberg/spark/ComputeUpdateIterator.java | 142 +++
.../iceberg/spark/PruneColumnsWithReordering.java | 50 +-
.../spark/PruneColumnsWithoutReordering.java | 52 +-
.../iceberg/spark/RemoveCarryoverIterator.java | 163 +++
.../java/org/apache/iceberg/spark/Spark3Util.java | 7 +
.../org/apache/iceberg/spark/SparkCatalog.java | 13 +-
.../org/apache/iceberg/spark/SparkConfParser.java | 9 +-
.../spark/SparkDistributionAndOrderingUtil.java | 239 ----
.../org/apache/iceberg/spark/SparkFilters.java | 3 +
.../iceberg/spark/SparkFixupTimestampType.java | 57 -
.../org/apache/iceberg/spark/SparkReadConf.java | 24 +-
.../org/apache/iceberg/spark/SparkReadOptions.java | 1 +
.../apache/iceberg/spark/SparkSQLProperties.java | 4 +-
.../org/apache/iceberg/spark/SparkSchemaUtil.java | 27 +-
.../apache/iceberg/spark/SparkSessionCatalog.java | 6 +-
.../org/apache/iceberg/spark/SparkTypeToType.java | 4 +
.../java/org/apache/iceberg/spark/SparkUtil.java | 78 +-
.../apache/iceberg/spark/SparkValueConverter.java | 15 +-
.../org/apache/iceberg/spark/SparkWriteConf.java | 101 +-
.../apache/iceberg/spark/SparkWriteOptions.java | 1 +
.../iceberg/spark/SparkWriteRequirements.java | 50 +
.../org/apache/iceberg/spark/SparkWriteUtil.java | 251 ++++
.../org/apache/iceberg/spark/TypeToSparkType.java | 8 +-
.../spark/actions/RewriteDataFilesSparkAction.java | 180 ++-
... => RewritePositionDeleteFilesSparkAction.java} | 166 +--
.../apache/iceberg/spark/actions/SparkActions.java | 4 +-
.../spark/actions/SparkShufflingDataRewriter.java | 51 +-
.../vectorized/VectorizedSparkParquetReaders.java | 33 -
.../iceberg/spark/functions/BucketFunction.java | 4 +
.../iceberg/spark/functions/DaysFunction.java | 26 +
.../iceberg/spark/functions/HoursFunction.java | 36 +
.../iceberg/spark/functions/MonthsFunction.java | 26 +
.../iceberg/spark/functions/YearsFunction.java | 26 +
.../procedures/CreateChangelogViewProcedure.java | 17 +-
.../procedures/RewriteDataFilesProcedure.java | 12 +-
.../RewritePositionDeleteFilesProcedure.java | 109 ++
.../iceberg/spark/procedures/SparkProcedures.java | 1 +
.../iceberg/spark/source/SparkChangelogScan.java | 8 -
.../source/SparkPositionDeletesRewriteBuilder.java | 5 -
.../spark/source/SparkPositionDeltaWrite.java | 183 +--
.../source/SparkPositionDeltaWriteBuilder.java | 40 +-
.../org/apache/iceberg/spark/source/SparkScan.java | 7 -
.../iceberg/spark/source/SparkScanBuilder.java | 5 +
.../apache/iceberg/spark/source/SparkTable.java | 5 +
.../apache/iceberg/spark/source/SparkWrite.java | 14 +-
.../iceberg/spark/source/SparkWriteBuilder.java | 69 +-
.../plans/logical/OrderAwareCoalesce.scala} | 47 +-
.../sql/execution/OrderAwareCoalesceExec.scala | 59 +
.../datasources/SparkExpressionConverter.scala | 4 +
.../iceberg/spark/SparkTestBaseWithCatalog.java | 2 +-
.../iceberg/spark/TestChangelogIterator.java | 141 +-
.../TestSparkDistributionAndOrderingUtil.java | 1348 +++++++++++++++++---
.../org/apache/iceberg/spark/TestSparkFilters.java | 17 +
.../apache/iceberg/spark/TestSparkWriteConf.java | 61 +-
.../spark/actions/TestRewriteDataFilesAction.java | 20 +-
.../TestRewritePositionDeleteFilesAction.java | 145 ++-
.../spark/actions/TestSparkFileRewriter.java | 2 +
.../apache/iceberg/spark/data/AvroDataTest.java | 20 +-
.../apache/iceberg/spark/data/GenericsHelpers.java | 20 +-
.../org/apache/iceberg/spark/data/TestHelpers.java | 43 +-
.../data/TestSparkParquetReadMetadataColumns.java | 3 +-
...estParquetDictionaryEncodedVectorizedReads.java | 1 -
.../vectorized/TestParquetVectorizedReads.java | 48 +-
.../spark/source/TestDataSourceOptions.java | 65 +-
.../spark/source/TestIcebergSourceTablesBase.java | 218 +++-
.../source/TestMetadataTableReadableMetrics.java | 67 +-
.../iceberg/spark/source/TestSparkDataFile.java | 2 +
.../spark/source/TestTimestampWithoutZone.java | 64 -
.../iceberg/spark/sql/TestSparkBucketFunction.java | 7 +
.../iceberg/spark/sql/TestSparkDaysFunction.java | 19 +-
.../iceberg/spark/sql/TestSparkHoursFunction.java | 17 +
.../iceberg/spark/sql/TestSparkMonthsFunction.java | 17 +
.../iceberg/spark/sql/TestSparkYearsFunction.java | 17 +
.../spark/sql/TestStoragePartitionedJoins.java | 25 +
.../spark/sql/TestTimestampWithoutZone.java | 266 ++--
versions.props | 14 +-
790 files changed, 35873 insertions(+), 14578 deletions(-)
create mode 100644
aws/src/main/java/org/apache/iceberg/aws/AwsClientProperties.java
create mode 100644
aws/src/main/java/org/apache/iceberg/aws/HttpClientProperties.java
create mode 100644
aws/src/main/java/org/apache/iceberg/aws/S3FileIOAwsClientFactories.java
create mode 100644
aws/src/main/java/org/apache/iceberg/aws/s3/DefaultS3FileIOAwsClientFactory.java
rename
flink/v1.15/flink/src/main/java/org/apache/iceberg/flink/sink/shuffle/MapDataStatisticsFactory.java
=> aws/src/main/java/org/apache/iceberg/aws/s3/S3FileIOAwsClientFactory.java
(63%)
create mode 100644
aws/src/test/java/org/apache/iceberg/aws/AwsClientPropertiesTest.java
create mode 100644
aws/src/test/java/org/apache/iceberg/aws/HttpClientPropertiesTest.java
create mode 100644
aws/src/test/java/org/apache/iceberg/aws/TestS3FileIOAwsClientFactories.java
create mode 100644
core/src/jmh/java/org/apache/iceberg/ManifestReadBenchmark.java
create mode 100644 core/src/main/java/org/apache/iceberg/ContentFileParser.java
create mode 100644
core/src/main/java/org/apache/iceberg/FileScanTaskParser.java
create mode 100644 core/src/main/java/org/apache/iceberg/SystemConfigs.java
create mode 100644 core/src/main/java/org/apache/iceberg/UpdateRequirement.java
copy core/src/main/java/org/apache/iceberg/{rest/requests =>
}/UpdateRequirementParser.java (97%)
create mode 100644
core/src/main/java/org/apache/iceberg/UpdateRequirements.java
create mode 100644
core/src/main/java/org/apache/iceberg/catalog/TableCommit.java
create mode 100644
core/src/main/java/org/apache/iceberg/data/avro/RawDecoder.java
create mode 100644
core/src/main/java/org/apache/iceberg/deletes/SortingPositionOnlyDeleteWriter.java
create mode 100644
core/src/main/java/org/apache/iceberg/encryption/KeyMetadata.java
create mode 100644
core/src/main/java/org/apache/iceberg/encryption/KeyMetadataDecoder.java
create mode 100644
core/src/main/java/org/apache/iceberg/encryption/KeyMetadataEncoder.java
create mode 100644
core/src/main/java/org/apache/iceberg/io/FanoutPositionOnlyDeleteWriter.java
create mode 100644
core/src/main/java/org/apache/iceberg/rest/requests/CommitTransactionRequest.java
create mode 100644
core/src/main/java/org/apache/iceberg/rest/requests/CommitTransactionRequestParser.java
create mode 100644
core/src/main/java/org/apache/iceberg/rest/requests/UpdateTableRequestParser.java
create mode 100644 core/src/main/java/org/apache/iceberg/view/ViewMetadata.java
create mode 100644
core/src/main/java/org/apache/iceberg/view/ViewMetadataParser.java
rename
flink/v1.16/flink/src/main/java/org/apache/iceberg/flink/sink/shuffle/DataStatisticsFactory.java
=> core/src/main/java/org/apache/iceberg/view/ViewProperties.java (67%)
create mode 100644
core/src/test/java/org/apache/iceberg/TestContentFileParser.java
create mode 100644
core/src/test/java/org/apache/iceberg/TestFileScanTaskParser.java
rename core/src/test/java/org/apache/iceberg/{rest/requests =>
}/TestUpdateRequirementParser.java (87%)
create mode 100644
core/src/test/java/org/apache/iceberg/TestUpdateRequirements.java
create mode 100644
core/src/test/java/org/apache/iceberg/data/avro/TestDecoderResolver.java
create mode 100644
core/src/test/java/org/apache/iceberg/encryption/TestKeyMetadataParser.java
create mode 100644
core/src/test/java/org/apache/iceberg/rest/requests/TestCommitTransactionRequestParser.java
create mode 100644
core/src/test/java/org/apache/iceberg/rest/requests/TestUpdateTableRequestParser.java
create mode 100644
core/src/test/java/org/apache/iceberg/view/TestViewMetadata.java
create mode 100644
core/src/test/java/org/apache/iceberg/view/TestViewMetadataParser.java
create mode 100644
core/src/test/resources/org/apache/iceberg/view/ValidViewMetadata.json
create mode 100644
core/src/test/resources/org/apache/iceberg/view/ViewMetadataInvalidCurrentSchema.json
create mode 100644
core/src/test/resources/org/apache/iceberg/view/ViewMetadataInvalidCurrentVersion.json
create mode 100644
core/src/test/resources/org/apache/iceberg/view/ViewMetadataLimitedVersions.json
create mode 100644
core/src/test/resources/org/apache/iceberg/view/ViewMetadataMissingCurrentSchema.json
create mode 100644
core/src/test/resources/org/apache/iceberg/view/ViewMetadataMissingCurrentVersion.json
create mode 100644
core/src/test/resources/org/apache/iceberg/view/ViewMetadataMissingLocation.json
create mode 100644
data/src/test/java/org/apache/iceberg/data/parquet/TestParquetEncryptionWithWriteSupport.java
create mode 100644
flink/v1.15/flink/src/main/java/org/apache/iceberg/flink/sink/shuffle/DataStatisticsOrRecordSerializer.java
create mode 100644
flink/v1.15/flink/src/main/java/org/apache/iceberg/flink/sink/shuffle/MapDataStatisticsSerializer.java
rename
flink/{v1.16/flink/src/main/java/org/apache/iceberg/flink/source/assigner/SimpleSplitAssigner.java
=>
v1.15/flink/src/main/java/org/apache/iceberg/flink/source/assigner/DefaultSplitAssigner.java}
(82%)
copy
flink/v1.15/flink/src/main/java/org/apache/iceberg/flink/source/assigner/{SimpleSplitAssignerFactory.java
=> OrderedSplitAssignerFactory.java} (54%)
rename
flink/v1.15/flink/src/main/java/org/apache/iceberg/flink/{sink/shuffle/DataStatisticsFactory.java
=> source/split/SerializableComparator.java} (67%)
create mode 100644
flink/v1.15/flink/src/main/java/org/apache/iceberg/flink/source/split/SplitComparators.java
create mode 100644
flink/v1.15/flink/src/test/java/org/apache/iceberg/flink/TestFlinkAnonymousTable.java
rename
flink/{v1.17/flink/src/test/java/org/apache/iceberg/flink/source/assigner/TestSimpleSplitAssigner.java
=>
v1.15/flink/src/test/java/org/apache/iceberg/flink/source/assigner/SplitAssignerTestBase.java}
(88%)
copy
flink/{v1.16/flink/src/main/java/org/apache/iceberg/flink/source/assigner/SimpleSplitAssignerFactory.java
=>
v1.15/flink/src/test/java/org/apache/iceberg/flink/source/assigner/TestDefaultSplitAssigner.java}
(52%)
create mode 100644
flink/v1.15/flink/src/test/java/org/apache/iceberg/flink/source/assigner/TestFileSequenceNumberBasedSplitAssigner.java
create mode 100644
flink/v1.16/flink/src/main/java/org/apache/iceberg/flink/sink/shuffle/DataStatisticsOrRecordSerializer.java
delete mode 100644
flink/v1.16/flink/src/main/java/org/apache/iceberg/flink/sink/shuffle/MapDataStatisticsFactory.java
create mode 100644
flink/v1.16/flink/src/main/java/org/apache/iceberg/flink/sink/shuffle/MapDataStatisticsSerializer.java
rename
flink/{v1.15/flink/src/main/java/org/apache/iceberg/flink/source/assigner/SimpleSplitAssigner.java
=>
v1.16/flink/src/main/java/org/apache/iceberg/flink/source/assigner/DefaultSplitAssigner.java}
(82%)
copy
flink/{v1.15/flink/src/main/java/org/apache/iceberg/flink/source/assigner/SimpleSplitAssignerFactory.java
=>
v1.16/flink/src/main/java/org/apache/iceberg/flink/source/assigner/OrderedSplitAssignerFactory.java}
(54%)
copy
flink/{v1.17/flink/src/test/java/org/apache/iceberg/flink/util/TestFlinkPackage.java
=>
v1.16/flink/src/main/java/org/apache/iceberg/flink/source/split/SerializableComparator.java}
(72%)
create mode 100644
flink/v1.16/flink/src/main/java/org/apache/iceberg/flink/source/split/SplitComparators.java
create mode 100644
flink/v1.16/flink/src/test/java/org/apache/iceberg/flink/TestFlinkAnonymousTable.java
rename
flink/v1.16/flink/src/test/java/org/apache/iceberg/flink/source/assigner/{TestSimpleSplitAssigner.java
=> SplitAssignerTestBase.java} (88%)
copy
flink/v1.16/flink/src/{main/java/org/apache/iceberg/flink/source/assigner/SimpleSplitAssignerFactory.java
=>
test/java/org/apache/iceberg/flink/source/assigner/TestDefaultSplitAssigner.java}
(52%)
create mode 100644
flink/v1.16/flink/src/test/java/org/apache/iceberg/flink/source/assigner/TestFileSequenceNumberBasedSplitAssigner.java
delete mode 100644
flink/v1.17/flink/src/main/java/org/apache/iceberg/flink/sink/shuffle/DataStatisticsFactory.java
create mode 100644
flink/v1.17/flink/src/main/java/org/apache/iceberg/flink/sink/shuffle/DataStatisticsOrRecordSerializer.java
delete mode 100644
flink/v1.17/flink/src/main/java/org/apache/iceberg/flink/sink/shuffle/MapDataStatisticsFactory.java
create mode 100644
flink/v1.17/flink/src/main/java/org/apache/iceberg/flink/sink/shuffle/MapDataStatisticsSerializer.java
rename
flink/v1.17/flink/src/main/java/org/apache/iceberg/flink/source/assigner/{SimpleSplitAssigner.java
=> DefaultSplitAssigner.java} (82%)
copy
flink/{v1.15/flink/src/main/java/org/apache/iceberg/flink/source/assigner/SimpleSplitAssignerFactory.java
=>
v1.17/flink/src/main/java/org/apache/iceberg/flink/source/assigner/OrderedSplitAssignerFactory.java}
(54%)
copy
flink/v1.17/flink/src/{test/java/org/apache/iceberg/flink/util/TestFlinkPackage.java
=>
main/java/org/apache/iceberg/flink/source/split/SerializableComparator.java}
(72%)
create mode 100644
flink/v1.17/flink/src/main/java/org/apache/iceberg/flink/source/split/SplitComparators.java
create mode 100644
flink/v1.17/flink/src/test/java/org/apache/iceberg/flink/TestFlinkAnonymousTable.java
rename
flink/{v1.15/flink/src/test/java/org/apache/iceberg/flink/source/assigner/TestSimpleSplitAssigner.java
=>
v1.17/flink/src/test/java/org/apache/iceberg/flink/source/assigner/SplitAssignerTestBase.java}
(88%)
copy
flink/{v1.16/flink/src/main/java/org/apache/iceberg/flink/source/assigner/SimpleSplitAssignerFactory.java
=>
v1.17/flink/src/test/java/org/apache/iceberg/flink/source/assigner/TestDefaultSplitAssigner.java}
(52%)
create mode 100644
flink/v1.17/flink/src/test/java/org/apache/iceberg/flink/source/assigner/TestFileSequenceNumberBasedSplitAssigner.java
copy python/pyiceberg/files.py => open-api/Makefile (63%)
create mode 100644 open-api/README.md
copy python/pyiceberg/__init__.py => open-api/header.txt (94%)
copy python/pyiceberg/__init__.py => open-api/requirements.txt (89%)
create mode 100644 open-api/rest-catalog-open-api.py
create mode 100644
parquet/src/test/java/org/apache/iceberg/parquet/TestParquetEncryption.java
create mode 100644 python/pyiceberg/catalog/noop.py
copy spark/{v3.3 =>
v3.2}/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/CreateOrReplaceTag.scala
(100%)
copy spark/{v3.3 =>
v3.2}/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/DropTag.scala
(100%)
copy spark/{v3.3 =>
v3.2}/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/TagOptions.scala
(100%)
copy spark/{v3.3 =>
v3.2}/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/CreateOrReplaceTagExec.scala
(81%)
copy
spark/{v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/CreateOrReplaceTagExec.scala
=>
v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropTagExec.scala}
(55%)
copy spark/{v3.3 =>
v3.2}/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestTagDDL.java
(97%)
create mode 100644
spark/v3.2/spark/src/main/java/org/apache/iceberg/spark/ComputeUpdateIterator.java
create mode 100644
spark/v3.2/spark/src/main/java/org/apache/iceberg/spark/RemoveCarryoverIterator.java
create mode 100644
spark/v3.3/spark-extensions/src/jmh/java/org/apache/iceberg/spark/MergeCardinalityCheckBenchmark.java
create mode 100644
spark/v3.3/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestRewritePositionDeleteFilesProcedure.java
create mode 100644
spark/v3.3/spark/src/main/java/org/apache/iceberg/spark/ComputeUpdateIterator.java
create mode 100644
spark/v3.3/spark/src/main/java/org/apache/iceberg/spark/RemoveCarryoverIterator.java
copy
spark/{v3.4/spark/src/main/java/org/apache/iceberg/spark/actions/RewritePositionDeleteSparkAction.java
=>
v3.3/spark/src/main/java/org/apache/iceberg/spark/actions/RewritePositionDeleteFilesSparkAction.java}
(78%)
copy spark/{v3.4 =>
v3.3}/spark/src/main/java/org/apache/iceberg/spark/actions/SparkBinPackPositionDeletesRewriter.java
(100%)
create mode 100644
spark/v3.3/spark/src/main/java/org/apache/iceberg/spark/procedures/RewritePositionDeleteFilesProcedure.java
copy spark/{v3.4 =>
v3.3}/spark/src/test/java/org/apache/iceberg/spark/actions/TestRewritePositionDeleteFilesAction.java
(85%)
copy spark/{v3.4 =>
v3.3}/spark/src/test/java/org/apache/iceberg/spark/source/FourColumnRecord.java
(100%)
create mode 100644
spark/v3.4/spark-extensions/src/jmh/java/org/apache/iceberg/spark/MergeCardinalityCheckBenchmark.java
create mode 100644
spark/v3.4/spark-extensions/src/jmh/java/org/apache/iceberg/spark/UpdateProjectionBenchmark.java
copy
spark/{v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/MergeRows.scala
=>
v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/UpdateRows.scala}
(71%)
create mode 100644
spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/UpdateRowsExec.scala
create mode 100644
spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestRewritePositionDeleteFilesProcedure.java
create mode 100644
spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/ComputeUpdateIterator.java
create mode 100644
spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/RemoveCarryoverIterator.java
delete mode 100644
spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/SparkDistributionAndOrderingUtil.java
delete mode 100644
spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/SparkFixupTimestampType.java
create mode 100644
spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/SparkWriteRequirements.java
create mode 100644
spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/SparkWriteUtil.java
rename
spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/actions/{RewritePositionDeleteSparkAction.java
=> RewritePositionDeleteFilesSparkAction.java} (79%)
create mode 100644
spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/procedures/RewritePositionDeleteFilesProcedure.java
copy
spark/{v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/MergeRows.scala
=>
v3.4/spark/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/OrderAwareCoalesce.scala}
(50%)
create mode 100644
spark/v3.4/spark/src/main/scala/org/apache/spark/sql/execution/OrderAwareCoalesceExec.scala
