This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a change to branch
dependabot/gradle/net.snowflake-snowflake-jdbc-3.14.5
in repository https://gitbox.apache.org/repos/asf/iceberg.git
discard ec032243d1 Build: Bump net.snowflake:snowflake-jdbc from 3.14.4 to
3.14.5
add f901ad2058 Build/Release: Upgrade to RAT 0.16.1 (#9579)
add fe42291907 Build: Bump org.assertj:assertj-core from 3.25.1 to 3.25.2
(#9576)
add 54756b6f5c Build: Bump org.apache.httpcomponents.client5:httpclient5
(#9572)
add 10ee51630f Spark 3.4: Support creating views via SQL (#9580)
add 3be1939afc Flink: Adds the ability to read from a branch on the Flink
Iceberg Source (#9547)
add e5dc5ec967 Flink: Implement enumerator metrics for pending splits,
pending records, and split discovery (#9524)
add adec50c01a Move nightly versioned docs to top-level docs directory
(#9578)
add d295a45f07 Spark 3.4: Fix writing of default values in CoW for rows
with NULL columns which are unmatched (#9556)
add 2af18b318e Spark: Rewrite identifier when using Subquery expressions
in View (#9587)
add 7f5e33dd03 Build: Bump
software.amazon.s3.accessgrants:aws-s3-accessgrants-java-plugin (#9575)
add 8b429a286d Build: Bump software.amazon.awssdk:bom from 2.23.2 to
2.23.12 (#9573)
add ac46000fca Docs: Update ASF site to deploy from this repository (#9520)
add a25f77d1cd Revert "Move nightly versioned docs to top-level docs
directory (#9578)" (#9589)
add f8866bf38e Spark 3.5: Fix flaky TestSparkExecutorCache (#9583)
add d5a9d34edc Spark: Add support for describing/showing views (#9513)
add 0536ff3989 Spark 3.4: Rewrite identifier when using Subquery
expressions in View (#9594)
add 8138671ab1 Spark 3.4: Add support for describing/showing views (#9595)
add 26d62c06bc Flink: Added error handling and default logic for Flink
version detection (#9452)
add 9de693f1e7 API, Spark: Fix aggregation pushdown on struct fields
(#9176)
add 974bde343f Open-API: Add table updates for statistics (#9564)
add 61532a042a Flink: Backport #9364 to 1.16 and 1.17 for Create
CatalogTestBase for migration to JUnit5 (#9601)
add 9c8e9ba67f Spark: Support altering view properties (#9582)
add a05cbcb4b8 Core: Add missing @Test to TestRESTCatalog (#9607)
add 66d4cf66e2 Spark: Throw exception on `ALTER VIEW <viewName> AS
<query>` (#9510)
add 187be85319 Spark 3.4: Support altering view properties (#9610)
add daaf5a1c87 Spark 3.4: Throw exception on `ALTER VIEW <viewName> AS
<query>` (#9612)
add e8c197d360 Spark 3.4: Rework DeleteFileIndexBenchmark (#9600)
add 9a0191e8de Spark 3.4: Fix rewriting manifests for evolved
unpartitioned V1 tables (#9599)
add 3547a99d05 Spark 3.4: Support file and partition delete granularity
(#9602)
add 6bbf70a52e Spark: Bypass Spark's ViewCatalog API when replacing a view
(#9596)
add ed288987cc Convert Hugo versioned docs to mkdocs format (#9591)
add f8a4cc225d Spark 3.4: Extend action for rewriting manifests to support
deletes (#9616)
add 2c247501c4 Flink: backport #9381 to 1.17 and 1.16 for Migrate
subclasses of FlinkCatalogTestBase to JUnit5 (#9598)
add 756fa6894a Spark 3.4: Bypass Spark's ViewCatalog API when replacing a
view (#9614)
add d1e24f4c95 Spark: Create ExtensionTestBase for migration to JUnit5
(#9613)
add 07c4345bbb Spark: Fix CREATE OR REPLACE VIEW when view doesn't exist
(#9621)
add 0f11340bfa Flink: change defaultFlinkVersion back to 1.18 (#9625)
add 770342cb53 Spark 3.4: Use ProcedureInput for RewriteDataFiles (#8583)
add aff5b39a7d Remove nightly and add .asf.yaml (#9622)
add 65a076deda Spark 3.4: Read deletes in parallel and cache them on
executors (#9603)
add 338c0b83d0 Parquet, Arrow: Rename BagePageReader to BasePageReader in
VectorizedPageIterator (#9630)
add 49986b7d65 Add REST spec for data access mechanisms (#9628)
add f4ba90d648 Kafka Connect: Sink connector with data writers and
converters (#9466)
add fb02bd2d71 Core: Fix performance issue when combining tasks by
partition (#9629)
add f8a4a74584 Docs: Enhance Java quickstart example (#9585)
add 9921937d82 Update blogs.md (#9552)
add d5e00f1022 Docs: Update Nessie URI to API v2 (#9648)
add 4a3e06bd96 Spark 3.4: Fix CREATE OR REPLACE VIEW when view doesn't
exist (#9646)
add a2c23a7622 Docs: Fix listing of catalog implementations (#9649)
add c516fef72d Label `site/` as documentation (#9652)
add a1e9e58c0c Core: Add catalog type for glue,jdbc,nessie (#9647)
add c34efa0687 Flink: backport #9547 to 1.17 and 1.16 for Adds the ability
to read from a branch on the Flink Iceberg Source (#9627)
add 67a8f01bff AWS: Add S3 Access Grants Documentation (#9590)
add c4cb0fb999 Docs: Move catalog under concepts folder and add code of
conduct page to act as root for ASF. (#9642)
add c745ac3b6a Spark 3.5: Support executor cache locality (#9563)
add b6c3f8f2ba Spark 3.4: Support executor cache locality (#9658)
add 24a1480f80 Docs: Add newline so that subsection is correctly rendered
(#9656)
add 4835549ac3 Spark: Migrate tests to JUnit5 (#9624)
add 396a8441c4 Docs: Add newline to fix lists (#9664)
add e61b0e51e1 Build: Bump software.amazon.awssdk:bom from 2.23.12 to
2.23.17 (#9633)
add 4751a37672 Build: Bump datamodel-code-generator from 0.25.2 to 0.25.3
(#9639)
add b89c395ca7 Build: Bump mkdocs-material from 9.5.3 to 9.5.7 (#9638)
add dd0ac5bce4 Build: Bump org.xerial:sqlite-jdbc from 3.44.0.0 to
3.45.1.0 (#9634)
add a130f8f1c8 Build: Bump mkdocs-material-extensions from 1.3 to 1.3.1
(#9160)
add af46487419 Build: Bump org.roaringbitmap:RoaringBitmap from 1.0.0 to
1.0.1 (#9317)
add defef48e21 Core: Only trim trailing slash when warehouse location is
not root path (#9619)
add 3348d88d96 Build: Bump io.delta:delta-spark_2.12 from 3.0.0 to 3.1.0
(#9631)
add 9f979a1ff1 Build: Bump io.delta:delta-standalone_2.12 from 0.6.0 to
3.1.0 (#9636)
add 5d3be12e0d Spark: Move the Writer to a visitor (#9440)
add 3c703ccce8 Azure: Bump Azurite container (#9668)
add 2a39af894f Spark: Handle concurrently dropped view during CREATE OR
REPLACE (#9623)
add 8a47f71895 Build: Bump com.azure:azure-sdk-bom from 1.2.18 to 1.2.20
(#9571)
add 97e9b3e775 Build: Bump org.testcontainers:testcontainers from 1.19.3
to 1.19.4 (#9577)
add b9285a5aeb Build: Bump net.snowflake:snowflake-jdbc from 3.14.4 to
3.14.5
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (ec032243d1)
\
N -- N -- N
refs/heads/dependabot/gradle/net.snowflake-snowflake-jdbc-3.14.5 (b9285a5aeb)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
No new revisions were added by this update.
Summary of changes:
.asf.yaml | 3 +
.github/labeler.yml | 1 +
.gitignore | 3 +-
.palantir/revapi.yml | 4 +
.../apache/iceberg/expressions/ValueAggregate.java | 6 +-
.../vectorized/parquet/VectorizedPageIterator.java | 22 +-
.../iceberg/azure/adlsv2/AzuriteContainer.java | 2 +-
.../main/java/org/apache/iceberg/CatalogUtil.java | 15 +
.../java/org/apache/iceberg/PartitionData.java | 54 +-
.../java/org/apache/iceberg/util/LocationUtil.java | 2 +-
.../org/apache/iceberg/util/TableScanUtil.java | 20 +-
.../java/org/apache/iceberg/MockFileScanTask.java | 11 +
.../org/apache/iceberg/jdbc/TestJdbcCatalog.java | 7 +-
.../org/apache/iceberg/rest/TestRESTCatalog.java | 1 +
.../org/apache/iceberg/util/TestLocationUtil.java | 26 +
dev/check-license | 2 +-
docs/_index.md | 54 --
docs/aws.md | 663 ---------------
docs/branching-and-tagging.md | 123 ---
docs/configuration.md | 200 -----
docs/dell.md | 136 ----
docs/delta-lake-migration.md | 123 ---
{site/docs/docs/nightly => docs}/docs/api.md | 0
.../docs/assets/images/audit-branch.png | Bin
.../docs/assets/images/historical-snapshot-tag.png | Bin
.../images/iceberg-in-place-metadata-migration.png | Bin
.../assets/images/iceberg-migrateaction-step1.png | Bin
.../assets/images/iceberg-migrateaction-step2.png | Bin
.../assets/images/iceberg-migrateaction-step3.png | Bin
.../assets/images/iceberg-snapshotaction-step1.png | Bin
.../assets/images/iceberg-snapshotaction-step2.png | Bin
.../assets/images/partition-spec-evolution.png | Bin
{site/docs/docs/nightly => docs}/docs/aws.md | 54 +-
{site/docs/docs/nightly => docs}/docs/branching.md | 2 +-
.../docs/nightly => docs}/docs/configuration.md | 8 +-
.../docs/nightly => docs}/docs/custom-catalog.md | 4 +-
{site/docs/docs/nightly => docs}/docs/dell.md | 0
.../nightly => docs}/docs/delta-lake-migration.md | 2 +
{site/docs/docs/nightly => docs}/docs/evolution.md | 0
.../docs/nightly => docs}/docs/flink-actions.md | 0
docs/{ => docs}/flink-configuration.md | 10 +-
.../docs/nightly => docs}/docs/flink-connector.md | 0
{site/docs/docs/nightly => docs}/docs/flink-ddl.md | 0
docs/{ => docs}/flink-queries.md | 34 +-
.../docs/nightly => docs}/docs/flink-writes.md | 11 +-
{site/docs/docs/nightly => docs}/docs/flink.md | 4 +-
.../docs/nightly => docs}/docs/hive-migration.md | 0
{site/docs/docs/nightly => docs}/docs/hive.md | 8 +-
{site/docs/docs/nightly => docs}/docs/index.md | 0
.../nightly => docs}/docs/java-api-quickstart.md | 62 +-
{site/docs/docs/nightly => docs}/docs/jdbc.md | 2 +-
.../docs/docs/nightly => docs}/docs/maintenance.md | 0
.../nightly => docs}/docs/metrics-reporting.md | 2 +
{site/docs/docs/nightly => docs}/docs/nessie.md | 14 +-
.../docs/nightly => docs}/docs/partitioning.md | 0
.../docs/docs/nightly => docs}/docs/performance.md | 0
.../docs/docs/nightly => docs}/docs/reliability.md | 0
{site/docs/docs/nightly => docs}/docs/schemas.md | 0
.../nightly => docs}/docs/spark-configuration.md | 6 +-
{site/docs/docs/nightly => docs}/docs/spark-ddl.md | 19 +-
.../nightly => docs}/docs/spark-getting-started.md | 0
.../docs/nightly => docs}/docs/spark-procedures.md | 24 +-
.../docs/nightly => docs}/docs/spark-queries.md | 28 +-
.../docs/spark-structured-streaming.md | 0
.../docs/nightly => docs}/docs/spark-writes.md | 10 +-
.../docs/nightly => docs}/docs/table-migration.md | 0
docs/evolution.md | 109 ---
docs/flink-actions.md | 43 -
docs/flink-connector.md | 149 ----
docs/flink-ddl.md | 229 ------
docs/flink-getting-started.md | 407 ----------
docs/flink-writes.md | 282 -------
docs/hive-migration.md | 61 --
docs/hive.md | 603 --------------
docs/java-api-quickstart.md | 325 --------
docs/java-api.md | 264 ------
docs/java-custom-catalog.md | 276 -------
docs/jdbc.md | 76 --
docs/maintenance.md | 165 ----
docs/metrics-reporting.md | 174 ----
{site/docs/docs/nightly => docs}/mkdocs.yml | 5 +-
docs/nessie.md | 168 ----
docs/partitioning.md | 103 ---
docs/performance.md | 63 --
docs/reliability.md | 76 --
docs/schemas.md | 52 --
docs/spark-configuration.md | 214 -----
docs/spark-ddl.md | 580 --------------
docs/spark-getting-started.md | 147 ----
docs/spark-procedures.md | 889 ---------------------
docs/spark-queries.md | 478 -----------
docs/spark-structured-streaming.md | 128 ---
docs/spark-writes.md | 477 -----------
docs/table-migration.md | 80 --
.../apache/iceberg/flink/source/ScanContext.java | 5 -
.../flink/source/StreamingMonitorFunction.java | 8 +-
.../enumerator/ContinuousSplitPlannerImpl.java | 6 +-
.../apache/iceberg/flink/util/FlinkPackage.java | 34 +-
.../org/apache/iceberg/flink/CatalogTestBase.java | 0
.../apache/iceberg/flink/FlinkCatalogTestBase.java | 155 ----
.../java/org/apache/iceberg/flink/TestBase.java | 2 +-
.../iceberg/flink/TestFlinkCatalogDatabase.java | 267 +++----
.../iceberg/flink/TestFlinkCatalogTable.java | 211 +++--
.../flink/TestFlinkCatalogTablePartitions.java | 48 +-
.../apache/iceberg/flink/TestFlinkHiveCatalog.java | 8 +-
.../apache/iceberg/flink/TestFlinkTableSink.java | 132 ++-
.../org/apache/iceberg/flink/TestFlinkUpsert.java | 56 +-
.../apache/iceberg/flink/TestIcebergConnector.java | 6 +-
.../flink/actions/TestRewriteDataFilesAction.java | 146 ++--
.../flink/source/TestFlinkMetaDataTable.java | 454 +++++------
.../flink/source/TestIcebergSourceContinuous.java | 87 ++
.../source/TestMetadataTableReadableMetrics.java | 48 +-
.../iceberg/flink/source/TestStreamScanSql.java | 192 ++++-
.../iceberg/flink/util/TestFlinkPackage.java | 23 +
.../apache/iceberg/flink/source/ScanContext.java | 5 -
.../flink/source/StreamingMonitorFunction.java | 8 +-
.../enumerator/ContinuousSplitPlannerImpl.java | 6 +-
.../apache/iceberg/flink/util/FlinkPackage.java | 34 +-
.../org/apache/iceberg/flink/CatalogTestBase.java | 0
.../apache/iceberg/flink/FlinkCatalogTestBase.java | 155 ----
.../java/org/apache/iceberg/flink/TestBase.java | 2 +-
.../iceberg/flink/TestFlinkCatalogDatabase.java | 267 +++----
.../iceberg/flink/TestFlinkCatalogTable.java | 492 ++++++------
.../flink/TestFlinkCatalogTablePartitions.java | 48 +-
.../apache/iceberg/flink/TestFlinkHiveCatalog.java | 8 +-
.../apache/iceberg/flink/TestFlinkTableSink.java | 132 ++-
.../org/apache/iceberg/flink/TestFlinkUpsert.java | 56 +-
.../apache/iceberg/flink/TestIcebergConnector.java | 6 +-
.../flink/actions/TestRewriteDataFilesAction.java | 146 ++--
.../flink/source/TestFlinkMetaDataTable.java | 454 +++++------
.../flink/source/TestIcebergSourceContinuous.java | 87 ++
.../source/TestMetadataTableReadableMetrics.java | 48 +-
.../iceberg/flink/source/TestStreamScanSql.java | 190 ++++-
.../iceberg/flink/util/TestFlinkPackage.java | 23 +
.../flink/sink/IcebergFilesCommitterMetrics.java | 25 +-
.../apache/iceberg/flink/source/ScanContext.java | 5 -
.../flink/source/StreamingMonitorFunction.java | 8 +-
.../source/assigner/DefaultSplitAssigner.java | 7 +
.../flink/source/assigner/SplitAssigner.java | 6 +
.../enumerator/AbstractIcebergEnumerator.java | 10 +-
.../enumerator/ContinuousIcebergEnumerator.java | 9 +
.../enumerator/ContinuousSplitPlannerImpl.java | 6 +-
.../iceberg/flink/util/ElapsedTimeGauge.java} | 30 +-
.../apache/iceberg/flink/util/FlinkPackage.java | 34 +-
.../apache/iceberg/flink/MiniClusterResource.java | 15 +
.../flink/source/TestIcebergSourceContinuous.java | 124 ++-
.../iceberg/flink/source/TestStreamScanSql.java | 140 +++-
.../iceberg/flink/util/TestFlinkPackage.java | 23 +
gradle/libs.versions.toml | 24 +-
kafka-connect/build.gradle | 27 +
.../apache/iceberg/connect/events/AvroUtil.java | 2 +-
.../iceberg/connect/events/CommitComplete.java | 2 +
.../iceberg/connect/events/CommitToTable.java | 3 +
.../iceberg/connect/events/DataComplete.java | 2 +
.../apache/iceberg/connect/events/DataWritten.java | 3 +
.../org/apache/iceberg/connect/events/Event.java | 4 +
.../apache/iceberg/connect/events/StartCommit.java | 2 +
.../iceberg/connect/events/TableReference.java | 4 +
.../connect/events/TopicPartitionOffset.java | 2 +
.../apache/iceberg/connect/IcebergSinkConfig.java | 468 +++++++++++
.../iceberg/connect/IcebergSinkConnector.java | 73 ++
.../apache/iceberg/connect/TableSinkConfig.java | 39 +-
.../apache/iceberg/connect/data/IcebergWriter.java | 118 +++
.../iceberg/connect/data/IcebergWriterFactory.java | 115 +++
.../apache/iceberg/connect/data/NoOpWriter.java | 25 +-
.../connect/data/PartitionedAppendWriter.java | 55 ++
.../apache/iceberg/connect/data/RecordWriter.java | 18 +-
.../apache/iceberg/connect/data/SchemaUpdate.java | 121 +++
.../apache/iceberg/connect/data/SchemaUtils.java | 351 ++++++++
.../org/apache/iceberg/connect/data/Utilities.java | 249 ++++++
.../apache/iceberg/connect/data/WriterResult.java | 60 ++
.../iceberg/connect/IcebergSinkConfigTest.java | 91 +++
.../iceberg/connect/IcebergSinkConnectorTest.java | 23 +-
.../iceberg/connect/data/BaseWriterTest.java | 93 +++
.../connect/data/IcebergWriterFactoryTest.java | 86 ++
.../connect/data/PartitionedAppendWriterTest.java | 66 ++
.../iceberg/connect/data/SchemaUtilsTest.java | 334 ++++++++
.../connect/data/UnpartitionedWriterTest.java | 63 ++
.../apache/iceberg/connect/data/UtilitiesTest.java | 183 +++++
.../kafka-connect/src/test/resources/core-site.xml | 25 +
.../apache/iceberg/nessie/TestNessieCatalog.java | 11 +-
open-api/requirements.txt | 2 +-
open-api/rest-catalog-open-api.py | 29 +
open-api/rest-catalog-open-api.yaml | 112 +++
settings.gradle | 4 +
site/README.md | 13 +-
site/dev/common.sh | 13 +-
site/dev/deploy.sh | 3 +-
site/dev/setup_env.sh | 3 +
site/docs/blogs.md | 5 +
site/docs/{ => concepts}/catalog.md | 9 +-
site/docs/docs/nightly/docs/flink-configuration.md | 159 ----
site/docs/docs/nightly/docs/flink-queries.md | 489 ------------
site/docs/how-to-release.md | 4 +-
site/docs/spec.md | 2 +
site/docs/view-spec.md | 2 +
site/mkdocs.yml | 3 +
site/nav.yml | 18 +-
site/requirements.txt | 4 +-
.../apache/iceberg/spark/extensions/TestMerge.java | 72 ++
spark/v3.4/build.gradle | 4 +
.../apache/iceberg/DeleteFileIndexBenchmark.java | 91 +--
.../extensions/IcebergSparkSessionExtensions.scala | 2 +
.../spark/sql/catalyst/analysis/CheckViews.scala | 23 +-
.../spark/sql/catalyst/analysis/ResolveViews.scala | 53 +-
.../catalyst/analysis/RewriteMergeIntoTable.scala | 2 +-
.../catalyst/analysis/RewriteViewCommands.scala | 101 ++-
.../spark/sql/catalyst/analysis/ViewUtil.scala | 51 ++
.../plans/logical/views/CreateIcebergView.scala | 0
.../plans/logical/views/ShowIcebergViews.scala | 23 +-
.../v2/AlterV2ViewSetPropertiesExec.scala | 49 ++
.../v2/AlterV2ViewUnsetPropertiesExec.scala | 54 ++
.../datasources/v2/CreateV2ViewExec.scala | 42 +-
.../datasources/v2/DescribeV2ViewExec.scala | 77 ++
.../v2/ExtendedDataSourceV2Strategy.scala | 41 +
.../datasources/v2/ShowCreateV2ViewExec.scala | 79 ++
.../datasources/v2/ShowV2ViewPropertiesExec.scala | 55 ++
.../execution/datasources/v2/ShowV2ViewsExec.scala | 66 ++
.../apache/iceberg/spark/extensions/TestMerge.java | 72 ++
.../spark/extensions/TestMergeOnReadDelete.java | 70 ++
.../spark/extensions/TestMergeOnReadMerge.java | 57 ++
.../spark/extensions/TestMergeOnReadUpdate.java | 55 ++
.../extensions/TestRewriteDataFilesProcedure.java | 2 +-
.../spark/extensions}/TestSparkExecutorCache.java | 197 +----
.../apache/iceberg/spark/extensions/TestViews.java | 722 ++++++++++++++++-
.../iceberg/spark/source/WritersBenchmark.java | 52 +-
.../java/org/apache/iceberg/spark/JavaUtils.java | 76 ++
.../org/apache/iceberg/spark/SparkCatalog.java | 123 ++-
.../org/apache/iceberg/spark/SparkConfParser.java | 40 +
.../org/apache/iceberg/spark/SparkContentFile.java | 0
.../org/apache/iceberg/spark/SparkDataFile.java | 191 +----
.../org/apache/iceberg/spark/SparkDeleteFile.java | 0
.../apache/iceberg/spark/SparkExecutorCache.java | 0
.../org/apache/iceberg/spark/SparkReadConf.java | 20 +
.../apache/iceberg/spark/SparkSQLProperties.java | 20 +
.../java/org/apache/iceberg/spark/SparkUtil.java | 30 +
.../org/apache/iceberg/spark/SparkWriteConf.java | 12 +
.../apache/iceberg/spark/SparkWriteOptions.java | 3 +
.../apache/iceberg/spark/SupportsReplaceView.java | 56 ++
.../spark/actions/RewriteManifestsSparkAction.java | 239 ++++--
.../procedures/RewriteDataFilesProcedure.java | 51 +-
.../apache/iceberg/spark/source/BaseReader.java | 29 +
.../spark/source/SerializableTableWithSize.java | 10 +
.../apache/iceberg/spark/source/SparkBatch.java | 45 +-
.../iceberg/spark/source/SparkInputPartition.java | 13 +-
.../spark/source/SparkMicroBatchStream.java | 32 +-
.../iceberg/spark/source/SparkPlanningUtil.java | 93 +++
.../spark/source/SparkPositionDeletesRewrite.java | 16 +-
.../spark/source/SparkPositionDeltaWrite.java | 14 +-
.../org/apache/iceberg/spark/source/SparkView.java | 9 +-
.../java/org/apache/iceberg/spark/Employee.java | 0
.../iceberg/spark/TestSparkExecutorCache.java | 125 +--
.../apache/iceberg/spark/TestSparkWriteConf.java | 81 ++
.../spark/actions/TestRewriteManifestsAction.java | 375 +++++++++
.../TestRewritePositionDeleteFilesAction.java | 38 +
.../iceberg/spark/source/TestSparkDataFile.java | 194 ++++-
.../spark/source/TestSparkPlanningUtil.java | 213 +++++
spark/v3.5/build.gradle | 4 +
.../iceberg/spark/TaskGroupPlanningBenchmark.java | 143 ++--
.../spark/sql/catalyst/analysis/CheckViews.scala | 5 +
.../spark/sql/catalyst/analysis/ResolveViews.scala | 33 +-
.../catalyst/analysis/RewriteViewCommands.scala | 46 +-
.../spark/sql/catalyst/analysis/ViewUtil.scala | 51 ++
.../plans/logical/views/ShowIcebergViews.scala | 23 +-
.../v2/AlterV2ViewSetPropertiesExec.scala | 49 ++
.../v2/AlterV2ViewUnsetPropertiesExec.scala | 54 ++
.../datasources/v2/CreateV2ViewExec.scala | 75 +-
.../datasources/v2/DescribeV2ViewExec.scala | 77 ++
.../v2/ExtendedDataSourceV2Strategy.scala | 25 +
.../datasources/v2/ShowCreateV2ViewExec.scala | 79 ++
.../datasources/v2/ShowV2ViewPropertiesExec.scala | 55 ++
.../execution/datasources/v2/ShowV2ViewsExec.scala | 66 ++
.../spark/extensions/ExtensionsTestBase.java | 65 ++
.../spark/extensions/TestAddFilesProcedure.java | 153 ++--
.../extensions/TestAlterTablePartitionFields.java | 224 +++---
.../spark/extensions/TestAlterTableSchema.java | 103 +--
.../iceberg/spark/extensions/TestBranchDDL.java | 173 ++--
.../apache/iceberg/spark/extensions/TestMerge.java | 72 ++
.../spark/extensions/TestMergeOnReadDelete.java | 25 +
.../spark/extensions/TestReplaceBranch.java | 92 +--
.../TestRequiredDistributionAndOrdering.java | 39 +-
.../extensions/TestRewriteDataFilesProcedure.java | 2 +-
.../TestSetWriteDistributionAndOrdering.java | 126 ++-
.../iceberg/spark/extensions/TestTagDDL.java | 152 ++--
.../apache/iceberg/spark/extensions/TestViews.java | 461 ++++++++++-
.../org/apache/iceberg/spark/SparkCatalog.java | 106 ++-
.../org/apache/iceberg/spark/SparkReadConf.java | 20 +
.../apache/iceberg/spark/SparkSQLProperties.java | 4 +
.../java/org/apache/iceberg/spark/SparkUtil.java | 30 +
.../apache/iceberg/spark/SupportsReplaceView.java | 56 ++
.../iceberg/spark/data/SparkParquetWriters.java | 169 ++--
.../procedures/RewriteDataFilesProcedure.java | 51 +-
.../apache/iceberg/spark/source/SparkBatch.java | 45 +-
.../iceberg/spark/source/SparkInputPartition.java | 13 +-
.../spark/source/SparkMicroBatchStream.java | 32 +-
.../iceberg/spark/source/SparkPlanningUtil.java | 93 +++
.../org/apache/iceberg/spark/source/SparkView.java | 4 +-
.../iceberg/spark/TestSparkExecutorCache.java | 7 +
.../spark/source/TestSparkPlanningUtil.java | 213 +++++
.../iceberg/spark/sql/TestAggregatePushDown.java | 122 +++
300 files changed, 11632 insertions(+), 12583 deletions(-)
delete mode 100644 docs/_index.md
delete mode 100644 docs/aws.md
delete mode 100644 docs/branching-and-tagging.md
delete mode 100644 docs/configuration.md
delete mode 100644 docs/dell.md
delete mode 100644 docs/delta-lake-migration.md
rename {site/docs/docs/nightly => docs}/docs/api.md (100%)
rename {site/docs/docs/nightly => docs}/docs/assets/images/audit-branch.png
(100%)
rename {site/docs/docs/nightly =>
docs}/docs/assets/images/historical-snapshot-tag.png (100%)
rename {site/docs/docs/nightly =>
docs}/docs/assets/images/iceberg-in-place-metadata-migration.png (100%)
rename {site/docs/docs/nightly =>
docs}/docs/assets/images/iceberg-migrateaction-step1.png (100%)
rename {site/docs/docs/nightly =>
docs}/docs/assets/images/iceberg-migrateaction-step2.png (100%)
rename {site/docs/docs/nightly =>
docs}/docs/assets/images/iceberg-migrateaction-step3.png (100%)
rename {site/docs/docs/nightly =>
docs}/docs/assets/images/iceberg-snapshotaction-step1.png (100%)
rename {site/docs/docs/nightly =>
docs}/docs/assets/images/iceberg-snapshotaction-step2.png (100%)
rename {site/docs/docs/nightly =>
docs}/docs/assets/images/partition-spec-evolution.png (100%)
rename {site/docs/docs/nightly => docs}/docs/aws.md (94%)
rename {site/docs/docs/nightly => docs}/docs/branching.md (98%)
rename {site/docs/docs/nightly => docs}/docs/configuration.md (98%)
rename {site/docs/docs/nightly => docs}/docs/custom-catalog.md (98%)
rename {site/docs/docs/nightly => docs}/docs/dell.md (100%)
rename {site/docs/docs/nightly => docs}/docs/delta-lake-migration.md (99%)
rename {site/docs/docs/nightly => docs}/docs/evolution.md (100%)
rename {site/docs/docs/nightly => docs}/docs/flink-actions.md (100%)
rename docs/{ => docs}/flink-configuration.md (98%)
rename {site/docs/docs/nightly => docs}/docs/flink-connector.md (100%)
rename {site/docs/docs/nightly => docs}/docs/flink-ddl.md (100%)
rename docs/{ => docs}/flink-queries.md (95%)
rename {site/docs/docs/nightly => docs}/docs/flink-writes.md (95%)
rename {site/docs/docs/nightly => docs}/docs/flink.md (98%)
rename {site/docs/docs/nightly => docs}/docs/hive-migration.md (100%)
rename {site/docs/docs/nightly => docs}/docs/hive.md (99%)
rename {site/docs/docs/nightly => docs}/docs/index.md (100%)
rename {site/docs/docs/nightly => docs}/docs/java-api-quickstart.md (80%)
rename {site/docs/docs/nightly => docs}/docs/jdbc.md (97%)
rename {site/docs/docs/nightly => docs}/docs/maintenance.md (100%)
rename {site/docs/docs/nightly => docs}/docs/metrics-reporting.md (99%)
rename {site/docs/docs/nightly => docs}/docs/nessie.md (95%)
rename {site/docs/docs/nightly => docs}/docs/partitioning.md (100%)
rename {site/docs/docs/nightly => docs}/docs/performance.md (100%)
rename {site/docs/docs/nightly => docs}/docs/reliability.md (100%)
rename {site/docs/docs/nightly => docs}/docs/schemas.md (100%)
rename {site/docs/docs/nightly => docs}/docs/spark-configuration.md (94%)
rename {site/docs/docs/nightly => docs}/docs/spark-ddl.md (97%)
rename {site/docs/docs/nightly => docs}/docs/spark-getting-started.md (100%)
rename {site/docs/docs/nightly => docs}/docs/spark-procedures.md (98%)
rename {site/docs/docs/nightly => docs}/docs/spark-queries.md (97%)
rename {site/docs/docs/nightly => docs}/docs/spark-structured-streaming.md
(100%)
rename {site/docs/docs/nightly => docs}/docs/spark-writes.md (98%)
rename {site/docs/docs/nightly => docs}/docs/table-migration.md (100%)
delete mode 100644 docs/evolution.md
delete mode 100644 docs/flink-actions.md
delete mode 100644 docs/flink-connector.md
delete mode 100644 docs/flink-ddl.md
delete mode 100644 docs/flink-getting-started.md
delete mode 100644 docs/flink-writes.md
delete mode 100644 docs/hive-migration.md
delete mode 100644 docs/hive.md
delete mode 100644 docs/java-api-quickstart.md
delete mode 100644 docs/java-api.md
delete mode 100644 docs/java-custom-catalog.md
delete mode 100644 docs/jdbc.md
delete mode 100644 docs/maintenance.md
delete mode 100644 docs/metrics-reporting.md
rename {site/docs/docs/nightly => docs}/mkdocs.yml (95%)
delete mode 100644 docs/nessie.md
delete mode 100644 docs/partitioning.md
delete mode 100644 docs/performance.md
delete mode 100644 docs/reliability.md
delete mode 100644 docs/schemas.md
delete mode 100644 docs/spark-configuration.md
delete mode 100644 docs/spark-ddl.md
delete mode 100644 docs/spark-getting-started.md
delete mode 100644 docs/spark-procedures.md
delete mode 100644 docs/spark-queries.md
delete mode 100644 docs/spark-structured-streaming.md
delete mode 100644 docs/spark-writes.md
delete mode 100644 docs/table-migration.md
copy flink/{v1.18 =>
v1.16}/flink/src/test/java/org/apache/iceberg/flink/CatalogTestBase.java (100%)
delete mode 100644
flink/v1.16/flink/src/test/java/org/apache/iceberg/flink/FlinkCatalogTestBase.java
copy flink/{v1.18 =>
v1.17}/flink/src/test/java/org/apache/iceberg/flink/CatalogTestBase.java (100%)
delete mode 100644
flink/v1.17/flink/src/test/java/org/apache/iceberg/flink/FlinkCatalogTestBase.java
copy
flink/{v1.16/flink/src/main/java/org/apache/iceberg/flink/util/FlinkPackage.java
=>
v1.18/flink/src/main/java/org/apache/iceberg/flink/util/ElapsedTimeGauge.java}
(52%)
create mode 100644
kafka-connect/kafka-connect/src/main/java/org/apache/iceberg/connect/IcebergSinkConfig.java
create mode 100644
kafka-connect/kafka-connect/src/main/java/org/apache/iceberg/connect/IcebergSinkConnector.java
copy
flink/v1.16/flink/src/main/java/org/apache/iceberg/flink/util/FlinkPackage.java
=>
kafka-connect/kafka-connect/src/main/java/org/apache/iceberg/connect/TableSinkConfig.java
(50%)
create mode 100644
kafka-connect/kafka-connect/src/main/java/org/apache/iceberg/connect/data/IcebergWriter.java
create mode 100644
kafka-connect/kafka-connect/src/main/java/org/apache/iceberg/connect/data/IcebergWriterFactory.java
copy
flink/v1.16/flink/src/test/java/org/apache/iceberg/flink/util/TestFlinkPackage.java
=>
kafka-connect/kafka-connect/src/main/java/org/apache/iceberg/connect/data/NoOpWriter.java
(68%)
create mode 100644
kafka-connect/kafka-connect/src/main/java/org/apache/iceberg/connect/data/PartitionedAppendWriter.java
copy
flink/v1.18/flink/src/test/java/org/apache/iceberg/flink/util/TestFlinkPackage.java
=>
kafka-connect/kafka-connect/src/main/java/org/apache/iceberg/connect/data/RecordWriter.java
(72%)
create mode 100644
kafka-connect/kafka-connect/src/main/java/org/apache/iceberg/connect/data/SchemaUpdate.java
create mode 100644
kafka-connect/kafka-connect/src/main/java/org/apache/iceberg/connect/data/SchemaUtils.java
create mode 100644
kafka-connect/kafka-connect/src/main/java/org/apache/iceberg/connect/data/Utilities.java
create mode 100644
kafka-connect/kafka-connect/src/main/java/org/apache/iceberg/connect/data/WriterResult.java
create mode 100644
kafka-connect/kafka-connect/src/test/java/org/apache/iceberg/connect/IcebergSinkConfigTest.java
copy
flink/v1.17/flink/src/test/java/org/apache/iceberg/flink/util/TestFlinkPackage.java
=>
kafka-connect/kafka-connect/src/test/java/org/apache/iceberg/connect/IcebergSinkConnectorTest.java
(51%)
create mode 100644
kafka-connect/kafka-connect/src/test/java/org/apache/iceberg/connect/data/BaseWriterTest.java
create mode 100644
kafka-connect/kafka-connect/src/test/java/org/apache/iceberg/connect/data/IcebergWriterFactoryTest.java
create mode 100644
kafka-connect/kafka-connect/src/test/java/org/apache/iceberg/connect/data/PartitionedAppendWriterTest.java
create mode 100644
kafka-connect/kafka-connect/src/test/java/org/apache/iceberg/connect/data/SchemaUtilsTest.java
create mode 100644
kafka-connect/kafka-connect/src/test/java/org/apache/iceberg/connect/data/UnpartitionedWriterTest.java
create mode 100644
kafka-connect/kafka-connect/src/test/java/org/apache/iceberg/connect/data/UtilitiesTest.java
create mode 100644 kafka-connect/kafka-connect/src/test/resources/core-site.xml
rename site/docs/{ => concepts}/catalog.md (92%)
delete mode 100644 site/docs/docs/nightly/docs/flink-configuration.md
delete mode 100644 site/docs/docs/nightly/docs/flink-queries.md
copy spark/{v3.5 =>
v3.4}/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckViews.scala
(69%)
create mode 100644
spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/ViewUtil.scala
copy spark/{v3.5 =>
v3.4}/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/views/CreateIcebergView.scala
(100%)
copy
flink/v1.16/flink/src/main/java/org/apache/iceberg/flink/util/FlinkPackage.java
=>
spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/views/ShowIcebergViews.scala
(55%)
create mode 100644
spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/AlterV2ViewSetPropertiesExec.scala
create mode 100644
spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/AlterV2ViewUnsetPropertiesExec.scala
copy spark/{v3.5 =>
v3.4}/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/CreateV2ViewExec.scala
(77%)
create mode 100644
spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DescribeV2ViewExec.scala
create mode 100644
spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowCreateV2ViewExec.scala
create mode 100644
spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowV2ViewPropertiesExec.scala
create mode 100644
spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowV2ViewsExec.scala
copy spark/{v3.5/spark/src/test/java/org/apache/iceberg/spark =>
v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions}/TestSparkExecutorCache.java
(69%)
create mode 100644
spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/JavaUtils.java
copy spark/{v3.5 =>
v3.4}/spark/src/main/java/org/apache/iceberg/spark/SparkContentFile.java (100%)
copy spark/{v3.5 =>
v3.4}/spark/src/main/java/org/apache/iceberg/spark/SparkDeleteFile.java (100%)
copy spark/{v3.5 =>
v3.4}/spark/src/main/java/org/apache/iceberg/spark/SparkExecutorCache.java
(100%)
create mode 100644
spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/SupportsReplaceView.java
create mode 100644
spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/SparkPlanningUtil.java
copy spark/{v3.5 =>
v3.4}/spark/src/test/java/org/apache/iceberg/spark/Employee.java (100%)
copy spark/{v3.5 =>
v3.4}/spark/src/test/java/org/apache/iceberg/spark/TestSparkExecutorCache.java
(79%)
create mode 100644
spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkPlanningUtil.java
create mode 100644
spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/ViewUtil.scala
copy
flink/v1.16/flink/src/main/java/org/apache/iceberg/flink/util/FlinkPackage.java
=>
spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/views/ShowIcebergViews.scala
(55%)
create mode 100644
spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/AlterV2ViewSetPropertiesExec.scala
create mode 100644
spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/AlterV2ViewUnsetPropertiesExec.scala
create mode 100644
spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DescribeV2ViewExec.scala
create mode 100644
spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowCreateV2ViewExec.scala
create mode 100644
spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowV2ViewPropertiesExec.scala
create mode 100644
spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowV2ViewsExec.scala
create mode 100644
spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/ExtensionsTestBase.java
create mode 100644
spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/SupportsReplaceView.java
create mode 100644
spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/source/SparkPlanningUtil.java
create mode 100644
spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkPlanningUtil.java
