This is an automated email from the ASF dual-hosted git repository.
stevenzwu pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg.git
The following commit(s) were added to refs/heads/main by this push:
new 36d79e74bb Spark: Trim TestStructuredStreamingRead3 parameter rows
from 8 to 2 (#16559)
36d79e74bb is described below
commit 36d79e74bb19e56b7b0ca3fcc25d9e557b118745
Author: Steven Zhen Wu <[email protected]>
AuthorDate: Wed May 27 11:35:46 2026 -0700
Spark: Trim TestStructuredStreamingRead3 parameter rows from 8 to 2 (#16559)
Reduces the parameter set in TestStructuredStreamingRead3 from 8 rows
(4 catalogs × async{T,F}) to 2 rows: testhive (async=true) and
testrest (async=false). Streaming read semantics aren't catalog-specific
in any meaningful way, and async-vs-sync planning is the only axis
TestStructuredStreamingRead3 actually exercises beyond basic Spark
streaming behavior. Both async values and the strategic catalog
backends (Hive metastore + REST) remain covered with one row each.
Drops testhadoop (HadoopCatalog isn't recommended for production)
and the spark_catalog SessionCatalog rows (the SessionCatalog wrapper
differences live in DDL/table-resolution paths, not streaming reads).
Each invocation runs 33 streaming tests, so 8→2 rows cuts the class
from 264 to 66 invocations - roughly 75% reduction in CPU time for
this test (the highest-CPU class in the Spark core CI job at 20.3% of
total test CPU).
Co-authored-by: Claude Opus 4.7 (1M context) <[email protected]>
---
.../spark/source/TestStructuredStreamingRead3.java | 39 ----------------------
.../spark/source/TestStructuredStreamingRead3.java | 39 ----------------------
.../spark/source/TestStructuredStreamingRead3.java | 39 ----------------------
3 files changed, 117 deletions(-)
diff --git
a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreamingRead3.java
b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreamingRead3.java
index d97e6ec00d..4efb883b5d 100644
---
a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreamingRead3.java
+++
b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreamingRead3.java
@@ -91,24 +91,6 @@ public final class TestStructuredStreamingRead3 extends
CatalogTestBase {
SparkCatalogConfig.HIVE.catalogName(),
SparkCatalogConfig.HIVE.implementation(),
SparkCatalogConfig.HIVE.properties(),
- false
- },
- {
- SparkCatalogConfig.HIVE.catalogName(),
- SparkCatalogConfig.HIVE.implementation(),
- SparkCatalogConfig.HIVE.properties(),
- true
- },
- {
- SparkCatalogConfig.HADOOP.catalogName(),
- SparkCatalogConfig.HADOOP.implementation(),
- SparkCatalogConfig.HADOOP.properties(),
- false
- },
- {
- SparkCatalogConfig.HADOOP.catalogName(),
- SparkCatalogConfig.HADOOP.implementation(),
- SparkCatalogConfig.HADOOP.properties(),
true
},
{
@@ -119,27 +101,6 @@ public final class TestStructuredStreamingRead3 extends
CatalogTestBase {
.put(CatalogProperties.URI,
restCatalog.properties().get(CatalogProperties.URI))
.build(),
false
- },
- {
- SparkCatalogConfig.REST.catalogName(),
- SparkCatalogConfig.REST.implementation(),
- ImmutableMap.builder()
- .putAll(SparkCatalogConfig.REST.properties())
- .put(CatalogProperties.URI,
restCatalog.properties().get(CatalogProperties.URI))
- .build(),
- true
- },
- {
- SparkCatalogConfig.SPARK_SESSION.catalogName(),
- SparkCatalogConfig.SPARK_SESSION.implementation(),
- SparkCatalogConfig.SPARK_SESSION.properties(),
- false
- },
- {
- SparkCatalogConfig.SPARK_SESSION.catalogName(),
- SparkCatalogConfig.SPARK_SESSION.implementation(),
- SparkCatalogConfig.SPARK_SESSION.properties(),
- true
}
};
}
diff --git
a/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreamingRead3.java
b/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreamingRead3.java
index 5f9b460f37..06189b3042 100644
---
a/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreamingRead3.java
+++
b/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreamingRead3.java
@@ -91,24 +91,6 @@ public final class TestStructuredStreamingRead3 extends
CatalogTestBase {
SparkCatalogConfig.HIVE.catalogName(),
SparkCatalogConfig.HIVE.implementation(),
SparkCatalogConfig.HIVE.properties(),
- false
- },
- {
- SparkCatalogConfig.HIVE.catalogName(),
- SparkCatalogConfig.HIVE.implementation(),
- SparkCatalogConfig.HIVE.properties(),
- true
- },
- {
- SparkCatalogConfig.HADOOP.catalogName(),
- SparkCatalogConfig.HADOOP.implementation(),
- SparkCatalogConfig.HADOOP.properties(),
- false
- },
- {
- SparkCatalogConfig.HADOOP.catalogName(),
- SparkCatalogConfig.HADOOP.implementation(),
- SparkCatalogConfig.HADOOP.properties(),
true
},
{
@@ -119,27 +101,6 @@ public final class TestStructuredStreamingRead3 extends
CatalogTestBase {
.put(CatalogProperties.URI,
restCatalog.properties().get(CatalogProperties.URI))
.build(),
false
- },
- {
- SparkCatalogConfig.REST.catalogName(),
- SparkCatalogConfig.REST.implementation(),
- ImmutableMap.builder()
- .putAll(SparkCatalogConfig.REST.properties())
- .put(CatalogProperties.URI,
restCatalog.properties().get(CatalogProperties.URI))
- .build(),
- true
- },
- {
- SparkCatalogConfig.SPARK_SESSION.catalogName(),
- SparkCatalogConfig.SPARK_SESSION.implementation(),
- SparkCatalogConfig.SPARK_SESSION.properties(),
- false
- },
- {
- SparkCatalogConfig.SPARK_SESSION.catalogName(),
- SparkCatalogConfig.SPARK_SESSION.implementation(),
- SparkCatalogConfig.SPARK_SESSION.properties(),
- true
}
};
}
diff --git
a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreamingRead3.java
b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreamingRead3.java
index 3957872be7..89947f73ea 100644
---
a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreamingRead3.java
+++
b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreamingRead3.java
@@ -92,24 +92,6 @@ public final class TestStructuredStreamingRead3 extends
CatalogTestBase {
SparkCatalogConfig.HIVE.catalogName(),
SparkCatalogConfig.HIVE.implementation(),
SparkCatalogConfig.HIVE.properties(),
- false
- },
- {
- SparkCatalogConfig.HIVE.catalogName(),
- SparkCatalogConfig.HIVE.implementation(),
- SparkCatalogConfig.HIVE.properties(),
- true
- },
- {
- SparkCatalogConfig.HADOOP.catalogName(),
- SparkCatalogConfig.HADOOP.implementation(),
- SparkCatalogConfig.HADOOP.properties(),
- false
- },
- {
- SparkCatalogConfig.HADOOP.catalogName(),
- SparkCatalogConfig.HADOOP.implementation(),
- SparkCatalogConfig.HADOOP.properties(),
true
},
{
@@ -120,27 +102,6 @@ public final class TestStructuredStreamingRead3 extends
CatalogTestBase {
.put(CatalogProperties.URI,
restCatalog.properties().get(CatalogProperties.URI))
.build(),
false
- },
- {
- SparkCatalogConfig.REST.catalogName(),
- SparkCatalogConfig.REST.implementation(),
- ImmutableMap.builder()
- .putAll(SparkCatalogConfig.REST.properties())
- .put(CatalogProperties.URI,
restCatalog.properties().get(CatalogProperties.URI))
- .build(),
- true
- },
- {
- SparkCatalogConfig.SPARK_SESSION.catalogName(),
- SparkCatalogConfig.SPARK_SESSION.implementation(),
- SparkCatalogConfig.SPARK_SESSION.properties(),
- false
- },
- {
- SparkCatalogConfig.SPARK_SESSION.catalogName(),
- SparkCatalogConfig.SPARK_SESSION.implementation(),
- SparkCatalogConfig.SPARK_SESSION.properties(),
- true
}
};
}
