This is an automated email from the ASF dual-hosted git repository.
stevenwu pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg.git
The following commit(s) were added to refs/heads/main by this push:
new a22d08c23e Spark: use lookback address for spark session in tests to
work with more restrictive firewall env on dev machines (#13993)
a22d08c23e is described below
commit a22d08c23e6471ff91048cfea4da09221d59167f
Author: Steven Zhen Wu <[email protected]>
AuthorDate: Thu Sep 4 10:38:29 2025 -0700
Spark: use lookback address for spark session in tests to work with more
restrictive firewall env on dev machines (#13993)
---
.../org/apache/iceberg/spark/extensions/ExtensionsTestBase.java | 2 ++
.../java/org/apache/iceberg/spark/source/TestFilteredScan.java | 7 ++++++-
.../org/apache/iceberg/spark/source/TestForwardCompatibility.java | 7 ++++++-
.../java/org/apache/iceberg/spark/source/TestIcebergSpark.java | 7 ++++++-
.../java/org/apache/iceberg/spark/source/TestPartitionPruning.java | 7 ++++++-
.../java/org/apache/iceberg/spark/source/TestPartitionValues.java | 7 ++++++-
.../org/apache/iceberg/spark/source/TestSnapshotSelection.java | 7 ++++++-
.../java/org/apache/iceberg/spark/source/TestSparkDataFile.java | 7 ++++++-
.../java/org/apache/iceberg/spark/source/TestSparkDataWrite.java | 7 ++++++-
.../org/apache/iceberg/spark/source/TestSparkReadProjection.java | 7 ++++++-
.../org/apache/iceberg/spark/source/TestStructuredStreaming.java | 2 ++
.../org/apache/iceberg/spark/source/TestWriteMetricsConfig.java | 7 ++++++-
12 files changed, 64 insertions(+), 10 deletions(-)
diff --git
a/spark/v4.0/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/ExtensionsTestBase.java
b/spark/v4.0/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/ExtensionsTestBase.java
index 2dd8469300..796c47b545 100644
---
a/spark/v4.0/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/ExtensionsTestBase.java
+++
b/spark/v4.0/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/ExtensionsTestBase.java
@@ -20,6 +20,7 @@ package org.apache.iceberg.spark.extensions;
import static org.apache.hadoop.hive.conf.HiveConf.ConfVars.METASTOREURIS;
+import java.net.InetAddress;
import java.util.Random;
import java.util.concurrent.ThreadLocalRandom;
import org.apache.iceberg.CatalogUtil;
@@ -48,6 +49,7 @@ public abstract class ExtensionsTestBase extends
CatalogTestBase {
TestBase.spark =
SparkSession.builder()
.master("local[2]")
+ .config("spark.driver.host",
InetAddress.getLoopbackAddress().getHostAddress())
.config("spark.testing", "true")
.config(SQLConf.PARTITION_OVERWRITE_MODE().key(), "dynamic")
.config("spark.sql.extensions",
IcebergSparkSessionExtensions.class.getName())
diff --git
a/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/source/TestFilteredScan.java
b/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/source/TestFilteredScan.java
index 8c85f360b5..fd7d52178f 100644
---
a/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/source/TestFilteredScan.java
+++
b/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/source/TestFilteredScan.java
@@ -25,6 +25,7 @@ import static org.assertj.core.api.Assertions.assertThat;
import java.io.File;
import java.io.IOException;
+import java.net.InetAddress;
import java.nio.file.Path;
import java.time.OffsetDateTime;
import java.util.Arrays;
@@ -109,7 +110,11 @@ public class TestFilteredScan {
@BeforeAll
public static void startSpark() {
- TestFilteredScan.spark =
SparkSession.builder().master("local[2]").getOrCreate();
+ TestFilteredScan.spark =
+ SparkSession.builder()
+ .master("local[2]")
+ .config("spark.driver.host",
InetAddress.getLoopbackAddress().getHostAddress())
+ .getOrCreate();
}
@AfterAll
diff --git
a/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java
b/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java
index 665c2d39cc..c4e0d26c1c 100644
---
a/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java
+++
b/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java
@@ -25,6 +25,7 @@ import static
org.assertj.core.api.Assertions.assertThatThrownBy;
import java.io.File;
import java.io.IOException;
+import java.net.InetAddress;
import java.nio.file.Path;
import java.util.List;
import java.util.UUID;
@@ -93,7 +94,11 @@ public class TestForwardCompatibility {
@BeforeAll
public static void startSpark() {
- TestForwardCompatibility.spark =
SparkSession.builder().master("local[2]").getOrCreate();
+ TestForwardCompatibility.spark =
+ SparkSession.builder()
+ .master("local[2]")
+ .config("spark.driver.host",
InetAddress.getLoopbackAddress().getHostAddress())
+ .getOrCreate();
}
@AfterAll
diff --git
a/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/source/TestIcebergSpark.java
b/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/source/TestIcebergSpark.java
index 5843fe9925..f4f57157e4 100644
---
a/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/source/TestIcebergSpark.java
+++
b/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/source/TestIcebergSpark.java
@@ -22,6 +22,7 @@ import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
import java.math.BigDecimal;
+import java.net.InetAddress;
import java.nio.ByteBuffer;
import java.sql.Date;
import java.sql.Timestamp;
@@ -46,7 +47,11 @@ public class TestIcebergSpark {
@BeforeAll
public static void startSpark() {
- TestIcebergSpark.spark =
SparkSession.builder().master("local[2]").getOrCreate();
+ TestIcebergSpark.spark =
+ SparkSession.builder()
+ .master("local[2]")
+ .config("spark.driver.host",
InetAddress.getLoopbackAddress().getHostAddress())
+ .getOrCreate();
}
@AfterAll
diff --git
a/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionPruning.java
b/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionPruning.java
index 5b1dc92e13..b31880e8e9 100644
---
a/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionPruning.java
+++
b/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionPruning.java
@@ -24,6 +24,7 @@ import static org.assertj.core.api.Assertions.assertThat;
import java.io.File;
import java.io.IOException;
+import java.net.InetAddress;
import java.net.URI;
import java.nio.file.Files;
import java.sql.Timestamp;
@@ -114,7 +115,11 @@ public class TestPartitionPruning {
@BeforeAll
public static void startSpark() {
- TestPartitionPruning.spark =
SparkSession.builder().master("local[2]").getOrCreate();
+ TestPartitionPruning.spark =
+ SparkSession.builder()
+ .master("local[2]")
+ .config("spark.driver.host",
InetAddress.getLoopbackAddress().getHostAddress())
+ .getOrCreate();
TestPartitionPruning.sparkContext =
JavaSparkContext.fromSparkContext(spark.sparkContext());
String optionKey = String.format("fs.%s.impl",
CountOpenLocalFileSystem.scheme);
diff --git
a/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionValues.java
b/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionValues.java
index 0153996eb5..0b6ab2052b 100644
---
a/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionValues.java
+++
b/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionValues.java
@@ -24,6 +24,7 @@ import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assumptions.assumeThat;
import java.io.File;
+import java.net.InetAddress;
import java.nio.file.Path;
import java.util.List;
import java.util.stream.Collectors;
@@ -107,7 +108,11 @@ public class TestPartitionValues {
@BeforeAll
public static void startSpark() {
- TestPartitionValues.spark =
SparkSession.builder().master("local[2]").getOrCreate();
+ TestPartitionValues.spark =
+ SparkSession.builder()
+ .master("local[2]")
+ .config("spark.driver.host",
InetAddress.getLoopbackAddress().getHostAddress())
+ .getOrCreate();
}
@AfterAll
diff --git
a/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/source/TestSnapshotSelection.java
b/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/source/TestSnapshotSelection.java
index 3d74b91359..11865db7fc 100644
---
a/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/source/TestSnapshotSelection.java
+++
b/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/source/TestSnapshotSelection.java
@@ -24,6 +24,7 @@ import static
org.apache.iceberg.types.Types.NestedField.optional;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
+import java.net.InetAddress;
import java.nio.file.Path;
import java.util.List;
import java.util.Map;
@@ -86,7 +87,11 @@ public class TestSnapshotSelection {
@BeforeAll
public static void startSpark() {
- TestSnapshotSelection.spark =
SparkSession.builder().master("local[2]").getOrCreate();
+ TestSnapshotSelection.spark =
+ SparkSession.builder()
+ .master("local[2]")
+ .config("spark.driver.host",
InetAddress.getLoopbackAddress().getHostAddress())
+ .getOrCreate();
}
@AfterAll
diff --git
a/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataFile.java
b/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataFile.java
index 8324ed1026..35a837b6b8 100644
---
a/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataFile.java
+++
b/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataFile.java
@@ -24,6 +24,7 @@ import static org.assertj.core.api.Assertions.assertThat;
import java.io.File;
import java.io.IOException;
+import java.net.InetAddress;
import java.nio.ByteBuffer;
import java.util.Arrays;
import java.util.Collections;
@@ -121,7 +122,11 @@ public class TestSparkDataFile {
@BeforeAll
public static void startSpark() {
- TestSparkDataFile.spark =
SparkSession.builder().master("local[2]").getOrCreate();
+ TestSparkDataFile.spark =
+ SparkSession.builder()
+ .master("local[2]")
+ .config("spark.driver.host",
InetAddress.getLoopbackAddress().getHostAddress())
+ .getOrCreate();
TestSparkDataFile.sparkContext =
JavaSparkContext.fromSparkContext(spark.sparkContext());
}
diff --git
a/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite.java
b/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite.java
index 141c887115..4ccbf86f12 100644
---
a/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite.java
+++
b/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite.java
@@ -28,6 +28,7 @@ import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.when;
import java.io.File;
+import java.net.InetAddress;
import java.nio.file.Path;
import java.util.List;
import java.util.Map;
@@ -95,7 +96,11 @@ public class TestSparkDataWrite {
@BeforeAll
public static void startSpark() {
- TestSparkDataWrite.spark =
SparkSession.builder().master("local[2]").getOrCreate();
+ TestSparkDataWrite.spark =
+ SparkSession.builder()
+ .master("local[2]")
+ .config("spark.driver.host",
InetAddress.getLoopbackAddress().getHostAddress())
+ .getOrCreate();
}
@AfterEach
diff --git
a/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReadProjection.java
b/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReadProjection.java
index becf6a064d..596d05d30b 100644
---
a/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReadProjection.java
+++
b/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReadProjection.java
@@ -27,6 +27,7 @@ import static org.assertj.core.api.Assertions.assertThat;
import java.io.File;
import java.io.IOException;
+import java.net.InetAddress;
import java.util.List;
import java.util.Map;
import java.util.UUID;
@@ -83,7 +84,11 @@ public class TestSparkReadProjection extends
TestReadProjection {
@BeforeAll
public static void startSpark() {
- TestSparkReadProjection.spark =
SparkSession.builder().master("local[2]").getOrCreate();
+ TestSparkReadProjection.spark =
+ SparkSession.builder()
+ .master("local[2]")
+ .config("spark.driver.host",
InetAddress.getLoopbackAddress().getHostAddress())
+ .getOrCreate();
ImmutableMap<String, String> config =
ImmutableMap.of(
"type", "hive",
diff --git
a/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreaming.java
b/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreaming.java
index 99130403a5..54048bbf21 100644
---
a/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreaming.java
+++
b/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreaming.java
@@ -23,6 +23,7 @@ import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
import java.io.File;
+import java.net.InetAddress;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
@@ -66,6 +67,7 @@ public class TestStructuredStreaming {
TestStructuredStreaming.spark =
SparkSession.builder()
.master("local[2]")
+ .config("spark.driver.host",
InetAddress.getLoopbackAddress().getHostAddress())
.config("spark.sql.shuffle.partitions", 4)
.getOrCreate();
}
diff --git
a/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/source/TestWriteMetricsConfig.java
b/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/source/TestWriteMetricsConfig.java
index 17e124dbf8..e2b5d8920e 100644
---
a/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/source/TestWriteMetricsConfig.java
+++
b/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/source/TestWriteMetricsConfig.java
@@ -24,6 +24,7 @@ import static
org.apache.iceberg.types.Types.NestedField.required;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
+import java.net.InetAddress;
import java.nio.ByteBuffer;
import java.nio.file.Path;
import java.util.List;
@@ -80,7 +81,11 @@ public class TestWriteMetricsConfig {
@BeforeAll
public static void startSpark() {
- TestWriteMetricsConfig.spark =
SparkSession.builder().master("local[2]").getOrCreate();
+ TestWriteMetricsConfig.spark =
+ SparkSession.builder()
+ .master("local[2]")
+ .config("spark.driver.host",
InetAddress.getLoopbackAddress().getHostAddress())
+ .getOrCreate();
TestWriteMetricsConfig.sc =
JavaSparkContext.fromSparkContext(spark.sparkContext());
}
