(iceberg) branch main updated: Spark 3.5: Remove obsolete conf parsing logic (#10309)

[aokolnychyi]

This is an automated email from the ASF dual-hosted git repository.

aokolnychyi pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg.git


The following commit(s) were added to refs/heads/main by this push:
     new 04792cf991 Spark 3.5: Remove obsolete conf parsing logic (#10309)
04792cf991 is described below

commit 04792cf991608d836a1ce60651bf65d1b7f67120
Author: Anton Okolnychyi <aokolnyc...@apple.com>
AuthorDate: Sat May 11 09:55:09 2024 -0700

    Spark 3.5: Remove obsolete conf parsing logic (#10309)
---
 .../org/apache/iceberg/spark/SparkConfParser.java  | 28 ++++++++++++----------
 .../org/apache/iceberg/spark/SparkReadConf.java    |  2 --
 .../apache/iceberg/spark/TestSparkWriteConf.java   |  9 +++++++
 3 files changed, 25 insertions(+), 14 deletions(-)

diff --git 
a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/SparkConfParser.java 
b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/SparkConfParser.java
index e3b01b8375..bdb66bbe3c 100644
--- 
a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/SparkConfParser.java
+++ 
b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/SparkConfParser.java
@@ -20,7 +20,6 @@ package org.apache.iceberg.spark;
 
 import java.time.Duration;
 import java.util.List;
-import java.util.Locale;
 import java.util.Map;
 import java.util.function.Function;
 import org.apache.iceberg.Table;
@@ -31,23 +30,24 @@ import org.apache.spark.network.util.JavaUtils;
 import org.apache.spark.sql.RuntimeConfig;
 import org.apache.spark.sql.SparkSession;
 import org.apache.spark.sql.internal.SQLConf;
+import org.apache.spark.sql.util.CaseInsensitiveStringMap;
 
 class SparkConfParser {
 
   private final Map<String, String> properties;
   private final RuntimeConfig sessionConf;
-  private final Map<String, String> options;
+  private final CaseInsensitiveStringMap options;
 
   SparkConfParser() {
     this.properties = ImmutableMap.of();
     this.sessionConf = new RuntimeConfig(SQLConf.get());
-    this.options = ImmutableMap.of();
+    this.options = CaseInsensitiveStringMap.empty();
   }
 
   SparkConfParser(SparkSession spark, Table table, Map<String, String> 
options) {
     this.properties = table.properties();
     this.sessionConf = spark.conf();
-    this.options = options;
+    this.options = asCaseInsensitiveStringMap(options);
   }
 
   public BooleanConfParser booleanConf() {
@@ -70,6 +70,14 @@ class SparkConfParser {
     return new DurationConfParser();
   }
 
+  private static CaseInsensitiveStringMap 
asCaseInsensitiveStringMap(Map<String, String> map) {
+    if (map instanceof CaseInsensitiveStringMap) {
+      return (CaseInsensitiveStringMap) map;
+    } else {
+      return new CaseInsensitiveStringMap(map);
+    }
+  }
+
   class BooleanConfParser extends ConfParser<BooleanConfParser, Boolean> {
     private Boolean defaultValue;
     private boolean negate = false;
@@ -220,14 +228,10 @@ class SparkConfParser {
     }
 
     protected T parse(Function<String, T> conversion, T defaultValue) {
-      if (!optionNames.isEmpty()) {
-        for (String optionName : optionNames) {
-          // use lower case comparison as DataSourceOptions.asMap() in Spark 2 
returns a lower case
-          // map
-          String optionValue = 
options.get(optionName.toLowerCase(Locale.ROOT));
-          if (optionValue != null) {
-            return conversion.apply(optionValue);
-          }
+      for (String optionName : optionNames) {
+        String optionValue = options.get(optionName);
+        if (optionValue != null) {
+          return conversion.apply(optionValue);
         }
       }
 
diff --git 
a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/SparkReadConf.java 
b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/SparkReadConf.java
index 2990d981d0..f5127e9cda 100644
--- a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/SparkReadConf.java
+++ b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/SparkReadConf.java
@@ -57,7 +57,6 @@ public class SparkReadConf {
   private final SparkSession spark;
   private final Table table;
   private final String branch;
-  private final Map<String, String> readOptions;
   private final SparkConfParser confParser;
 
   public SparkReadConf(SparkSession spark, Table table, Map<String, String> 
readOptions) {
@@ -69,7 +68,6 @@ public class SparkReadConf {
     this.spark = spark;
     this.table = table;
     this.branch = branch;
-    this.readOptions = readOptions;
     this.confParser = new SparkConfParser(spark, table, readOptions);
   }
 
diff --git 
a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/TestSparkWriteConf.java
 
b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/TestSparkWriteConf.java
index 9f4a4f47bf..bf2f59f1a4 100644
--- 
a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/TestSparkWriteConf.java
+++ 
b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/TestSparkWriteConf.java
@@ -79,6 +79,15 @@ public class TestSparkWriteConf extends TestBaseWithCatalog {
     sql("DROP TABLE IF EXISTS %s", tableName);
   }
 
+  @TestTemplate
+  public void testOptionCaseInsensitive() {
+    Table table = validationCatalog.loadTable(tableIdent);
+    Map<String, String> options = ImmutableMap.of("option", "value");
+    SparkConfParser parser = new SparkConfParser(spark, table, options);
+    String parsedValue = parser.stringConf().option("oPtIoN").parseOptional();
+    assertThat(parsedValue).isEqualTo("value");
+  }
+
   @TestTemplate
   public void testDurationConf() {
     Table table = validationCatalog.loadTable(tableIdent);