[orc] branch branch-1.8 updated: ORC-1414: [Java] Upgrade java bench module to spark3.4

[dongjoon]

This is an automated email from the ASF dual-hosted git repository.

dongjoon pushed a commit to branch branch-1.8
in repository https://gitbox.apache.org/repos/asf/orc.git


The following commit(s) were added to refs/heads/branch-1.8 by this push:
     new 30960694d ORC-1414: [Java] Upgrade java bench module to spark3.4
30960694d is described below

commit 30960694d95d3df95986a3be6bef89e8940cb2a3
Author: deshanxiao <deshanx...@microsoft.com>
AuthorDate: Tue Apr 25 16:49:08 2023 -0700

    ORC-1414: [Java] Upgrade java bench module to spark3.4
    
    ### What changes were proposed in this pull request?
    This PR is aimed to upgrade java bench module to spark3.4.
    The PR also fix some compile errors.
    
    ### Why are the changes needed?
    To test & bench with the latest Spark version.
    
    ### How was this patch tested?
    N/A
    
    Closes #1472
    
    Closes #1473 from deshanxiao/update-spark3.4.
    
    Authored-by: deshanxiao <deshanx...@microsoft.com>
    Signed-off-by: William Hyun <will...@apache.org>
---
 java/bench/pom.xml                                                 | 2 +-
 .../spark/src/java/org/apache/orc/bench/spark/SparkBenchmark.java  | 7 ++++---
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/java/bench/pom.xml b/java/bench/pom.xml
index f825c7fee..43bfde1e4 100644
--- a/java/bench/pom.xml
+++ b/java/bench/pom.xml
@@ -40,7 +40,7 @@
     <junit.version>5.8.2</junit.version>
     <orc.version>${project.version}</orc.version>
     <parquet.version>1.13.0</parquet.version>
-    <spark.version>3.3.2</spark.version>
+    <spark.version>3.4.0</spark.version>
   </properties>
 
   <dependencyManagement>
diff --git 
a/java/bench/spark/src/java/org/apache/orc/bench/spark/SparkBenchmark.java 
b/java/bench/spark/src/java/org/apache/orc/bench/spark/SparkBenchmark.java
index c8883152e..4991c030c 100644
--- a/java/bench/spark/src/java/org/apache/orc/bench/spark/SparkBenchmark.java
+++ b/java/bench/spark/src/java/org/apache/orc/bench/spark/SparkBenchmark.java
@@ -29,6 +29,7 @@ import org.apache.orc.bench.core.IOCounters;
 import org.apache.orc.bench.core.OrcBenchmark;
 import org.apache.orc.bench.core.Utilities;
 import org.apache.orc.bench.core.convert.GenerateVariants;
+import org.apache.spark.paths.SparkPath;
 import org.apache.spark.sql.SparkSession;
 import org.apache.spark.sql.avro.AvroFileFormat;
 import org.apache.spark.sql.catalyst.InternalRow;
@@ -201,7 +202,7 @@ public class SparkBenchmark implements OrcBenchmark {
             
JavaConverters.collectionAsScalaIterableConverter(filters).asScala().toSeq(),
             scalaMap, source.conf);
     PartitionedFile file = new PartitionedFile(InternalRow.empty(),
-        source.path.toString(), 0, Long.MAX_VALUE, new String[0], 0L, 0L);
+        SparkPath.fromPath(source.path), 0, Long.MAX_VALUE, new String[0], 0L, 
0L);
     processReader(factory.apply(file), statistics, counters, blackhole);
   }
 
@@ -249,7 +250,7 @@ public class SparkBenchmark implements OrcBenchmark {
             
JavaConverters.collectionAsScalaIterableConverter(filters).asScala().toSeq(),
             scalaMap, source.conf);
     PartitionedFile file = new PartitionedFile(InternalRow.empty(),
-        source.path.toString(), 0, Long.MAX_VALUE, new String[0], 0L, 0L);
+        SparkPath.fromPath(source.path), 0, Long.MAX_VALUE, new String[0], 0L, 
0L);
     processReader(factory.apply(file), statistics, counters, blackhole);
   }
 
@@ -301,7 +302,7 @@ public class SparkBenchmark implements OrcBenchmark {
             
JavaConverters.collectionAsScalaIterableConverter(filters).asScala().toSeq(),
             scalaMap, source.conf);
     PartitionedFile file = new PartitionedFile(InternalRow.empty(),
-        source.path.toString(), 0, Long.MAX_VALUE, new String[0], 0L, 0L);
+        SparkPath.fromPath(source.path), 0, Long.MAX_VALUE, new String[0], 0L, 
0L);
     processReader(factory.apply(file), statistics, counters, blackhole);
   }
 }