This is an automated email from the ASF dual-hosted git repository.
hvanhovell pushed a commit to branch branch-3.4
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.4 by this push:
new d1c3902921f [SPARK-42564][CONNECT] Implement SparkSession.version and
SparkSession.time
d1c3902921f is described below
commit d1c3902921f6978fbdcda6de5def42bff08c39bd
Author: panbingkun <pbk1...@gmail.com>
AuthorDate: Sun Feb 26 20:39:12 2023 -0400
[SPARK-42564][CONNECT] Implement SparkSession.version and SparkSession.time
### What changes were proposed in this pull request?
The pr aims to implement SparkSession.version and SparkSession.time.
### Why are the changes needed?
API coverage.
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
Add new UT.
Closes #40176 from panbingkun/SPARK-42564.
Authored-by: panbingkun <pbk1...@gmail.com>
Signed-off-by: Herman van Hovell <her...@databricks.com>
(cherry picked from commit 9c7aa16c9a5ede3a712a49f84118bfff89273f60)
Signed-off-by: Herman van Hovell <her...@databricks.com>
---
.../scala/org/apache/spark/sql/SparkSession.scala | 20 ++++++++++++++++++++
.../org/apache/spark/sql/ClientE2ETestSuite.scala | 10 ++++++++++
2 files changed, 30 insertions(+)
diff --git
a/connector/connect/client/jvm/src/main/scala/org/apache/spark/sql/SparkSession.scala
b/connector/connect/client/jvm/src/main/scala/org/apache/spark/sql/SparkSession.scala
index 53cd3955232..e39a6779e25 100644
---
a/connector/connect/client/jvm/src/main/scala/org/apache/spark/sql/SparkSession.scala
+++
b/connector/connect/client/jvm/src/main/scala/org/apache/spark/sql/SparkSession.scala
@@ -17,12 +17,14 @@
package org.apache.spark.sql
import java.io.Closeable
+import java.util.concurrent.TimeUnit._
import java.util.concurrent.atomic.AtomicLong
import scala.collection.JavaConverters._
import org.apache.arrow.memory.RootAllocator
+import org.apache.spark.SPARK_VERSION
import org.apache.spark.annotation.Experimental
import org.apache.spark.connect.proto
import org.apache.spark.internal.Logging
@@ -59,6 +61,24 @@ class SparkSession(
private[this] val allocator = new RootAllocator()
+ def version: String = SPARK_VERSION
+
+ /**
+ * Executes some code block and prints to stdout the time taken to execute
the block. This is
+ * available in Scala only and is used primarily for interactive testing and
debugging.
+ *
+ * @since 3.4.0
+ */
+ def time[T](f: => T): T = {
+ val start = System.nanoTime()
+ val ret = f
+ val end = System.nanoTime()
+ // scalastyle:off println
+ println(s"Time taken: ${NANOSECONDS.toMillis(end - start)} ms")
+ // scalastyle:on println
+ ret
+ }
+
/**
* Executes a SQL query substituting named parameters by the given
arguments, returning the
* result as a `DataFrame`. This API eagerly runs DDL/DML commands, but not
for SELECT queries.
diff --git
a/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/ClientE2ETestSuite.scala
b/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/ClientE2ETestSuite.scala
index 23a5d5f5e9e..33e9d0756c1 100644
---
a/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/ClientE2ETestSuite.scala
+++
b/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/ClientE2ETestSuite.scala
@@ -26,6 +26,7 @@ import org.apache.commons.io.FileUtils
import org.apache.commons.io.output.TeeOutputStream
import org.scalactic.TolerantNumerics
+import org.apache.spark.SPARK_VERSION
import org.apache.spark.sql.connect.client.util.{IntegrationTestUtils,
RemoteSparkSession}
import org.apache.spark.sql.functions.{aggregate, array, col, lit, rand,
sequence, shuffle, transform, udf}
import org.apache.spark.sql.types._
@@ -417,4 +418,13 @@ class ClientE2ETestSuite extends RemoteSparkSession {
spark.range(1000).createOrReplaceGlobalTempView("view1")
assert(spark.sql("SELECT * FROM global_temp.view1").count() == 1000)
}
+
+ test("version") {
+ assert(spark.version == SPARK_VERSION)
+ }
+
+ test("time") {
+ val timeFragments = Seq("Time taken: ", " ms")
+ testCapturedStdOut(spark.time(spark.sql("select 1").collect()),
timeFragments: _*)
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
