This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 51cdf34226ed [SPARK-46828][SQL] Remove the invalid assertion of remote
mode for spark sql shell
51cdf34226ed is described below
commit 51cdf34226ed8d137ac1c8374cc2473dc4818bbf
Author: Kent Yao <[email protected]>
AuthorDate: Wed Jan 24 22:17:16 2024 -0800
[SPARK-46828][SQL] Remove the invalid assertion of remote mode for spark
sql shell
### What changes were proposed in this pull request?
It is safe to clean up the read side code in SparkSQLCLIDriver as
`org.apache.hadoop.hive.ql.session.SessionState.setIsHiveServerQuery` is never
invoked.
### Why are the changes needed?
code refactoring for the purpose of having more upgradable hive deps.
### Does this PR introduce _any_ user-facing change?
no
### How was this patch tested?
- build and run `bin/spark-sql`
```
Spark Web UI available at http://***:4040
Spark master: local[*], Application Id: local-1706087266338
spark-sql (default)> show tables;
Time taken: 0.327 seconds
spark-sql (default)> show databases;
default
Time taken: 0.161 seconds, Fetched 1 row(s)
````
- CliSuite
### Was this patch authored or co-authored using generative AI tooling?
no
Closes #44868 from yaooqinn/SPARK-46828.
Authored-by: Kent Yao <[email protected]>
Signed-off-by: Dongjoon Hyun <[email protected]>
---
.../src/main/resources/error/error-classes.json | 5 ---
.../spark/sql/errors/QueryExecutionErrors.scala | 6 ----
.../sql/hive/thriftserver/SparkSQLCLIDriver.scala | 39 ++++------------------
3 files changed, 7 insertions(+), 43 deletions(-)
diff --git a/common/utils/src/main/resources/error/error-classes.json
b/common/utils/src/main/resources/error/error-classes.json
index 6088300f8e64..1f3122a502c5 100644
--- a/common/utils/src/main/resources/error/error-classes.json
+++ b/common/utils/src/main/resources/error/error-classes.json
@@ -6201,11 +6201,6 @@
"Cannot create array with <numElements> elements of data due to
exceeding the limit <maxRoundedArrayLength> elements for ArrayData.
<additionalErrorMessage>"
]
},
- "_LEGACY_ERROR_TEMP_2178" : {
- "message" : [
- "Remote operations not supported."
- ]
- },
"_LEGACY_ERROR_TEMP_2179" : {
"message" : [
"HiveServer2 Kerberos principal or keytab is not correctly configured."
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala
index a3e905090bf3..69794517f917 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala
@@ -1525,12 +1525,6 @@ private[sql] object QueryExecutionErrors extends
QueryErrorsBase with ExecutionE
cause = e)
}
- def remoteOperationsUnsupportedError(): SparkRuntimeException = {
- new SparkRuntimeException(
- errorClass = "_LEGACY_ERROR_TEMP_2178",
- messageParameters = Map.empty)
- }
-
def invalidKerberosConfigForHiveServer2Error(): Throwable = {
new SparkException(
errorClass = "_LEGACY_ERROR_TEMP_2179",
diff --git
a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala
b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala
index e0a1a31a36f3..0d3538e30941 100644
---
a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala
+++
b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala
@@ -36,7 +36,6 @@ import org.apache.hadoop.hive.ql.Driver
import org.apache.hadoop.hive.ql.processors._
import org.apache.hadoop.hive.ql.session.SessionState
import org.apache.hadoop.security.{Credentials, UserGroupInformation}
-import org.slf4j.LoggerFactory
import sun.misc.{Signal, SignalHandler}
import org.apache.spark.{ErrorMessageFormat, SparkConf, SparkThrowable,
SparkThrowableHelper}
@@ -45,7 +44,6 @@ import org.apache.spark.internal.Logging
import org.apache.spark.sql.AnalysisException
import org.apache.spark.sql.catalyst.analysis.FunctionRegistry
import org.apache.spark.sql.catalyst.util.SQLKeywordUtils
-import org.apache.spark.sql.errors.QueryExecutionErrors
import org.apache.spark.sql.hive.client.HiveClientImpl
import org.apache.spark.sql.hive.security.HiveDelegationTokenProvider
import
org.apache.spark.sql.hive.thriftserver.SparkSQLCLIDriver.closeHiveSessionStateIfStarted
@@ -149,10 +147,6 @@ private[hive] object SparkSQLCLIDriver extends Logging {
SparkSQLEnv.stop(exitCode)
}
- if (isRemoteMode(sessionState)) {
- // Hive 1.2 + not supported in CLI
- throw QueryExecutionErrors.remoteOperationsUnsupportedError()
- }
// Respect the configurations set by --hiveconf from the command line
// (based on Hive's CliDriver).
val hiveConfFromCmd =
sessionState.getOverriddenConfigurations.entrySet().asScala
@@ -305,12 +299,6 @@ private[hive] object SparkSQLCLIDriver extends Logging {
exit(ret)
}
-
- def isRemoteMode(state: CliSessionState): Boolean = {
- // sessionState.isRemoteMode
- state.isHiveServerQuery
- }
-
def printUsage(): Unit = {
val processor = new OptionsProcessor()
ReflectionUtils.invoke(classOf[OptionsProcessor], processor, "printUsage")
@@ -404,29 +392,17 @@ private[hive] object SparkSQLCLIDriver extends Logging {
}
private[hive] class SparkSQLCLIDriver extends CliDriver with Logging {
- private val sessionState = SessionState.get().asInstanceOf[CliSessionState]
+ private val sessionState = SessionState.get()
- private val LOG = LoggerFactory.getLogger(classOf[SparkSQLCLIDriver])
-
- private val console = new SessionState.LogHelper(LOG)
-
- private val isRemoteMode = {
- SparkSQLCLIDriver.isRemoteMode(sessionState)
- }
+ private val console = new SessionState.LogHelper(log)
- private val conf: Configuration =
- if (sessionState != null) sessionState.getConf else new Configuration()
+ private val conf: Configuration = sessionState.getConf
// Force initializing SparkSQLEnv. This is put here but not object
SparkSQLCliDriver
// because the Hive unit tests do not go through the main() code path.
- if (!isRemoteMode) {
- SparkSQLEnv.init()
- if (sessionState.getIsSilent) {
- SparkSQLEnv.sparkContext.setLogLevel("warn")
- }
- } else {
- // Hive 1.2 + not supported in CLI
- throw QueryExecutionErrors.remoteOperationsUnsupportedError()
+ SparkSQLEnv.init()
+ if (sessionState.getIsSilent) {
+ SparkSQLEnv.sparkContext.setLogLevel("warn")
}
override def setHiveVariables(hiveVariables: java.util.Map[String, String]):
Unit = {
@@ -453,8 +429,7 @@ private[hive] class SparkSQLCLIDriver extends CliDriver
with Logging {
closeHiveSessionStateIfStarted(sessionState)
SparkSQLCLIDriver.exit(EXIT_SUCCESS)
}
- if (tokens(0).toLowerCase(Locale.ROOT).equals("source") ||
- cmd_trimmed.startsWith("!") || isRemoteMode) {
+ if (tokens(0).toLowerCase(Locale.ROOT).equals("source") ||
cmd_trimmed.startsWith("!")) {
val startTimeNs = System.nanoTime()
super.processCmd(cmd)
val endTimeNs = System.nanoTime()
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
