This is an automated email from the ASF dual-hosted git repository.
liuxun pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/submarine.git
The following commit(s) were added to refs/heads/master by this push:
new f2b484c SUBMARINE-458. ResetCommand should not wipe out all configs
f2b484c is described below
commit f2b484c7fce2c32f51a380ba41622c56cf1b0e0a
Author: Kent Yao <[email protected]>
AuthorDate: Fri Mar 27 10:55:29 2020 +0800
SUBMARINE-458. ResetCommand should not wipe out all configs
### What is this PR for?
add a runtime replacement for `ResetCommand` to keep the static configs of
spark
### What type of PR is it?
Improvement
### Todos
* [ ] - Task
### What is the Jira issue?
https://issues.apache.org/jira/browse/SUBMARINE-458
### How should this be tested?
add uts
### Screenshots (if appropriate)
### Questions:
* Does the licenses files need update? No
* Is there breaking changes for older versions? No
* Does this needs documentation? No
Author: Kent Yao <[email protected]>
Closes #251 from yaooqinn/SUBMARINE-458 and squashes the following commits:
87d87ea [Kent Yao] SUBMARINE-458. ResetCommand should not wipe out all
configs
---
...ubmarineSparkRangerAuthorizationExtension.scala | 1 +
.../execution/command/SubmarineResetCommand.scala | 38 ++++++++++++++++++++++
.../spark/security/AuthorizationTest.scala | 22 +++++++++++++
3 files changed, 61 insertions(+)
diff --git
a/submarine-security/spark-security/src/main/scala/org/apache/spark/sql/catalyst/optimizer/SubmarineSparkRangerAuthorizationExtension.scala
b/submarine-security/spark-security/src/main/scala/org/apache/spark/sql/catalyst/optimizer/SubmarineSparkRangerAuthorizationExtension.scala
index 22fe1e6..0236d7c 100644
---
a/submarine-security/spark-security/src/main/scala/org/apache/spark/sql/catalyst/optimizer/SubmarineSparkRangerAuthorizationExtension.scala
+++
b/submarine-security/spark-security/src/main/scala/org/apache/spark/sql/catalyst/optimizer/SubmarineSparkRangerAuthorizationExtension.scala
@@ -56,6 +56,7 @@ case class SubmarineSparkRangerAuthorizationExtension(spark:
SparkSession)
case s: SubmarineShowDatabasesCommand => s
case s: ShowTablesCommand => SubmarineShowTablesCommand(s)
case s: SubmarineShowTablesCommand => s
+ case ResetCommand => SubmarineResetCommand
case _ =>
val operationType: SparkOperationType = toOperationType(plan)
val (in, out) = PrivilegesBuilder.build(plan)
diff --git
a/submarine-security/spark-security/src/main/scala/org/apache/spark/sql/execution/command/SubmarineResetCommand.scala
b/submarine-security/spark-security/src/main/scala/org/apache/spark/sql/execution/command/SubmarineResetCommand.scala
new file mode 100644
index 0000000..837624e
--- /dev/null
+++
b/submarine-security/spark-security/src/main/scala/org/apache/spark/sql/execution/command/SubmarineResetCommand.scala
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.spark.sql.execution.command
+
+import org.apache.spark.sql.{Row, SparkSession}
+
+/**
+ * Runtime replacement for spark's original [[ResetCommand]], since the
operation will
+ * wipe out all configuration including our security-specific ones
+ * see: https://issues.apache.org/jira/browse/SPARK-31234
+ */
+case object SubmarineResetCommand extends RunnableCommand {
+ override def run(sparkSession: SparkSession): Seq[Row] = {
+ val conf = sparkSession.sessionState.conf
+ conf.clear()
+ sparkSession.sparkContext.getConf.getAll.foreach { case (k, v) =>
+ conf.setConfString(k, v)
+ }
+ Seq.empty[Row]
+ }
+}
diff --git
a/submarine-security/spark-security/src/test/scala/org/apache/submarine/spark/security/AuthorizationTest.scala
b/submarine-security/spark-security/src/test/scala/org/apache/submarine/spark/security/AuthorizationTest.scala
index f38c6d0..5cbf439 100644
---
a/submarine-security/spark-security/src/test/scala/org/apache/submarine/spark/security/AuthorizationTest.scala
+++
b/submarine-security/spark-security/src/test/scala/org/apache/submarine/spark/security/AuthorizationTest.scala
@@ -19,6 +19,9 @@
package org.apache.submarine.spark.security
+import java.util.NoSuchElementException
+
+import org.apache.spark.sql.execution.command.SubmarineResetCommand
import org.apache.spark.sql.hive.test.TestHive
import org.scalatest.{BeforeAndAfterAll, FunSuite}
@@ -71,6 +74,25 @@ class AuthorizationTest extends FunSuite with
BeforeAndAfterAll {
spark.reset()
}
+ test("reset command") {
+ val sparkConf = spark.sparkContext.getConf
+ sql("set submarine.spark.some=any")
+ assert(spark.sessionState.conf.getConfString("submarine.spark.some") ===
"any")
+ val reset = sql("reset")
+
+ assert(reset.queryExecution.optimizedPlan.getClass ===
SubmarineResetCommand.getClass)
+
+ intercept[NoSuchElementException] {
+ spark.sessionState.conf.getConfString("submarine.spark.some")
+ }
+
+ assert(spark.sessionState.conf.getConfString("spark.ui.enabled") ===
+ sparkConf.get("spark.ui.enabled"))
+
+ assert(spark.sessionState.conf.getConfString("spark.app.id") ===
+ sparkConf.getAppId)
+ }
+
test("show databases") {
withUser("alice") {
assert(sql("show databases").count() === 0)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
