This is an automated email from the ASF dual-hosted git repository.
ulyssesyou pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-kyuubi.git
The following commit(s) were added to refs/heads/master by this push:
new eb7ad512e [KYUUBI #2390] RuleEliminateMarker stays in analyze phase
for data masking
eb7ad512e is described below
commit eb7ad512e112a66786f0fedd2a023496cd66c40c
Author: Kent Yao <[email protected]>
AuthorDate: Mon Apr 18 10:58:11 2022 +0800
[KYUUBI #2390] RuleEliminateMarker stays in analyze phase for data masking
### _Why are the changes needed?_
Avoid creating duplicate data masking projection to prevent errors in #2390
### _How was this patch tested?_
- [x] Add some test cases that check the changes thoroughly including
negative and positive cases if possible
- [ ] Add screenshots for manual tests if appropriate
- [x] [Run
test](https://kyuubi.apache.org/docs/latest/develop_tools/testing.html#running-tests)
locally before make a pull request
Closes #2394 from yaooqinn/2390.
Closes #2390
31c5d071 [Kent Yao] [KYUUBI #2390] RuleEliminateMarker stays in analyze
phase for data masking
ad0138c6 [Kent Yao] [KYUUBI #2390] RuleEliminateMarker stays in analyze
phase for data masking
Authored-by: Kent Yao <[email protected]>
Signed-off-by: ulysses-you <[email protected]>
---
.../spark/authz/ranger/RangerSparkExtension.scala | 2 +-
.../authz/ranger/RangerSparkExtensionSuite.scala | 20 ++++++++++++++++++++
2 files changed, 21 insertions(+), 1 deletion(-)
diff --git
a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/ranger/RangerSparkExtension.scala
b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/ranger/RangerSparkExtension.scala
index 979fc550b..3e6d4ef99 100644
---
a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/ranger/RangerSparkExtension.scala
+++
b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/ranger/RangerSparkExtension.scala
@@ -40,7 +40,7 @@ class RangerSparkExtension extends (SparkSessionExtensions =>
Unit) {
override def apply(v1: SparkSessionExtensions): Unit = {
v1.injectResolutionRule(new RuleApplyRowFilterAndDataMasking(_))
- v1.injectPostHocResolutionRule(_ => new RuleEliminateMarker())
+ v1.injectOptimizerRule(_ => new RuleEliminateMarker())
v1.injectOptimizerRule(new RuleAuthorization(_))
}
}
diff --git
a/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/ranger/RangerSparkExtensionSuite.scala
b/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/ranger/RangerSparkExtensionSuite.scala
index 631ed6e7d..cb8c63aac 100644
---
a/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/ranger/RangerSparkExtensionSuite.scala
+++
b/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/ranger/RangerSparkExtensionSuite.scala
@@ -235,6 +235,26 @@ abstract class RangerSparkExtensionSuite extends
KyuubiFunSuite with SparkSessio
doAs("admin", sql(s"DROP TABLE IF EXISTS $db.$table"))
}
}
+
+ test("KYUUBI #2390: RuleEliminateMarker stays in analyze phase for data
masking") {
+ val db = "default"
+ val table = "src"
+ val create =
+ s"CREATE TABLE IF NOT EXISTS $db.$table (key int, value1 int) USING
$format"
+ try {
+ doAs("admin", sql(create))
+ doAs("admin", sql(s"INSERT INTO $db.$table SELECT 1, 1"))
+ // scalastyle: off
+ doAs(
+ "bob", {
+ assert(sql(s"select * from $db.$table").collect() ===
+ Seq(Row(1, DigestUtils.md5Hex("1"))))
+ assert(Try(sql(s"select * from $db.$table").show(1)).isSuccess)
+ })
+ } finally {
+ doAs("admin", sql(s"DROP TABLE IF EXISTS $db.$table"))
+ }
+ }
}
class InMemoryCatalogRangerSparkExtensionSuite extends
RangerSparkExtensionSuite {
