This is an automated email from the ASF dual-hosted git repository.
chengpan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kyuubi.git
The following commit(s) were added to refs/heads/master by this push:
new dbbcf4f4c [KYUUBI #5073] Correct the method name in
SparkSQLLineageParserHelperSuite
dbbcf4f4c is described below
commit dbbcf4f4cd680954d33ade6b20087b7f42a0062a
Author: hezhao2 <[email protected]>
AuthorDate: Thu Jul 20 15:37:29 2023 +0800
[KYUUBI #5073] Correct the method name in SparkSQLLineageParserHelperSuite
### _Why are the changes needed?_
There are some typos in SparkSQLLineageParserHelperSuite, so this pr is to
fix these mistakes.
### _How was this patch tested?_
- [ ] Add some test cases that check the changes thoroughly including
negative and positive cases if possible
- [ ] Add screenshots for manual tests if appropriate
- [ ] [Run
test](https://kyuubi.readthedocs.io/en/master/contributing/code/testing.html#running-tests)
locally before make a pull request
Closes #5073 from zhaohehuhu/Improvement-0720.
Closes #5073
b6346369e [hezhao2] correct the method name in
SparkSQLLineageParserHelperSuite
Authored-by: hezhao2 <[email protected]>
Signed-off-by: Cheng Pan <[email protected]>
---
.../helper/SparkSQLLineageParserHelperSuite.scala | 173 ++++++++++-----------
1 file changed, 86 insertions(+), 87 deletions(-)
diff --git
a/extensions/spark/kyuubi-spark-lineage/src/test/scala/org/apache/kyuubi/plugin/lineage/helper/SparkSQLLineageParserHelperSuite.scala
b/extensions/spark/kyuubi-spark-lineage/src/test/scala/org/apache/kyuubi/plugin/lineage/helper/SparkSQLLineageParserHelperSuite.scala
index 96003f051..ebc4610ee 100644
---
a/extensions/spark/kyuubi-spark-lineage/src/test/scala/org/apache/kyuubi/plugin/lineage/helper/SparkSQLLineageParserHelperSuite.scala
+++
b/extensions/spark/kyuubi-spark-lineage/src/test/scala/org/apache/kyuubi/plugin/lineage/helper/SparkSQLLineageParserHelperSuite.scala
@@ -17,7 +17,6 @@
package org.apache.kyuubi.plugin.lineage.helper
-import scala.collection.immutable.List
import scala.reflect.io.File
import org.apache.spark.SparkConf
@@ -75,7 +74,7 @@ class SparkSQLLineageParserHelperSuite extends KyuubiFunSuite
withView("alterviewascommand", "alterviewascommand1") { _ =>
spark.sql("create view alterviewascommand as select key from
test_db0.test_table0")
val ret0 =
- exectractLineage("alter view alterviewascommand as select key from
test_db0.test_table0")
+ extractLineage("alter view alterviewascommand as select key from
test_db0.test_table0")
assert(ret0 == Lineage(
List("test_db0.test_table0"),
List("default.alterviewascommand"),
@@ -83,7 +82,7 @@ class SparkSQLLineageParserHelperSuite extends KyuubiFunSuite
spark.sql("create view alterviewascommand1 as select * from
test_db0.test_table0")
val ret1 =
- exectractLineage("alter view alterviewascommand1 as select * from
test_db0.test_table0")
+ extractLineage("alter view alterviewascommand1 as select * from
test_db0.test_table0")
assert(ret1 == Lineage(
List("test_db0.test_table0"),
@@ -102,7 +101,7 @@ class SparkSQLLineageParserHelperSuite extends
KyuubiFunSuite
ddls.split("\n").filter(_.nonEmpty).foreach(spark.sql(_).collect())
withView("test_view") { _ =>
- val result = exectractLineage(
+ val result = extractLineage(
"create view test_view(a, b, c) as" +
" select col1 as a, col2 as b, col3 as c from v2_catalog.db.tbb")
assert(result == Lineage(
@@ -123,7 +122,7 @@ class SparkSQLLineageParserHelperSuite extends
KyuubiFunSuite
ddls.split("\n").filter(_.nonEmpty).foreach(spark.sql(_).collect())
withTable("v2_catalog.db.tb0") { _ =>
val ret0 =
- exectractLineage(
+ extractLineage(
s"insert into table v2_catalog.db.tb0 " +
s"select key as col1, value as col2 from test_db0.test_table0")
assert(ret0 == Lineage(
@@ -134,7 +133,7 @@ class SparkSQLLineageParserHelperSuite extends
KyuubiFunSuite
("v2_catalog.db.tb0.col2", Set("test_db0.test_table0.value")))))
val ret1 =
- exectractLineage(
+ extractLineage(
s"insert overwrite table v2_catalog.db.tb0 partition(col2) " +
s"select key as col1, value as col2 from test_db0.test_table0")
assert(ret1 == Lineage(
@@ -145,7 +144,7 @@ class SparkSQLLineageParserHelperSuite extends
KyuubiFunSuite
("v2_catalog.db.tb0.col2", Set("test_db0.test_table0.value")))))
val ret2 =
- exectractLineage(
+ extractLineage(
s"insert overwrite table v2_catalog.db.tb0 partition(col2 = 'bb') " +
s"select key as col1 from test_db0.test_table0")
assert(ret2 == Lineage(
@@ -166,7 +165,7 @@ class SparkSQLLineageParserHelperSuite extends
KyuubiFunSuite
|""".stripMargin
ddls.split("\n").filter(_.nonEmpty).foreach(spark.sql(_).collect())
withTable("v2_catalog.db.target_t", "v2_catalog.db.source_t") { _ =>
- val ret0 = exectractLineageWithoutExecuting("MERGE INTO
v2_catalog.db.target_t AS target " +
+ val ret0 = extractLineageWithoutExecuting("MERGE INTO
v2_catalog.db.target_t AS target " +
"USING v2_catalog.db.source_t AS source " +
"ON target.id = source.id " +
"WHEN MATCHED THEN " +
@@ -181,7 +180,7 @@ class SparkSQLLineageParserHelperSuite extends
KyuubiFunSuite
("v2_catalog.db.target_t.name", Set("v2_catalog.db.source_t.name")),
("v2_catalog.db.target_t.price",
Set("v2_catalog.db.source_t.price")))))
- val ret1 = exectractLineageWithoutExecuting("MERGE INTO
v2_catalog.db.target_t AS target " +
+ val ret1 = extractLineageWithoutExecuting("MERGE INTO
v2_catalog.db.target_t AS target " +
"USING v2_catalog.db.source_t AS source " +
"ON target.id = source.id " +
"WHEN MATCHED THEN " +
@@ -196,7 +195,7 @@ class SparkSQLLineageParserHelperSuite extends
KyuubiFunSuite
("v2_catalog.db.target_t.name", Set("v2_catalog.db.source_t.name")),
("v2_catalog.db.target_t.price",
Set("v2_catalog.db.source_t.price")))))
- val ret2 = exectractLineageWithoutExecuting("MERGE INTO
v2_catalog.db.target_t AS target " +
+ val ret2 = extractLineageWithoutExecuting("MERGE INTO
v2_catalog.db.target_t AS target " +
"USING (select a.id, a.name, b.price " +
"from v2_catalog.db.source_t a join v2_catalog.db.pivot_t b) AS source
" +
"ON target.id = source.id " +
@@ -218,7 +217,7 @@ class SparkSQLLineageParserHelperSuite extends
KyuubiFunSuite
test("columns lineage extract - CreateViewCommand") {
withView("createviewcommand", "createviewcommand1", "createviewcommand2")
{ _ =>
- val ret0 = exectractLineage(
+ val ret0 = extractLineage(
"create view createviewcommand(a, b) as select key, value from
test_db0.test_table0")
assert(ret0 == Lineage(
List("test_db0.test_table0"),
@@ -227,7 +226,7 @@ class SparkSQLLineageParserHelperSuite extends
KyuubiFunSuite
("default.createviewcommand.a", Set("test_db0.test_table0.key")),
("default.createviewcommand.b", Set("test_db0.test_table0.value")))))
- val ret1 = exectractLineage(
+ val ret1 = extractLineage(
"create view createviewcommand1 as select key, value from
test_db0.test_table0")
assert(ret1 == Lineage(
List("test_db0.test_table0"),
@@ -236,7 +235,7 @@ class SparkSQLLineageParserHelperSuite extends
KyuubiFunSuite
("default.createviewcommand1.key", Set("test_db0.test_table0.key")),
("default.createviewcommand1.value",
Set("test_db0.test_table0.value")))))
- val ret2 = exectractLineage(
+ val ret2 = extractLineage(
"create view createviewcommand2 as select * from test_db0.test_table0")
assert(ret2 == Lineage(
List("test_db0.test_table0"),
@@ -251,7 +250,7 @@ class SparkSQLLineageParserHelperSuite extends
KyuubiFunSuite
withTable("createdatasourcetableasselectcommand",
"createdatasourcetableasselectcommand1") {
_ =>
val ret0 =
- exectractLineage("create table createdatasourcetableasselectcommand
using parquet" +
+ extractLineage("create table createdatasourcetableasselectcommand
using parquet" +
" AS SELECT key, value FROM test_db0.test_table0")
assert(ret0 == Lineage(
List("test_db0.test_table0"),
@@ -263,7 +262,7 @@ class SparkSQLLineageParserHelperSuite extends
KyuubiFunSuite
Set("test_db0.test_table0.value")))))
val ret1 =
- exectractLineage("create table createdatasourcetableasselectcommand1
using parquet" +
+ extractLineage("create table createdatasourcetableasselectcommand1
using parquet" +
" AS SELECT * FROM test_db0.test_table0")
assert(ret1 == Lineage(
List("test_db0.test_table0"),
@@ -278,7 +277,7 @@ class SparkSQLLineageParserHelperSuite extends
KyuubiFunSuite
test("columns lineage extract - CreateHiveTableAsSelectCommand") {
withTable("createhivetableasselectcommand",
"createhivetableasselectcommand1") { _ =>
- val ret0 = exectractLineage("create table createhivetableasselectcommand
using hive" +
+ val ret0 = extractLineage("create table createhivetableasselectcommand
using hive" +
" as select key, value from test_db0.test_table0")
assert(ret0 == Lineage(
List("test_db0.test_table0"),
@@ -287,7 +286,7 @@ class SparkSQLLineageParserHelperSuite extends
KyuubiFunSuite
("default.createhivetableasselectcommand.key",
Set("test_db0.test_table0.key")),
("default.createhivetableasselectcommand.value",
Set("test_db0.test_table0.value")))))
- val ret1 = exectractLineage("create table
createhivetableasselectcommand1 using hive" +
+ val ret1 = extractLineage("create table createhivetableasselectcommand1
using hive" +
" as select * from test_db0.test_table0")
assert(ret1 == Lineage(
List("test_db0.test_table0"),
@@ -301,7 +300,7 @@ class SparkSQLLineageParserHelperSuite extends
KyuubiFunSuite
test("columns lineage extract - OptimizedCreateHiveTableAsSelectCommand") {
withTable("optimizedcreatehivetableasselectcommand") { _ =>
val ret =
- exectractLineage(
+ extractLineage(
"create table optimizedcreatehivetableasselectcommand stored as
parquet " +
"as select * from test_db0.test_table0")
assert(ret == Lineage(
@@ -319,7 +318,7 @@ class SparkSQLLineageParserHelperSuite extends
KyuubiFunSuite
withTable(
"v2_catalog.db.createhivetableasselectcommand",
"v2_catalog.db.createhivetableasselectcommand1") { _ =>
- val ret0 = exectractLineage("create table
v2_catalog.db.createhivetableasselectcommand" +
+ val ret0 = extractLineage("create table
v2_catalog.db.createhivetableasselectcommand" +
" as select key, value from test_db0.test_table0")
assert(ret0 == Lineage(
List("test_db0.test_table0"),
@@ -330,7 +329,7 @@ class SparkSQLLineageParserHelperSuite extends
KyuubiFunSuite
"v2_catalog.db.createhivetableasselectcommand.value",
Set("test_db0.test_table0.value")))))
- val ret1 = exectractLineage("create table
v2_catalog.db.createhivetableasselectcommand1" +
+ val ret1 = extractLineage("create table
v2_catalog.db.createhivetableasselectcommand1" +
" as select * from test_db0.test_table0")
assert(ret1 == Lineage(
List("test_db0.test_table0"),
@@ -364,7 +363,7 @@ class SparkSQLLineageParserHelperSuite extends
KyuubiFunSuite
spark.sessionState.catalog.createTable(newTable, ignoreIfExists = false)
val ret0 =
- exectractLineage(
+ extractLineage(
s"insert into table $tableName select key, value from
test_db0.test_table0")
assert(ret0 == Lineage(
List("test_db0.test_table0"),
@@ -374,7 +373,7 @@ class SparkSQLLineageParserHelperSuite extends
KyuubiFunSuite
("default.insertintodatasourcecommand.b",
Set("test_db0.test_table0.value")))))
val ret1 =
- exectractLineage(
+ extractLineage(
s"insert into table $tableName select * from test_db0.test_table0")
assert(ret1 == Lineage(
List("test_db0.test_table0"),
@@ -384,7 +383,7 @@ class SparkSQLLineageParserHelperSuite extends
KyuubiFunSuite
("default.insertintodatasourcecommand.b",
Set("test_db0.test_table0.value")))))
val ret2 =
- exectractLineage(
+ extractLineage(
s"insert into table $tableName " +
s"select (select key from test_db0.test_table1 limit 1) + 1 as aa,
" +
s"value as bb from test_db0.test_table0")
@@ -403,7 +402,7 @@ class SparkSQLLineageParserHelperSuite extends
KyuubiFunSuite
withTable(tableName) { _ =>
spark.sql(s"CREATE TABLE $tableName (a int, b string) USING parquet")
val ret0 =
- exectractLineage(
+ extractLineage(
s"insert into table $tableName select key, value from
test_db0.test_table0")
assert(ret0 == Lineage(
@@ -419,10 +418,10 @@ class SparkSQLLineageParserHelperSuite extends
KyuubiFunSuite
test("columns lineage extract - InsertIntoDatasourceDirCommand") {
val tableDirectory = getClass.getResource("/").getPath + "table_directory"
val directory = File(tableDirectory).createDirectory()
- val ret0 = exectractLineage(s"""
- |INSERT OVERWRITE DIRECTORY
'$directory.path'
- |USING parquet
- |SELECT * FROM
test_db0.test_table_part0""".stripMargin)
+ val ret0 = extractLineage(s"""
+ |INSERT OVERWRITE DIRECTORY '$directory.path'
+ |USING parquet
+ |SELECT * FROM
test_db0.test_table_part0""".stripMargin)
assert(ret0 == Lineage(
List("test_db0.test_table_part0"),
List(s"""`$directory.path`"""),
@@ -435,10 +434,10 @@ class SparkSQLLineageParserHelperSuite extends
KyuubiFunSuite
test("columns lineage extract - InsertIntoHiveDirCommand") {
val tableDirectory = getClass.getResource("/").getPath + "table_directory"
val directory = File(tableDirectory).createDirectory()
- val ret0 = exectractLineage(s"""
- |INSERT OVERWRITE DIRECTORY
'$directory.path'
- |USING parquet
- |SELECT * FROM
test_db0.test_table_part0""".stripMargin)
+ val ret0 = extractLineage(s"""
+ |INSERT OVERWRITE DIRECTORY '$directory.path'
+ |USING parquet
+ |SELECT * FROM
test_db0.test_table_part0""".stripMargin)
assert(ret0 == Lineage(
List("test_db0.test_table_part0"),
List(s"""`$directory.path`"""),
@@ -453,7 +452,7 @@ class SparkSQLLineageParserHelperSuite extends
KyuubiFunSuite
withTable(tableName) { _ =>
spark.sql(s"CREATE TABLE $tableName (a int, b string) USING hive")
val ret0 =
- exectractLineage(
+ extractLineage(
s"insert into table $tableName select * from test_db0.test_table0")
assert(ret0 == Lineage(
@@ -467,7 +466,7 @@ class SparkSQLLineageParserHelperSuite extends
KyuubiFunSuite
}
test("columns lineage extract - logical relation sql") {
- val ret0 = exectractLineage("select key, value from test_db0.test_table0")
+ val ret0 = extractLineage("select key, value from test_db0.test_table0")
assert(ret0 == Lineage(
List("test_db0.test_table0"),
List(),
@@ -475,7 +474,7 @@ class SparkSQLLineageParserHelperSuite extends
KyuubiFunSuite
("key", Set("test_db0.test_table0.key")),
("value", Set("test_db0.test_table0.value")))))
- val ret1 = exectractLineage("select * from test_db0.test_table_part0")
+ val ret1 = extractLineage("select * from test_db0.test_table_part0")
assert(ret1 == Lineage(
List("test_db0.test_table_part0"),
List(),
@@ -487,7 +486,7 @@ class SparkSQLLineageParserHelperSuite extends
KyuubiFunSuite
}
test("columns lineage extract - not generate lineage sql") {
- val ret0 = exectractLineage("create table test_table1(a string, b string,
c string)")
+ val ret0 = extractLineage("create table test_table1(a string, b string, c
string)")
assert(ret0 == Lineage(List[String](), List[String](), List[(String,
Set[String])]()))
}
@@ -500,14 +499,14 @@ class SparkSQLLineageParserHelperSuite extends
KyuubiFunSuite
ddls.split("\n").filter(_.nonEmpty).foreach(spark.sql(_).collect())
withTable("v2_catalog.db.tb") { _ =>
val sql0 = "select col1 from v2_catalog.db.tb"
- val ret0 = exectractLineage(sql0)
+ val ret0 = extractLineage(sql0)
assert(ret0 == Lineage(
List("v2_catalog.db.tb"),
List(),
List("col1" -> Set("v2_catalog.db.tb.col1"))))
val sql1 = "select col1, hash(hash(col1)) as col2 from v2_catalog.db.tb"
- val ret1 = exectractLineage(sql1)
+ val ret1 = extractLineage(sql1)
assert(ret1 == Lineage(
List("v2_catalog.db.tb"),
List(),
@@ -515,7 +514,7 @@ class SparkSQLLineageParserHelperSuite extends
KyuubiFunSuite
val sql2 =
"select col1, case col1 when '1' then 's1' else col1 end col2 from
v2_catalog.db.tb"
- val ret2 = exectractLineage(sql2)
+ val ret2 = extractLineage(sql2)
assert(ret2 == Lineage(
List("v2_catalog.db.tb"),
List(),
@@ -524,7 +523,7 @@ class SparkSQLLineageParserHelperSuite extends
KyuubiFunSuite
val sql3 =
"select col1 as col2, 'col2' as col2, 'col2', first(col3) as col2 " +
"from v2_catalog.db.tb group by col1"
- val ret3 = exectractLineage(sql3)
+ val ret3 = extractLineage(sql3)
assert(ret3 == Lineage(
List("v2_catalog.db.tb"),
List[String](),
@@ -537,7 +536,7 @@ class SparkSQLLineageParserHelperSuite extends
KyuubiFunSuite
val sql4 =
"select col1 as col2, sum(hash(col1) + hash(hash(col1))) " +
"from v2_catalog.db.tb group by col1"
- val ret4 = exectractLineage(sql4)
+ val ret4 = extractLineage(sql4)
assert(ret4 == Lineage(
List("v2_catalog.db.tb"),
List(),
@@ -553,7 +552,7 @@ class SparkSQLLineageParserHelperSuite extends
KyuubiFunSuite
| on t1.col2 = t2.col3
| group by 1
|""".stripMargin
- val ret5 = exectractLineage(sql5)
+ val ret5 = extractLineage(sql5)
assert(ret5 == Lineage(
List("v2_catalog.db.tb"),
List(),
@@ -596,9 +595,9 @@ class SparkSQLLineageParserHelperSuite extends
KyuubiFunSuite
"cnt" -> Set("default.tmp1.tmp1_0"),
"tmp1_1" -> Set("default.tmp1.tmp1_1")))
- val ret0 = exectractLineage(sql0)
+ val ret0 = extractLineage(sql0)
assert(ret0 == sql0ExpectResult)
- val ret1 = exectractLineage(sql1)
+ val ret1 = extractLineage(sql1)
assert(ret1 == sql1ExpectResult)
}
}
@@ -658,7 +657,7 @@ class SparkSQLLineageParserHelperSuite extends
KyuubiFunSuite
|FROM goods_cat_new
|LIMIT 10""".stripMargin
- val ret0 = exectractLineage(sql0)
+ val ret0 = extractLineage(sql0)
assert(ret0 == Lineage(
List(
"test_db.goods_detail0",
@@ -692,7 +691,7 @@ class SparkSQLLineageParserHelperSuite extends
KyuubiFunSuite
|on t1.col1 = t2.col1
|""".stripMargin
- val ret0 = exectractLineage(sql0)
+ val ret0 = extractLineage(sql0)
assert(
ret0 == Lineage(
List("v2_catalog.db.tb1", "v2_catalog.db.tb2"),
@@ -727,7 +726,7 @@ class SparkSQLLineageParserHelperSuite extends
KyuubiFunSuite
|select a, b, c as c from test_db.test_table1
|) a
|""".stripMargin
- val ret0 = exectractLineage(sql0)
+ val ret0 = extractLineage(sql0)
assert(ret0 == Lineage(
List("test_db.test_table0", "test_db.test_table1"),
List(),
@@ -768,7 +767,7 @@ class SparkSQLLineageParserHelperSuite extends
KyuubiFunSuite
|GROUP BY
|stat_date, channel_id, sub_channel_id, user_type, country_name
|""".stripMargin
- val ret0 = exectractLineage(sql0)
+ val ret0 = extractLineage(sql0)
assert(ret0 == Lineage(
List("test_db.test_order_item"),
List(),
@@ -806,7 +805,7 @@ class SparkSQLLineageParserHelperSuite extends
KyuubiFunSuite
|) a
|GROUP BY a.channel_id, a.sub_channel_id, a.country_name
|""".stripMargin
- val ret1 = exectractLineage(sql1)
+ val ret1 = extractLineage(sql1)
assert(ret1 == Lineage(
List("test_db.test_order_item", "test_db.test_p0_order_item"),
List(),
@@ -832,7 +831,7 @@ class SparkSQLLineageParserHelperSuite extends
KyuubiFunSuite
test("columns lineage extract - agg sql") {
val sql0 = """select key as a, count(*) as b, 1 as c from
test_db0.test_table0 group by key"""
- val ret0 = exectractLineage(sql0)
+ val ret0 = extractLineage(sql0)
assert(ret0 == Lineage(
List("test_db0.test_table0"),
List(),
@@ -842,7 +841,7 @@ class SparkSQLLineageParserHelperSuite extends
KyuubiFunSuite
("c", Set()))))
val sql1 = """select count(*) as a, 1 as b from test_db0.test_table0"""
- val ret1 = exectractLineage(sql1)
+ val ret1 = extractLineage(sql1)
assert(ret1 == Lineage(
List("test_db0.test_table0"),
List(),
@@ -851,7 +850,7 @@ class SparkSQLLineageParserHelperSuite extends
KyuubiFunSuite
("b", Set()))))
val sql2 = """select every(key == 1) as a, 1 as b from
test_db0.test_table0"""
- val ret2 = exectractLineage(sql2)
+ val ret2 = extractLineage(sql2)
assert(ret2 == Lineage(
List("test_db0.test_table0"),
List(),
@@ -860,7 +859,7 @@ class SparkSQLLineageParserHelperSuite extends
KyuubiFunSuite
("b", Set()))))
val sql3 = """select count(*) as a, 1 as b from test_db0.test_table0"""
- val ret3 = exectractLineage(sql3)
+ val ret3 = extractLineage(sql3)
assert(ret3 == Lineage(
List("test_db0.test_table0"),
List(),
@@ -869,7 +868,7 @@ class SparkSQLLineageParserHelperSuite extends
KyuubiFunSuite
("b", Set()))))
val sql4 = """select first(key) as a, 1 as b from test_db0.test_table0"""
- val ret4 = exectractLineage(sql4)
+ val ret4 = extractLineage(sql4)
assert(ret4 == Lineage(
List("test_db0.test_table0"),
List(),
@@ -878,7 +877,7 @@ class SparkSQLLineageParserHelperSuite extends
KyuubiFunSuite
("b", Set()))))
val sql5 = """select avg(key) as a, 1 as b from test_db0.test_table0"""
- val ret5 = exectractLineage(sql5)
+ val ret5 = extractLineage(sql5)
assert(ret5 == Lineage(
List("test_db0.test_table0"),
List(),
@@ -889,7 +888,7 @@ class SparkSQLLineageParserHelperSuite extends
KyuubiFunSuite
val sql6 =
"""select count(value) + sum(key) as a,
| 1 as b from test_db0.test_table0""".stripMargin
- val ret6 = exectractLineage(sql6)
+ val ret6 = extractLineage(sql6)
assert(ret6 == Lineage(
List("test_db0.test_table0"),
List(),
@@ -898,7 +897,7 @@ class SparkSQLLineageParserHelperSuite extends
KyuubiFunSuite
("b", Set()))))
val sql7 = """select count(*) + sum(key) as a, 1 as b from
test_db0.test_table0"""
- val ret7 = exectractLineage(sql7)
+ val ret7 = extractLineage(sql7)
assert(ret7 == Lineage(
List("test_db0.test_table0"),
List(),
@@ -921,7 +920,7 @@ class SparkSQLLineageParserHelperSuite extends
KyuubiFunSuite
"""
|select b.a as aa, t0_cached.b0 as bb from t0_cached join table1 b
on b.a = t0_cached.a0
|""".stripMargin
- val ret0 = exectractLineage(sql0)
+ val ret0 = extractLineage(sql0)
assert(ret0 == Lineage(
List("default.table1", "default.table0"),
List(),
@@ -956,7 +955,7 @@ class SparkSQLLineageParserHelperSuite extends
KyuubiFunSuite
"""
|select a as aa, bb, cc from (select b as bb, c as cc from table1)
t0, table0
|""".stripMargin
- val ret0 = exectractLineage(sql0)
+ val ret0 = extractLineage(sql0)
assert(ret0 == Lineage(
List("default.table0", "default.table1"),
List(),
@@ -969,7 +968,7 @@ class SparkSQLLineageParserHelperSuite extends
KyuubiFunSuite
"""
|select (select a from table1) as aa, b as bb from table1
|""".stripMargin
- val ret1 = exectractLineage(sql1)
+ val ret1 = extractLineage(sql1)
assert(ret1 == Lineage(
List("default.table1"),
List(),
@@ -981,7 +980,7 @@ class SparkSQLLineageParserHelperSuite extends
KyuubiFunSuite
"""
|select (select count(*) from table0) as aa, b as bb from table1
|""".stripMargin
- val ret2 = exectractLineage(sql2)
+ val ret2 = extractLineage(sql2)
assert(ret2 == Lineage(
List("default.table0", "default.table1"),
List(),
@@ -994,7 +993,7 @@ class SparkSQLLineageParserHelperSuite extends
KyuubiFunSuite
"""
|select * from table0 where table0.a in (select a from table1)
|""".stripMargin
- val ret3 = exectractLineage(sql3)
+ val ret3 = extractLineage(sql3)
assert(ret3 == Lineage(
List("default.table0"),
List(),
@@ -1008,7 +1007,7 @@ class SparkSQLLineageParserHelperSuite extends
KyuubiFunSuite
"""
|select * from table0 where exists (select * from table1 where
table0.c = table1.c)
|""".stripMargin
- val ret4 = exectractLineage(sql4)
+ val ret4 = extractLineage(sql4)
assert(ret4 == Lineage(
List("default.table0"),
List(),
@@ -1021,7 +1020,7 @@ class SparkSQLLineageParserHelperSuite extends
KyuubiFunSuite
"""
|select * from table0 where exists (select * from table1 where c =
"odone")
|""".stripMargin
- val ret5 = exectractLineage(sql5)
+ val ret5 = extractLineage(sql5)
assert(ret5 == Lineage(
List("default.table0"),
List(),
@@ -1034,7 +1033,7 @@ class SparkSQLLineageParserHelperSuite extends
KyuubiFunSuite
"""
|select * from table0 where not exists (select * from table1 where c
= "odone")
|""".stripMargin
- val ret6 = exectractLineage(sql6)
+ val ret6 = extractLineage(sql6)
assert(ret6 == Lineage(
List("default.table0"),
List(),
@@ -1047,7 +1046,7 @@ class SparkSQLLineageParserHelperSuite extends
KyuubiFunSuite
"""
|select * from table0 where table0.a not in (select a from table1)
|""".stripMargin
- val ret7 = exectractLineage(sql7)
+ val ret7 = extractLineage(sql7)
assert(ret7 == Lineage(
List("default.table0"),
List(),
@@ -1060,7 +1059,7 @@ class SparkSQLLineageParserHelperSuite extends
KyuubiFunSuite
"""
|select (select a from table1) + 1, b as bb from table1
|""".stripMargin
- val ret8 = exectractLineage(sql8)
+ val ret8 = extractLineage(sql8)
assert(ret8 == Lineage(
List("default.table1"),
List(),
@@ -1072,7 +1071,7 @@ class SparkSQLLineageParserHelperSuite extends
KyuubiFunSuite
"""
|select (select a from table1 limit 1) + 1 as aa, b as bb from table1
|""".stripMargin
- val ret9 = exectractLineage(sql9)
+ val ret9 = extractLineage(sql9)
assert(ret9 == Lineage(
List("default.table1"),
List(),
@@ -1085,7 +1084,7 @@ class SparkSQLLineageParserHelperSuite extends
KyuubiFunSuite
|select (select a from table1 limit 1) + (select a from table0 limit
1) + 1 as aa,
| b as bb from table1
|""".stripMargin
- val ret10 = exectractLineage(sql10)
+ val ret10 = extractLineage(sql10)
assert(ret10 == Lineage(
List("default.table1", "default.table0"),
List(),
@@ -1098,7 +1097,7 @@ class SparkSQLLineageParserHelperSuite extends
KyuubiFunSuite
|select tmp.a, b from (select * from table1) tmp;
|""".stripMargin
- val ret11 = exectractLineage(sql11)
+ val ret11 = extractLineage(sql11)
assert(ret11 == Lineage(
List("default.table1"),
List(),
@@ -1115,7 +1114,7 @@ class SparkSQLLineageParserHelperSuite extends
KyuubiFunSuite
spark.sql("CREATE TABLE v2_catalog.db.t1 (a string, b string, c string)")
spark.sql("CREATE TABLE v2_catalog.db.t2 (a string, b string, c string)")
val ret0 =
- exectractLineage(
+ extractLineage(
s"insert into table t1 select a," +
s"concat_ws('/', collect_set(b))," +
s"count(distinct(b)) * count(distinct(c))" +
@@ -1129,7 +1128,7 @@ class SparkSQLLineageParserHelperSuite extends
KyuubiFunSuite
("default.t1.c", Set("default.t2.b", "default.t2.c")))))
val ret1 =
- exectractLineage(
+ extractLineage(
s"insert into table v2_catalog.db.t1 select a," +
s"concat_ws('/', collect_set(b))," +
s"count(distinct(b)) * count(distinct(c))" +
@@ -1143,7 +1142,7 @@ class SparkSQLLineageParserHelperSuite extends
KyuubiFunSuite
("v2_catalog.db.t1.c", Set("v2_catalog.db.t2.b",
"v2_catalog.db.t2.c")))))
val ret2 =
- exectractLineage(
+ extractLineage(
s"insert into table v2_catalog.db.t1 select a," +
s"count(distinct(b+c))," +
s"count(distinct(b)) * count(distinct(c))" +
@@ -1163,7 +1162,7 @@ class SparkSQLLineageParserHelperSuite extends
KyuubiFunSuite
spark.sql("CREATE TABLE t1 (a string, b string, c string) USING hive")
spark.sql("CREATE TABLE t2 (a string, b string, c string, d string)
USING hive")
val ret0 =
- exectractLineage(
+ extractLineage(
s"insert into table t1 select a,b,GROUPING__ID " +
s"from t2 group by a,b,c,d grouping sets ((a,b,c), (a,b,d))")
assert(ret0 == Lineage(
@@ -1185,7 +1184,7 @@ class SparkSQLLineageParserHelperSuite extends
KyuubiFunSuite
s"cache table c1 select * from (" +
s"select a, b, row_number() over (partition by a order by b asc )
rank from t2)" +
s" where rank=1")
- val ret0 = exectractLineage("insert overwrite table t1 select a, b from
c1")
+ val ret0 = extractLineage("insert overwrite table t1 select a, b from
c1")
assert(ret0 == Lineage(
List("default.t2"),
List("default.t1"),
@@ -1193,7 +1192,7 @@ class SparkSQLLineageParserHelperSuite extends
KyuubiFunSuite
("default.t1.a", Set("default.t2.a")),
("default.t1.b", Set("default.t2.b")))))
- val ret1 = exectractLineage("insert overwrite table t1 select a, rank
from c1")
+ val ret1 = extractLineage("insert overwrite table t1 select a, rank from
c1")
assert(ret1 == Lineage(
List("default.t2"),
List("default.t1"),
@@ -1205,7 +1204,7 @@ class SparkSQLLineageParserHelperSuite extends
KyuubiFunSuite
s"cache table c2 select * from (" +
s"select b, a, row_number() over (partition by a order by b asc )
rank from t2)" +
s" where rank=1")
- val ret2 = exectractLineage("insert overwrite table t1 select a, b from
c2")
+ val ret2 = extractLineage("insert overwrite table t1 select a, b from
c2")
assert(ret2 == Lineage(
List("default.t2"),
List("default.t1"),
@@ -1217,7 +1216,7 @@ class SparkSQLLineageParserHelperSuite extends
KyuubiFunSuite
s"cache table c3 select * from (" +
s"select a as aa, b as bb, row_number() over (partition by a order
by b asc ) rank" +
s" from t2) where rank=1")
- val ret3 = exectractLineage("insert overwrite table t1 select aa, bb
from c3")
+ val ret3 = extractLineage("insert overwrite table t1 select aa, bb from
c3")
assert(ret3 == Lineage(
List("default.t2"),
List("default.t1"),
@@ -1231,7 +1230,7 @@ class SparkSQLLineageParserHelperSuite extends
KyuubiFunSuite
withTable("t1", "t2") { _ =>
spark.sql("CREATE TABLE t1 (a string, b string, c string) USING hive")
spark.sql("CREATE TABLE t2 (a string, b string, c string) USING hive")
- val ret0 = exectractLineage("insert into t1 select 1,2,(select
count(distinct" +
+ val ret0 = extractLineage("insert into t1 select 1,2,(select
count(distinct" +
" ifnull(get_json_object(a, '$.b.imei'), get_json_object(a,
'$.b.android_id'))) from t2)")
assert(ret0 == Lineage(
@@ -1250,7 +1249,7 @@ class SparkSQLLineageParserHelperSuite extends
KyuubiFunSuite
withView("t2") { _ =>
spark.sql("CREATE VIEW t2 as select * from t1")
val ret0 =
- exectractLineage(
+ extractLineage(
s"create or replace view view_tst comment 'view'" +
s" as select a as k,b" +
s" from t2" +
@@ -1272,7 +1271,7 @@ class SparkSQLLineageParserHelperSuite extends
KyuubiFunSuite
withView("t2") { _ =>
spark.sql("CREATE VIEW t2 as select * from t1")
val ret0 =
- exectractLineage(
+ extractLineage(
s"select a as k, b" +
s" from t2" +
s" where a in ('HELLO') and c = 'HELLO'")
@@ -1291,7 +1290,7 @@ class SparkSQLLineageParserHelperSuite extends
KyuubiFunSuite
spark.sql("CREATE TABLE t1 (a string, b string) USING hive")
spark.sql("CREATE TABLE t2 (a string, b string) USING hive")
spark.sql("CREATE TABLE t3 (a string, b string) USING hive")
- val ret0 = exectractLineage("from (select a,b from t1)" +
+ val ret0 = extractLineage("from (select a,b from t1)" +
" insert overwrite table t2 select a,b where a=1" +
" insert overwrite table t3 select a,b where b=1")
assert(ret0 == Lineage(
@@ -1310,7 +1309,7 @@ class SparkSQLLineageParserHelperSuite extends
KyuubiFunSuite
spark.sql("CREATE TABLE t1 (a string, b string, c string, d string)
USING hive")
spark.sql("CREATE TABLE t2 (a string, b string, c string, d string)
USING hive")
- val ret0 = exectractLineage("insert into t1 select 1, t2.b, cc.action,
t2.d " +
+ val ret0 = extractLineage("insert into t1 select 1, t2.b, cc.action,
t2.d " +
"from t2 lateral view explode(split(c,'\\},\\{')) cc as action")
assert(ret0 == Lineage(
List("default.t2"),
@@ -1321,7 +1320,7 @@ class SparkSQLLineageParserHelperSuite extends
KyuubiFunSuite
("default.t1.c", Set("default.t2.c")),
("default.t1.d", Set("default.t2.d")))))
- val ret1 = exectractLineage("insert into t1 select 1, t2.b, cc.action0,
dd.action1 " +
+ val ret1 = extractLineage("insert into t1 select 1, t2.b, cc.action0,
dd.action1 " +
"from t2 " +
"lateral view explode(split(c,'\\},\\{')) cc as action0 " +
"lateral view explode(split(d,'\\},\\{')) dd as action1")
@@ -1334,7 +1333,7 @@ class SparkSQLLineageParserHelperSuite extends
KyuubiFunSuite
("default.t1.c", Set("default.t2.c")),
("default.t1.d", Set("default.t2.d")))))
- val ret2 = exectractLineage("insert into t1 select 1, t2.b, dd.pos,
dd.action1 " +
+ val ret2 = extractLineage("insert into t1 select 1, t2.b, dd.pos,
dd.action1 " +
"from t2 " +
"lateral view posexplode(split(d,'\\},\\{')) dd as pos, action1")
assert(ret2 == Lineage(
@@ -1348,14 +1347,14 @@ class SparkSQLLineageParserHelperSuite extends
KyuubiFunSuite
}
}
- private def exectractLineageWithoutExecuting(sql: String): Lineage = {
+ private def extractLineageWithoutExecuting(sql: String): Lineage = {
val parsed = spark.sessionState.sqlParser.parsePlan(sql)
val analyzed = spark.sessionState.analyzer.execute(parsed)
spark.sessionState.analyzer.checkAnalysis(analyzed)
SparkSQLLineageParseHelper(spark).transformToLineage(0, analyzed).get
}
- private def exectractLineage(sql: String): Lineage = {
+ private def extractLineage(sql: String): Lineage = {
val parsed = spark.sessionState.sqlParser.parsePlan(sql)
val qe = spark.sessionState.executePlan(parsed)
val analyzed = qe.analyzed
