maropu commented on a change in pull request #30525:
URL: https://github.com/apache/spark/pull/30525#discussion_r532041461
##########
File path:
sql/hive/src/test/scala/org/apache/spark/sql/hive/client/HivePartitionFilteringSuite.scala
##########
@@ -108,119 +119,147 @@ class HivePartitionFilteringSuite(version: String)
// Should return all partitions where <=> is not supported
testMetastorePartitionFiltering(
attr("ds") <=> 20170101,
- 20170101 to 20170103,
- 0 to 4,
- "aa" :: "ab" :: "ba" :: "bb" :: Nil)
+ dsValue,
+ hValue,
+ chunkValue,
+ dateValue,
+ dateStrValue)
}
test("getPartitionsByFilter: ds=20170101") {
testMetastorePartitionFiltering(
attr("ds") === 20170101,
20170101 to 20170101,
- 0 to 4,
- "aa" :: "ab" :: "ba" :: "bb" :: Nil)
+ hValue,
+ chunkValue,
+ dateValue,
+ dateStrValue)
}
test("getPartitionsByFilter: ds=(20170101 + 1) and h=0") {
// Should return all partitions where h=0 because getPartitionsByFilter
does not support
// comparisons to non-literal values
testMetastorePartitionFiltering(
attr("ds") === (Literal(20170101) + 1) && attr("h") === 0,
- 20170101 to 20170103,
+ dsValue,
0 to 0,
- "aa" :: "ab" :: "ba" :: "bb" :: Nil)
+ chunkValue,
+ dateValue,
+ dateStrValue)
}
test("getPartitionsByFilter: chunk='aa'") {
testMetastorePartitionFiltering(
attr("chunk") === "aa",
- 20170101 to 20170103,
- 0 to 4,
- "aa" :: Nil)
+ dsValue,
+ hValue,
+ "aa" :: Nil,
+ dateValue,
+ dateStrValue)
}
test("getPartitionsByFilter: cast(chunk as int)=1 (not a valid partition
predicate)") {
testMetastorePartitionFiltering(
attr("chunk").cast(IntegerType) === 1,
- 20170101 to 20170103,
- 0 to 4,
- "aa" :: "ab" :: "ba" :: "bb" :: Nil)
+ dsValue,
+ hValue,
+ chunkValue,
+ dateValue,
+ dateStrValue)
}
test("getPartitionsByFilter: cast(chunk as boolean)=true (not a valid
partition predicate)") {
testMetastorePartitionFiltering(
attr("chunk").cast(BooleanType) === true,
- 20170101 to 20170103,
- 0 to 4,
- "aa" :: "ab" :: "ba" :: "bb" :: Nil)
+ dsValue,
+ hValue,
+ chunkValue,
+ dateValue,
+ dateStrValue)
}
test("getPartitionsByFilter: 20170101=ds") {
testMetastorePartitionFiltering(
Literal(20170101) === attr("ds"),
20170101 to 20170101,
- 0 to 4,
- "aa" :: "ab" :: "ba" :: "bb" :: Nil)
+ hValue,
+ chunkValue,
+ dateValue,
+ dateStrValue)
}
test("getPartitionsByFilter: ds=20170101 and h=2") {
testMetastorePartitionFiltering(
attr("ds") === 20170101 && attr("h") === 2,
20170101 to 20170101,
2 to 2,
- "aa" :: "ab" :: "ba" :: "bb" :: Nil)
+ chunkValue,
+ dateValue,
+ dateStrValue)
}
test("getPartitionsByFilter: cast(ds as long)=20170101L and h=2") {
testMetastorePartitionFiltering(
attr("ds").cast(LongType) === 20170101L && attr("h") === 2,
20170101 to 20170101,
2 to 2,
- "aa" :: "ab" :: "ba" :: "bb" :: Nil)
+ chunkValue,
+ dateValue,
+ dateStrValue)
}
test("getPartitionsByFilter: ds=20170101 or ds=20170102") {
testMetastorePartitionFiltering(
attr("ds") === 20170101 || attr("ds") === 20170102,
20170101 to 20170102,
- 0 to 4,
- "aa" :: "ab" :: "ba" :: "bb" :: Nil)
+ hValue,
+ chunkValue,
+ dateValue,
+ dateStrValue)
}
test("getPartitionsByFilter: ds in (20170102, 20170103) (using IN
expression)") {
testMetastorePartitionFiltering(
attr("ds").in(20170102, 20170103),
20170102 to 20170103,
- 0 to 4,
- "aa" :: "ab" :: "ba" :: "bb" :: Nil)
+ hValue,
+ chunkValue,
+ dateValue,
+ dateStrValue)
}
- test("getPartitionsByFilter: cast(ds as long) in (20170102L, 20170103L)
(using IN expression)") {
+ test("getPartitionsByFilter: cast(ds as long) in (20170102L, 20170103L)
(using IN expression)") {
testMetastorePartitionFiltering(
attr("ds").cast(LongType).in(20170102L, 20170103L),
20170102 to 20170103,
- 0 to 4,
- "aa" :: "ab" :: "ba" :: "bb" :: Nil)
+ hValue,
+ chunkValue,
+ dateValue,
+ dateStrValue)
}
test("getPartitionsByFilter: ds in (20170102, 20170103) (using INSET
expression)") {
testMetastorePartitionFiltering(
attr("ds").in(20170102, 20170103),
20170102 to 20170103,
- 0 to 4,
- "aa" :: "ab" :: "ba" :: "bb" :: Nil, {
+ hValue,
+ chunkValue,
+ dateValue,
+ dateStrValue, {
case expr @ In(v, list) if expr.inSetConvertible =>
InSet(v, list.map(_.eval(EmptyRow)).toSet)
})
}
- test("getPartitionsByFilter: cast(ds as long) in (20170102L, 20170103L)
(using INSET expression)")
+ test("getPartitionsByFilter: cast(ds as long) in (20170102L, 20170103L)
(using INSET expression")
Review comment:
nit: wrong indent
##########
File path:
sql/hive/src/test/scala/org/apache/spark/sql/hive/client/HivePartitionFilteringSuite.scala
##########
@@ -108,119 +119,147 @@ class HivePartitionFilteringSuite(version: String)
// Should return all partitions where <=> is not supported
testMetastorePartitionFiltering(
attr("ds") <=> 20170101,
- 20170101 to 20170103,
- 0 to 4,
- "aa" :: "ab" :: "ba" :: "bb" :: Nil)
+ dsValue,
+ hValue,
+ chunkValue,
+ dateValue,
+ dateStrValue)
}
test("getPartitionsByFilter: ds=20170101") {
testMetastorePartitionFiltering(
attr("ds") === 20170101,
20170101 to 20170101,
- 0 to 4,
- "aa" :: "ab" :: "ba" :: "bb" :: Nil)
+ hValue,
+ chunkValue,
+ dateValue,
+ dateStrValue)
}
test("getPartitionsByFilter: ds=(20170101 + 1) and h=0") {
// Should return all partitions where h=0 because getPartitionsByFilter
does not support
// comparisons to non-literal values
testMetastorePartitionFiltering(
attr("ds") === (Literal(20170101) + 1) && attr("h") === 0,
- 20170101 to 20170103,
+ dsValue,
0 to 0,
- "aa" :: "ab" :: "ba" :: "bb" :: Nil)
+ chunkValue,
+ dateValue,
+ dateStrValue)
}
test("getPartitionsByFilter: chunk='aa'") {
testMetastorePartitionFiltering(
attr("chunk") === "aa",
- 20170101 to 20170103,
- 0 to 4,
- "aa" :: Nil)
+ dsValue,
+ hValue,
+ "aa" :: Nil,
+ dateValue,
+ dateStrValue)
}
test("getPartitionsByFilter: cast(chunk as int)=1 (not a valid partition
predicate)") {
testMetastorePartitionFiltering(
attr("chunk").cast(IntegerType) === 1,
- 20170101 to 20170103,
- 0 to 4,
- "aa" :: "ab" :: "ba" :: "bb" :: Nil)
+ dsValue,
+ hValue,
+ chunkValue,
+ dateValue,
+ dateStrValue)
}
test("getPartitionsByFilter: cast(chunk as boolean)=true (not a valid
partition predicate)") {
testMetastorePartitionFiltering(
attr("chunk").cast(BooleanType) === true,
- 20170101 to 20170103,
- 0 to 4,
- "aa" :: "ab" :: "ba" :: "bb" :: Nil)
+ dsValue,
+ hValue,
+ chunkValue,
+ dateValue,
+ dateStrValue)
}
test("getPartitionsByFilter: 20170101=ds") {
testMetastorePartitionFiltering(
Literal(20170101) === attr("ds"),
20170101 to 20170101,
- 0 to 4,
- "aa" :: "ab" :: "ba" :: "bb" :: Nil)
+ hValue,
+ chunkValue,
+ dateValue,
+ dateStrValue)
}
test("getPartitionsByFilter: ds=20170101 and h=2") {
testMetastorePartitionFiltering(
attr("ds") === 20170101 && attr("h") === 2,
20170101 to 20170101,
2 to 2,
- "aa" :: "ab" :: "ba" :: "bb" :: Nil)
+ chunkValue,
+ dateValue,
+ dateStrValue)
}
test("getPartitionsByFilter: cast(ds as long)=20170101L and h=2") {
testMetastorePartitionFiltering(
attr("ds").cast(LongType) === 20170101L && attr("h") === 2,
20170101 to 20170101,
2 to 2,
- "aa" :: "ab" :: "ba" :: "bb" :: Nil)
+ chunkValue,
+ dateValue,
+ dateStrValue)
}
test("getPartitionsByFilter: ds=20170101 or ds=20170102") {
testMetastorePartitionFiltering(
attr("ds") === 20170101 || attr("ds") === 20170102,
20170101 to 20170102,
- 0 to 4,
- "aa" :: "ab" :: "ba" :: "bb" :: Nil)
+ hValue,
+ chunkValue,
+ dateValue,
+ dateStrValue)
}
test("getPartitionsByFilter: ds in (20170102, 20170103) (using IN
expression)") {
testMetastorePartitionFiltering(
attr("ds").in(20170102, 20170103),
20170102 to 20170103,
- 0 to 4,
- "aa" :: "ab" :: "ba" :: "bb" :: Nil)
+ hValue,
+ chunkValue,
+ dateValue,
+ dateStrValue)
}
- test("getPartitionsByFilter: cast(ds as long) in (20170102L, 20170103L)
(using IN expression)") {
+ test("getPartitionsByFilter: cast(ds as long) in (20170102L, 20170103L)
(using IN expression)") {
Review comment:
nit: wrong indent
##########
File path:
sql/hive/src/test/scala/org/apache/spark/sql/hive/client/HivePartitionFilteringSuite.scala
##########
@@ -272,93 +315,105 @@ class HivePartitionFilteringSuite(version: String)
test("getPartitionsByFilter: chunk contains bb") {
testMetastorePartitionFiltering(
attr("chunk").contains("bb"),
- (20170101 to 20170103, 0 to 4, Seq("bb")) :: Nil)
+ dsValue,
+ hValue,
+ Seq("bb"),
+ dateValue,
+ dateStrValue)
}
test("getPartitionsByFilter: chunk startsWith b") {
testMetastorePartitionFiltering(
attr("chunk").startsWith("b"),
- (20170101 to 20170103, 0 to 4, Seq("ba", "bb")) :: Nil)
+ dsValue,
+ hValue,
+ Seq("ba", "bb"),
+ dateValue,
+ dateStrValue)
}
test("getPartitionsByFilter: chunk endsWith b") {
testMetastorePartitionFiltering(
attr("chunk").endsWith("b"),
- (20170101 to 20170103, 0 to 4, Seq("ab", "bb")) :: Nil)
+ dsValue,
+ hValue,
+ Seq("ab", "bb"),
+ dateValue,
+ dateStrValue)
}
test("getPartitionsByFilter: chunk in ('ab', 'ba') and ((cast(ds as
string)>'20170102')") {
- val day = (20170101 to 20170103, 0 to 4, Seq("ab", "ba"))
testMetastorePartitionFiltering(
attr("chunk").in("ab", "ba") && (attr("ds").cast(StringType) >
"20170102"),
- day :: Nil)
+ dsValue,
+ hValue,
+ Seq("ab", "ba"),
+ dateValue,
+ dateStrValue)
}
- test("getPartitionsByFilter: date type pruning by metastore") {
Review comment:
You removed this test?
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
