Github user raghavgautam commented on a diff in the pull request:
https://github.com/apache/spark/pull/22414#discussion_r223853069
--- Diff:
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/TimeWindow.scala
---
@@ -137,16 +139,44 @@ object TimeWindow {
"an integer, long or string literal.")
}
+ private def parseWindowDuration(windowDuration: Expression): Long = {
+ val windowDurationMicroSec = parseExpression(windowDuration)
+ require(windowDurationMicroSec > 0, "The window duration must be " +
+ s"a positive integer, long or string literal, found:
$windowDuration")
+ windowDurationMicroSec
+ }
+
+ private def parseSlideDuration(slideDuration: Expression): Long = {
+ val slideDurationMicroSec = parseExpression(slideDuration)
+ require(slideDurationMicroSec > 0, "The slide duration must be " +
+ s"a positive integer, long or string literal, found: $slideDuration")
+ slideDurationMicroSec
+ }
+
def apply(
timeColumn: Expression,
windowDuration: String,
slideDuration: String,
startTime: String): TimeWindow = {
+ val windowDurationMicroSec = getIntervalInMicroSeconds(windowDuration)
+ val slideDurationMicroSec = getIntervalInMicroSeconds(slideDuration)
+ checkWindowAndSlideDuration(windowDurationMicroSec,
slideDurationMicroSec,
+ windowDuration, slideDuration)
TimeWindow(timeColumn,
- getIntervalInMicroSeconds(windowDuration),
- getIntervalInMicroSeconds(slideDuration),
+ windowDurationMicroSec,
+ slideDurationMicroSec,
getIntervalInMicroSeconds(startTime))
}
+
+ private def checkWindowAndSlideDuration(windowDurationMicroSec: Long,
+ slideDurationMicroSec: Long,
+ windowDuration: Any,
--- End diff --
@srowen The method has been called in two places:
https://github.com/apache/spark/blob/3ba3252c9bf56f9cde04c1aa757e9f50826abbd2/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/TimeWindow.scala#L38
and
https://github.com/apache/spark/blob/3ba3252c9bf56f9cde04c1aa757e9f50826abbd2/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/TimeWindow.scala#L163
windowDuration, slideDuration argument that is passed is Long in one case
and String in another, that is why I have used type Any.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
