HeartSaVioR commented on code in PR #48853:
URL: https://github.com/apache/spark/pull/48853#discussion_r1858599307
##########
sql/core/src/test/scala/org/apache/spark/sql/streaming/TransformWithStateTTLTest.scala:
##########
@@ -143,43 +143,54 @@ abstract class TransformWithStateTTLTest
AddData(inputStream, InputEvent("k1", "put", 1)),
// advance clock to trigger processing
AdvanceManualClock(1 * 1000),
+ // In the primary index, we should have that k1 -> [1].
+ // The TTL index has (6100, k1) -> empty. The min index has k1 ->
61000.
CheckNewAnswer(),
+
// get this state, and make sure we get unexpired value
AddData(inputStream, InputEvent("k1", "get", -1)),
AdvanceManualClock(1 * 1000),
CheckNewAnswer(OutputEvent("k1", 1, isTTLValue = false, -1)),
+
// ensure ttl values were added correctly
AddData(inputStream, InputEvent("k1", "get_ttl_value_from_state", -1)),
AdvanceManualClock(1 * 1000),
CheckNewAnswer(OutputEvent("k1", 1, isTTLValue = true, 61000)),
+
AddData(inputStream, InputEvent("k1", "get_values_in_ttl_state", -1)),
AdvanceManualClock(1 * 1000),
CheckNewAnswer(OutputEvent("k1", -1, isTTLValue = true, 61000)),
+
// advance clock and update expiration time
AdvanceManualClock(30 * 1000),
AddData(inputStream, InputEvent("k1", "put", 1)),
AddData(inputStream, InputEvent("k1", "get", -1)),
// advance clock to trigger processing
AdvanceManualClock(1 * 1000),
// validate value is not expired
+ //
+ // In the primary index, we still get that k1 -> [1].
+ // The TTL index should now have (9500, k1) -> empty. The min index
should have k1 -> 95000.
Review Comment:
ditto
##########
sql/core/src/test/scala/org/apache/spark/sql/streaming/TransformWithStateTTLTest.scala:
##########
@@ -143,43 +143,54 @@ abstract class TransformWithStateTTLTest
AddData(inputStream, InputEvent("k1", "put", 1)),
// advance clock to trigger processing
AdvanceManualClock(1 * 1000),
+ // In the primary index, we should have that k1 -> [1].
Review Comment:
nit: (1, 61000), but please feel free to leave it as it is if you intend to
do this to cover all types
##########
sql/core/src/test/scala/org/apache/spark/sql/streaming/TransformWithStateTTLTest.scala:
##########
@@ -143,43 +143,54 @@ abstract class TransformWithStateTTLTest
AddData(inputStream, InputEvent("k1", "put", 1)),
// advance clock to trigger processing
AdvanceManualClock(1 * 1000),
+ // In the primary index, we should have that k1 -> [1].
+ // The TTL index has (6100, k1) -> empty. The min index has k1 ->
61000.
Review Comment:
nit: TTL index and min-expiry index are off? Looks like this happens in
below code comment as well.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
