slilichenko commented on code in PR #28171:
URL: https://github.com/apache/beam/pull/28171#discussion_r1309577603
##########
runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/WindmillStateInternalsTest.java:
##########
@@ -2169,6 +2169,71 @@ public void testOrderedListMergePendingAdds() {
assertArrayEquals(expected, read);
}
+ @Test
+ public void testOrderedListMergePendingAddsAndDeletes() {
+ SettableFuture<Map<Range<Instant>, RangeSet<Long>>> orderedListFuture =
SettableFuture.create();
+ orderedListFuture.set(null);
+ SettableFuture<Map<Range<Instant>, RangeSet<Instant>>> deletionsFuture =
+ SettableFuture.create();
+ deletionsFuture.set(null);
+ when(mockReader.valueFuture(
+ systemKey(NAMESPACE, "orderedList" + IdTracker.IDS_AVAILABLE_STR),
+ STATE_FAMILY,
+ IdTracker.IDS_AVAILABLE_CODER))
+ .thenReturn(orderedListFuture);
+ when(mockReader.valueFuture(
+ systemKey(NAMESPACE, "orderedList" + IdTracker.DELETIONS_STR),
+ STATE_FAMILY,
+ IdTracker.SUBRANGE_DELETIONS_CODER))
+ .thenReturn(deletionsFuture);
+
+ SettableFuture<Iterable<TimestampedValue<String>>> fromStorage =
SettableFuture.create();
+ when(mockReader.orderedListFuture(
+ FULL_ORDERED_LIST_RANGE,
+ key(NAMESPACE, "orderedList"),
+ STATE_FAMILY,
+ StringUtf8Coder.of()))
+ .thenReturn(fromStorage);
+
+ StateTag<OrderedListState<String>> addr =
+ StateTags.orderedList("orderedList", StringUtf8Coder.of());
+ OrderedListState<String> orderedListState = underTest.state(NAMESPACE,
addr);
+
+ orderedListState.add(TimestampedValue.of("second",
Instant.ofEpochMilli(1)));
+ orderedListState.add(TimestampedValue.of("third",
Instant.ofEpochMilli(2)));
+ orderedListState.add(TimestampedValue.of("fourth",
Instant.ofEpochMilli(2)));
Review Comment:
Is it a typo for the "fourth" element - same ts as "third" and gets added
later with a different ts?
##########
runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/WindmillStateInternalsTest.java:
##########
@@ -2169,6 +2169,71 @@ public void testOrderedListMergePendingAdds() {
assertArrayEquals(expected, read);
}
+ @Test
+ public void testOrderedListMergePendingAddsAndDeletes() {
+ SettableFuture<Map<Range<Instant>, RangeSet<Long>>> orderedListFuture =
SettableFuture.create();
+ orderedListFuture.set(null);
+ SettableFuture<Map<Range<Instant>, RangeSet<Instant>>> deletionsFuture =
+ SettableFuture.create();
+ deletionsFuture.set(null);
+ when(mockReader.valueFuture(
+ systemKey(NAMESPACE, "orderedList" + IdTracker.IDS_AVAILABLE_STR),
+ STATE_FAMILY,
+ IdTracker.IDS_AVAILABLE_CODER))
+ .thenReturn(orderedListFuture);
+ when(mockReader.valueFuture(
+ systemKey(NAMESPACE, "orderedList" + IdTracker.DELETIONS_STR),
+ STATE_FAMILY,
+ IdTracker.SUBRANGE_DELETIONS_CODER))
+ .thenReturn(deletionsFuture);
+
+ SettableFuture<Iterable<TimestampedValue<String>>> fromStorage =
SettableFuture.create();
+ when(mockReader.orderedListFuture(
+ FULL_ORDERED_LIST_RANGE,
+ key(NAMESPACE, "orderedList"),
+ STATE_FAMILY,
+ StringUtf8Coder.of()))
+ .thenReturn(fromStorage);
+
+ StateTag<OrderedListState<String>> addr =
+ StateTags.orderedList("orderedList", StringUtf8Coder.of());
+ OrderedListState<String> orderedListState = underTest.state(NAMESPACE,
addr);
+
+ orderedListState.add(TimestampedValue.of("second",
Instant.ofEpochMilli(1)));
+ orderedListState.add(TimestampedValue.of("third",
Instant.ofEpochMilli(2)));
+ orderedListState.add(TimestampedValue.of("fourth",
Instant.ofEpochMilli(2)));
+ orderedListState.add(TimestampedValue.of("eighth",
Instant.ofEpochMilli(10)));
+ orderedListState.add(TimestampedValue.of("ninth",
Instant.ofEpochMilli(15)));
+
+ orderedListState.clearRange(Instant.ofEpochMilli(2),
Instant.ofEpochMilli(5));
+ orderedListState.add(TimestampedValue.of("fourth",
Instant.ofEpochMilli(4)));
+
+ fromStorage.set(
+ ImmutableList.of(
+ TimestampedValue.of("first", Instant.ofEpochMilli(-1)),
+ TimestampedValue.of("fifth", Instant.ofEpochMilli(5)),
+ TimestampedValue.of("sixth", Instant.ofEpochMilli(5)),
+ TimestampedValue.of("seventh", Instant.ofEpochMilli(5)),
+ TimestampedValue.of("tenth", Instant.ofEpochMilli(20))));
+
+ TimestampedValue[] expected =
+ Iterables.toArray(
+ ImmutableList.of(
+ TimestampedValue.of("first", Instant.ofEpochMilli(-1)),
+ TimestampedValue.of("second", Instant.ofEpochMilli(1)),
+ TimestampedValue.of("fourth", Instant.ofEpochMilli(4)),
+ TimestampedValue.of("fifth", Instant.ofEpochMilli(5)),
+ TimestampedValue.of("sixth", Instant.ofEpochMilli(5)),
+ TimestampedValue.of("seventh", Instant.ofEpochMilli(5)),
Review Comment:
Side note - would be good to describe the behavior of the list when multiple
elements are added with the same timestamp (the fact that it behaves as a
multimap and ordering guarantees for the entries under the same key, if any).
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
