Martijn Visser created FLINK-41553:
--------------------------------------
Summary: CepOperator accepts late events after a restore and
corrupts the SharedBuffer
Key: FLINK-41553
URL: https://issues.apache.org/jira/browse/FLINK-41553
Project: Flink
Issue Type: Bug
Components: Library / CEP
Affects Versions: 2.3.0, 2.4.0
Reporter: Martijn Visser
After a restore, {{CepOperator}} uses {{timerService.currentWatermark()}} to
decide whether an event is late. That watermark is not part of the snapshot and
starts at {{{}Long.MIN_VALUE{}}}, so until the first watermark arrives, events
older than the pre-restore watermark are buffered instead of dropped.
{{SharedBuffer#advanceTime}} already removed the event counter for those
timestamps, while a partial match within {{within()}} can still reference an
event at the same timestamp. {{SharedBuffer#registerEvent}} then hands out the
same {{EventId}} again. Depending on the input this fails with an
{{IllegalStateException}} in {{NFA#extractCurrentMatches}}
({{{}checkState(paths.size() == 1){}}}) or with an NPE in
{{{}SharedBufferAccessor#materializeMatch{}}}. Every restart restores the same
state and replays the same input, so the job fails in a loop.
This test can be dropped it into
{{flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/operator as a
reproducer}}
{code:java}
package org.apache.flink.cep.operator;
import org.apache.flink.api.common.serialization.SerializerConfigImpl;
import org.apache.flink.api.common.typeinfo.BasicTypeInfo;
import org.apache.flink.api.common.typeinfo.TypeInformation;
import org.apache.flink.api.java.functions.KeySelector;
import org.apache.flink.cep.functions.PatternProcessFunction;
import org.apache.flink.cep.nfa.aftermatch.AfterMatchSkipStrategy;
import org.apache.flink.cep.nfa.compiler.NFACompiler;
import org.apache.flink.cep.pattern.Pattern;
import org.apache.flink.cep.pattern.conditions.IterativeCondition;
import org.apache.flink.runtime.checkpoint.OperatorSubtaskState;
import org.apache.flink.state.rocksdb.EmbeddedRocksDBStateBackend;
import org.apache.flink.streaming.util.KeyedOneInputStreamOperatorTestHarness;
import org.apache.flink.util.Collector;
import org.junit.jupiter.api.Test;
import java.time.Duration;
import java.util.List;
import java.util.Map;
import static org.junit.jupiter.api.Assertions.assertEquals;
/**
* After a restore the operator's watermark is Long.MIN_VALUE, so an event
older than the
* pre-restore watermark is not dropped as late. Its timestamp's event counter
was already removed
* by SharedBuffer#advanceTime, so it reuses the EventId of an event a partial
match still
* references.
*/
class CepOperatorRestoreLateEventTest {
/** Input row. */
public static class Event {
public String key;
public String kind;
public int id;
public Event() {}
Event(String kind, int id) {
this.key = "k";
this.kind = kind;
this.id = id;
}
@Override
public String toString() {
return kind + "#" + id;
}
}
/** Like SQL DEFINE x AS x.kind = '...': the generated condition reads
getEventsForPattern(x). */
private static IterativeCondition<Event> kind(String var, String kind,
boolean negate) {
return new IterativeCondition<>() {
@Override
public boolean filter(Event value, Context<Event> ctx) throws
Exception {
ctx.getEventsForPattern(var).forEach(e -> {});
return negate != kind.equals(value.kind);
}
};
}
private static KeyedOneInputStreamOperatorTestHarness<String, Event,
String> harness()
throws Exception {
// PATTERN (V M* L) WITHIN INTERVAL '30' MINUTE AFTER MATCH SKIP PAST
LAST ROW
Pattern<Event, Event> pattern =
Pattern.<Event>begin("V",
AfterMatchSkipStrategy.skipPastLastEvent())
.where(kind("V", "start", false))
.next("M")
.where(kind("M", "end", true))
.oneOrMore()
.optional()
.consecutive()
.greedy()
.next("L")
.where(kind("L", "end", false))
.within(Duration.ofMinutes(30));
CepOperator<Event, String, String> operator =
new CepOperator<>(
TypeInformation.of(Event.class).createSerializer(new
SerializerConfigImpl()),
false,
NFACompiler.compileFactory(pattern, false),
null,
pattern.getAfterMatchSkipStrategy(),
new PatternProcessFunction<Event, String>() {
@Override
public void processMatch(
Map<String, List<Event>> match,
Context ctx,
Collector<String> out) {
out.collect(match.toString());
}
},
null);
KeyedOneInputStreamOperatorTestHarness<String, Event, String> harness =
new KeyedOneInputStreamOperatorTestHarness<>(
operator,
(KeySelector<Event, String>) e -> e.key,
BasicTypeInfo.STRING_TYPE_INFO);
harness.setStateBackend(new EmbeddedRocksDBStateBackend());
return harness;
}
private static void runRestoreWithLateEvents(Event... lateEvents) throws
Exception {
OperatorSubtaskState snapshot;
try (KeyedOneInputStreamOperatorTestHarness<String, Event, String>
harness = harness()) {
harness.open();
harness.processElement(new Event("start", 1), 1000L);
harness.processElement(new Event("middle", 2), 2000L);
harness.processWatermark(5000L);
snapshot = harness.snapshot(1L, 1L);
}
try (KeyedOneInputStreamOperatorTestHarness<String, Event, String>
harness = harness()) {
harness.initializeState(snapshot);
harness.open();
for (Event late : lateEvents) {
// older than the pre-restore watermark of 5000, so these are
late
harness.processElement(late, late.kind.equals("start") ? 1000L
: 2000L);
}
harness.processElement(new Event("end", 9), 6000L);
harness.processWatermark(7000L);
assertEquals(
List.of("{V=[start#1], M=[middle#2], L=[end#9]}"),
harness.extractOutputValues());
}
}
@Test
void testLateEventAfterRestoreFailsCheckState() throws Exception {
// IllegalStateException at NFA#extractCurrentMatches
runRestoreWithLateEvents(new Event("start", 3), new Event("middle", 4));
}
@Test
void testLateEventAfterRestoreFailsWithNpe() throws Exception {
// NullPointerException, SharedBuffer#getEvent returns null in
materializeMatch
runRestoreWithLateEvents(new Event("middle", 3));
}
}
{code}
{noformat}
java.lang.IllegalStateException
at org.apache.flink.util.Preconditions.checkState(Preconditions.java:177)
at org.apache.flink.cep.nfa.NFA.extractCurrentMatches(NFA.java:877)
at
org.apache.flink.cep.nfa.NFA$ConditionContext.getEventsForPattern(NFA.java:922)
java.lang.NullPointerException: Cannot invoke
"org.apache.flink.cep.nfa.sharedbuffer.Lockable.getElement()" because the
return value of
"org.apache.flink.cep.nfa.sharedbuffer.SharedBuffer.getEvent(org.apache.flink.cep.nfa.sharedbuffer.EventId)"
is null
at
org.apache.flink.cep.nfa.sharedbuffer.SharedBufferAccessor.materializeMatch(SharedBufferAccessor.java:215)
{noformat}
--
This message was sent by Atlassian Jira
(v10.3.26#10030026)