This is an automated email from the ASF dual-hosted git repository.
jt2594838 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new 2cd3b07f7af [Pipe] Fix stalled region ordering after TsFile-to-Tablet
conversion (#19243)
2cd3b07f7af is described below
commit 2cd3b07f7af82a05f4d14a7ba9dcab5cffb5115f
Author: Caideyipi <[email protected]>
AuthorDate: Thu Nov 12 18:56:33 2026 +0800
[Pipe] Fix stalled region ordering after TsFile-to-Tablet conversion
(#19243)
* [Pipe] Fix stalled region ordering after TsFile-to-Tablet conversion
* [Pipe] Track processed tablet descendants for TsFile completion
---
.../agent/task/connection/PipeEventCollector.java | 6 +-
.../common/tablet/PipeRawTabletInsertionEvent.java | 23 +-
.../common/tsfile/PipeTsFileInsertionEvent.java | 26 +-
.../db/pipe/source/PipeRealtimeExtractTest.java | 311 ++++++++++++++++++++-
4 files changed, 347 insertions(+), 19 deletions(-)
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/connection/PipeEventCollector.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/connection/PipeEventCollector.java
index 07ed903b671..1191c1e0739 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/connection/PipeEventCollector.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/connection/PipeEventCollector.java
@@ -286,11 +286,11 @@ public class PipeEventCollector implements EventCollector
{
}
if (event instanceof PipeRawTabletInsertionEvent
- && ((PipeRawTabletInsertionEvent) event).getSourceEvent()
- instanceof PipeTsFileInsertionEvent) {
+ && ((PipeRawTabletInsertionEvent)
event).getSourceTsFileInsertionEvent() != null) {
// Register before publishing to the queue so a concurrent queue cleanup
can report a
// generated tablet as discarded instead of racing with registration.
- ((PipeTsFileInsertionEvent) ((PipeRawTabletInsertionEvent)
event).getSourceEvent())
+ ((PipeRawTabletInsertionEvent) event)
+ .getSourceTsFileInsertionEvent()
.registerGeneratedTabletInsertionEvent((PipeRawTabletInsertionEvent)
event);
}
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/PipeRawTabletInsertionEvent.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/PipeRawTabletInsertionEvent.java
index fe8ce50130a..e41b8d39f63 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/PipeRawTabletInsertionEvent.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/PipeRawTabletInsertionEvent.java
@@ -66,6 +66,7 @@ public class PipeRawTabletInsertionEvent extends
PipeInsertionEvent
private final boolean isAligned;
private final EnrichedEvent sourceEvent;
+ private final PipeTsFileInsertionEvent sourceTsFileInsertionEvent;
private final AtomicBoolean generatedEventRegisteredWithSource = new
AtomicBoolean(false);
private final AtomicBoolean generatedEventOutcomeReported = new
AtomicBoolean(false);
private boolean needToReport;
@@ -115,6 +116,12 @@ public class PipeRawTabletInsertionEvent extends
PipeInsertionEvent
this.tablet = Objects.requireNonNull(tablet);
this.isAligned = isAligned;
this.sourceEvent = sourceEvent;
+ this.sourceTsFileInsertionEvent =
+ sourceEvent instanceof PipeTsFileInsertionEvent
+ ? (PipeTsFileInsertionEvent) sourceEvent
+ : sourceEvent instanceof PipeRawTabletInsertionEvent
+ ? ((PipeRawTabletInsertionEvent)
sourceEvent).getSourceTsFileInsertionEvent()
+ : null;
this.needToReport = needToReport;
inheritSourceEventReportSkippingIfNecessary();
@@ -135,7 +142,7 @@ public class PipeRawTabletInsertionEvent extends
PipeInsertionEvent
parentMemoryBlock);
this.allocatedMemoryBlock.setAssigner(this);
- if (needToReport || sourceEvent instanceof PipeTsFileInsertionEvent) {
+ if (needToReport || sourceTsFileInsertionEvent != null) {
addOnCommittedHook(
() -> {
// The last generated tablet still owns the progress-index
cleanup. Other generated
@@ -143,10 +150,9 @@ public class PipeRawTabletInsertionEvent extends
PipeInsertionEvent
if (shouldReportOnCommit && needToReport) {
eliminateProgressIndex();
}
- if (sourceEvent instanceof PipeTsFileInsertionEvent
+ if (sourceTsFileInsertionEvent != null
&& generatedEventOutcomeReported.compareAndSet(false, true)) {
- ((PipeTsFileInsertionEvent) sourceEvent)
- .markGeneratedTabletInsertionEventAsTransferred();
+
sourceTsFileInsertionEvent.markGeneratedTabletInsertionEventAsTransferred();
}
});
}
@@ -366,8 +372,8 @@ public class PipeRawTabletInsertionEvent extends
PipeInsertionEvent
final boolean cleared = super.clearReferenceCount(holderMessage);
if (generatedEventRegisteredWithSource.get()
&& generatedEventOutcomeReported.compareAndSet(false, true)
- && sourceEvent instanceof PipeTsFileInsertionEvent) {
- ((PipeTsFileInsertionEvent)
sourceEvent).markGeneratedTabletInsertionEventAsDiscarded();
+ && sourceTsFileInsertionEvent != null) {
+
sourceTsFileInsertionEvent.markGeneratedTabletInsertionEventAsDiscarded();
}
return cleared;
}
@@ -494,6 +500,11 @@ public class PipeRawTabletInsertionEvent extends
PipeInsertionEvent
return sourceEvent;
}
+ /** Returns the original TsFile, including when a processor has transformed
its tablets. */
+ public PipeTsFileInsertionEvent getSourceTsFileInsertionEvent() {
+ return sourceTsFileInsertionEvent;
+ }
+
@Override
public boolean isShouldReportOnCommit() {
return shouldReportOnCommit && needToReport;
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/PipeTsFileInsertionEvent.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/PipeTsFileInsertionEvent.java
index adc3e1d23fe..bb382721779 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/PipeTsFileInsertionEvent.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/PipeTsFileInsertionEvent.java
@@ -592,6 +592,18 @@ public class PipeTsFileInsertionEvent extends
PipeInsertionEvent
return isSuccessful;
}
+ @Override
+ public synchronized boolean decreaseReferenceCount(
+ final String holderMessage, final boolean shouldReport) {
+ final boolean decreased = super.decreaseReferenceCount(holderMessage,
shouldReport);
+ if (decreased && isReleased() && Objects.isNull(getCommitterKey())) {
+ // A split TsFile may only commit its generated tablets. Releasing that
original event ends
+ // its processing, but the transfer hook must still wait for parsing and
all tablet commits.
+ markAsTransferredIfGeneratedTabletEventsCompleted();
+ }
+ return decreased;
+ }
+
@Override
public boolean clearReferenceCount(final String holderMessage) {
final boolean cleared = super.clearReferenceCount(holderMessage);
@@ -698,14 +710,12 @@ public class PipeTsFileInsertionEvent extends
PipeInsertionEvent
final boolean generationBoundaryReached =
!generatedTabletInsertionEventsParsingStarted.get()
|| generatedTabletInsertionEventsParsingCompleted.get();
- // The normal processor path does not retain a tablet iterable unless
parsing has explicitly
- // been marked as started. In that path, registering the generated
tablets and reaching the
- // generation boundary is sufficient to establish the transfer boundary
(the source TsFile
- // commit may be reported separately or may be intentionally skipped).
Once an iterable has
- // been retained, however, the source TsFile commit must also be observed
before the generated
- // tablet counts can release the TsFile. This distinction keeps deferred
parsing safe without
- // delaying the established non-deferred path.
- if ((isTsFileEventCommitted.get() ||
!generatedTabletInsertionEventsParsingStarted.get())
+ // The collector only assigns commit ids to the generated tablets when it
splits a TsFile.
+ // The original event then finishes processing without entering the commit
queue. If the
+ // original event does enter that queue, resource release alone must not
bypass its commit.
+ if ((isTsFileEventCommitted.get()
+ || (isReleased() && Objects.isNull(getCommitterKey()))
+ || !generatedTabletInsertionEventsParsingStarted.get())
&& generationBoundaryReached
&& transferredGeneratedTabletInsertionEventCount.get()
+ discardedGeneratedTabletInsertionEventCount.get()
diff --git
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/source/PipeRealtimeExtractTest.java
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/source/PipeRealtimeExtractTest.java
index 2bb9501f2e4..13664d88942 100644
---
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/source/PipeRealtimeExtractTest.java
+++
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/source/PipeRealtimeExtractTest.java
@@ -25,6 +25,7 @@ import
org.apache.iotdb.commons.consensus.index.impl.MinimumProgressIndex;
import org.apache.iotdb.commons.consensus.index.impl.SimpleProgressIndex;
import org.apache.iotdb.commons.path.PartialPath;
import org.apache.iotdb.commons.pipe.agent.task.PipeTaskAgent;
+import
org.apache.iotdb.commons.pipe.agent.task.connection.UnboundedBlockingPendingQueue;
import org.apache.iotdb.commons.pipe.agent.task.meta.PipeMeta;
import org.apache.iotdb.commons.pipe.agent.task.meta.PipeMetaKeeper;
import org.apache.iotdb.commons.pipe.agent.task.meta.PipeRuntimeMeta;
@@ -41,10 +42,12 @@ import
org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNodeId;
import org.apache.iotdb.commons.utils.FileUtils;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.pipe.agent.PipeDataNodeAgent;
+import org.apache.iotdb.db.pipe.agent.task.connection.PipeEventCollector;
import
org.apache.iotdb.db.pipe.event.common.tablet.PipeRawTabletInsertionEvent;
import org.apache.iotdb.db.pipe.event.common.tsfile.PipeTsFileInsertionEvent;
import org.apache.iotdb.db.pipe.event.realtime.PipeRealtimeEvent;
import org.apache.iotdb.db.pipe.event.realtime.PipeRealtimeEventFactory;
+import org.apache.iotdb.db.pipe.metric.source.PipeDataRegionEventCounter;
import
org.apache.iotdb.db.pipe.source.dataregion.realtime.PipeRealtimeDataRegionHybridSource;
import
org.apache.iotdb.db.pipe.source.dataregion.realtime.PipeRealtimeDataRegionLogSource;
import
org.apache.iotdb.db.pipe.source.dataregion.realtime.PipeRealtimeDataRegionSource;
@@ -61,8 +64,11 @@ import
org.apache.iotdb.pipe.api.event.dml.insertion.TabletInsertionEvent;
import org.apache.iotdb.pipe.api.event.dml.insertion.TsFileInsertionEvent;
import org.apache.tsfile.common.constant.TsFileConstant;
+import org.apache.tsfile.enums.ColumnCategory;
import org.apache.tsfile.enums.TSDataType;
import org.apache.tsfile.file.metadata.IDeviceID;
+import org.apache.tsfile.file.metadata.TableSchema;
+import org.apache.tsfile.write.TsFileWriter;
import org.apache.tsfile.write.record.Tablet;
import org.apache.tsfile.write.schema.MeasurementSchema;
import org.junit.After;
@@ -76,10 +82,12 @@ import java.io.File;
import java.io.IOException;
import java.lang.reflect.Field;
import java.nio.file.Files;
+import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
+import java.util.Map;
import java.util.Set;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
@@ -1506,8 +1514,9 @@ public class PipeRealtimeExtractTest {
final PipeTsFileInsertionEvent firstTsFile = (PipeTsFileInsertionEvent)
extractor.supply();
firstTsFile.markGeneratedTabletInsertionEventsParsingStarted();
- commitSuppliedEvent(firstTsFile, commitManager);
- // Committing a TsFile with a retained iterable and no tablets yet must
not release the gate.
+ Assert.assertNull(firstTsFile.getCommitterKey());
+
Assert.assertTrue(firstTsFile.decreaseReferenceCount(TEST_REFERENCE_HOLDER,
false));
+ // Releasing the original event must not release the gate while
generation is still open.
Assert.assertNull(extractor.supply());
final PipeRawTabletInsertionEvent firstTablet =
createGeneratedTabletEvent(firstTsFile, pipeTaskMeta, "first",
false);
@@ -1535,6 +1544,248 @@ public class PipeRealtimeExtractTest {
}
}
+ @Test
+ public void testTsFileSourceReleasesSplitTableFileAfterTabletCommit() throws
Exception {
+ verifySplitTableFileCompletion(false);
+ }
+
+ @Test
+ public void testTsFileSourceReleasesSplitTableFileAfterOriginalRelease()
throws Exception {
+ verifySplitTableFileCompletion(true);
+ }
+
+ private void verifySplitTableFileCompletion(final boolean
commitTabletBeforeOriginalRelease)
+ throws Exception {
+ final PipeEventCommitManager commitManager =
PipeEventCommitManager.getInstance();
+ commitManager.register(TEST_PIPE_NAME, TEST_PIPE_CREATION_TIME,
dataRegion1, "test");
+ final PipeTaskMeta pipeTaskMeta = new
PipeTaskMeta(MinimumProgressIndex.INSTANCE, 1);
+ final UnboundedBlockingPendingQueue<Event> pendingQueue =
+ new UnboundedBlockingPendingQueue<>(new PipeDataRegionEventCounter());
+ final PipeRealtimeEvent[] events = new PipeRealtimeEvent[2];
+ try (final PipeRealtimeDataRegionTsFileSource extractor =
+ new PipeRealtimeDataRegionTsFileSource()) {
+ final PipeParameters parameters =
+ new PipeParameters(
+ Map.of(
+ PipeSourceConstant.SOURCE_CAPTURE_TABLE_KEY, "true",
+ PipeSourceConstant.SOURCE_TABLE_NAME_KEY, "t0",
+
PipeSourceConstant.SOURCE_REALTIME_REGION_LEVEL_DOWNGRADING_KEY, "true"));
+ extractor.validate(new PipeParameterValidator(parameters));
+ extractor.customize(
+ parameters,
+ new PipeTaskRuntimeConfiguration(
+ new PipeTaskSourceRuntimeEnvironment(
+ TEST_PIPE_NAME, TEST_PIPE_CREATION_TIME, dataRegion1,
pipeTaskMeta)));
+ for (int i = 0; i < events.length; i++) {
+ events[i] =
+ PipeRealtimeEventFactory.createRealtimeEvent(
+ true,
+ "table_db",
+ createTableTsFileResource("126-" + (i + 1) +
"-0-0.tsfile"),
+ false)
+ .shallowCopySelfAndBindPipeTaskMetaForProgressReport(
+ TEST_PIPE_NAME,
+ TEST_PIPE_CREATION_TIME,
+ pipeTaskMeta,
+ extractor.getTreePattern(),
+ extractor.getTablePattern(),
+ "0",
+ "root",
+ "localhost",
+ true,
+ Long.MIN_VALUE,
+ Long.MAX_VALUE);
+
Assert.assertTrue(events[i].increaseReferenceCount(TEST_REFERENCE_HOLDER));
+ extractor.extract(events[i]);
+ }
+
+ final PipeTsFileInsertionEvent firstTsFile = (PipeTsFileInsertionEvent)
extractor.supply();
+ final PipeEventCollector collector =
+ new PipeEventCollector(
+ pendingQueue,
+ TEST_PIPE_CREATION_TIME,
+ dataRegion1,
+ extractor.getSourceId(),
+ false,
+ false,
+ false);
+ collector.collect(firstTsFile);
+ Assert.assertFalse(collector.hasNoGeneratedEvent());
+ Assert.assertFalse(collector.isFailedToIncreaseReferenceCount());
+ Assert.assertNull(firstTsFile.getCommitterKey());
+ Assert.assertEquals(1, pendingQueue.size());
+ final Event generatedTablet = pendingQueue.directPoll();
+ Assert.assertTrue(generatedTablet instanceof
PipeRawTabletInsertionEvent);
+ Assert.assertNotNull(((EnrichedEvent)
generatedTablet).getCommitterKey());
+ try {
+ if (commitTabletBeforeOriginalRelease) {
+ Assert.assertTrue(
+ ((EnrichedEvent) generatedTablet)
+ .decreaseReferenceCount(TEST_REFERENCE_HOLDER, true));
+ Assert.assertNull(extractor.supply());
+ }
+
+ // Mirror the processor: only the generated tablet enters the commit
queue.
+
Assert.assertTrue(firstTsFile.decreaseReferenceCount(TEST_REFERENCE_HOLDER,
false));
+ if (!commitTabletBeforeOriginalRelease) {
+ Assert.assertNull(extractor.supply());
+ Assert.assertTrue(
+ ((EnrichedEvent) generatedTablet)
+ .decreaseReferenceCount(TEST_REFERENCE_HOLDER, true));
+ }
+
+ final Event secondTsFile = extractor.supply();
+ Assert.assertSame(events[1].getEvent(), secondTsFile);
+ releaseSuppliedEvent(secondTsFile);
+ } finally {
+ ((EnrichedEvent)
generatedTablet).clearReferenceCount(TEST_REFERENCE_HOLDER);
+ }
+ } finally {
+ pendingQueue.discardAllEvents();
+ for (final PipeRealtimeEvent event : events) {
+ if (event != null) {
+ event.clearReferenceCount(TEST_REFERENCE_HOLDER);
+ }
+ }
+ commitManager.deregister(TEST_PIPE_NAME, TEST_PIPE_CREATION_TIME,
dataRegion1);
+ }
+ }
+
+ @Test
+ public void testTsFileSourceWaitsForProcessedTabletCommits() throws
Exception {
+ verifyProcessedTabletCompletion(false, false);
+ }
+
+ @Test
+ public void testTsFileSourceWaitsForDiscardedProcessedTablets() throws
Exception {
+ verifyProcessedTabletCompletion(true, false);
+ }
+
+ @Test
+ public void
testTsFileSourceWaitsForOriginalOrderedCommitAfterProcessedTablets()
+ throws Exception {
+ verifyProcessedTabletCompletion(true, true);
+ }
+
+ private void verifyProcessedTabletCompletion(
+ final boolean discardTablets, final boolean originalCommitPending)
throws Exception {
+ final PipeEventCommitManager commitManager =
PipeEventCommitManager.getInstance();
+ commitManager.register(TEST_PIPE_NAME, TEST_PIPE_CREATION_TIME,
dataRegion1, "test");
+ final PipeTaskMeta pipeTaskMeta = new
PipeTaskMeta(MinimumProgressIndex.INSTANCE, 1);
+ final UnboundedBlockingPendingQueue<Event> pendingQueue =
+ new UnboundedBlockingPendingQueue<>(new PipeDataRegionEventCounter());
+ final List<PipeRawTabletInsertionEvent> parsedTablets = new ArrayList<>();
+ final List<PipeRawTabletInsertionEvent> generatedTablets = new
ArrayList<>();
+ final PipeRealtimeEvent[] events = new PipeRealtimeEvent[2];
+ final ProgressReportEvent earlierEvent =
+ new ProgressReportEvent(TEST_PIPE_NAME, TEST_PIPE_CREATION_TIME,
pipeTaskMeta);
+ try (final PipeRealtimeDataRegionTsFileSource extractor =
+ createTsFileSourceForTest(
+ PipeSourceConstant.SOURCE_REALTIME_REGION_LEVEL_DOWNGRADING_KEY,
pipeTaskMeta)) {
+ events[0] =
+ extractTsFileForTest(
+ extractor, pipeTaskMeta,
createTreeTsFileResource("127-1-0-0.tsfile"), false);
+ events[1] =
+ extractTsFileForTest(
+ extractor,
+ pipeTaskMeta,
+ createTsFileResource(dataRegion1, "127-2-0-0.tsfile"),
+ false);
+ final PipeTsFileInsertionEvent firstTsFile = (PipeTsFileInsertionEvent)
extractor.supply();
+ final AtomicInteger completionCount = new AtomicInteger();
+ firstTsFile.addOnTransferredHook(completionCount::incrementAndGet);
+ if (originalCommitPending) {
+
Assert.assertTrue(earlierEvent.increaseReferenceCount(TEST_REFERENCE_HOLDER));
+ commitManager.enrichWithCommitterKeyAndCommitId(
+ earlierEvent, TEST_PIPE_CREATION_TIME, dataRegion1);
+ commitManager.enrichWithCommitterKeyAndCommitId(
+ firstTsFile, TEST_PIPE_CREATION_TIME, dataRegion1);
+ Assert.assertNotNull(earlierEvent.getCommitterKey());
+ Assert.assertNotNull(firstTsFile.getCommitterKey());
+ } else {
+ Assert.assertNull(firstTsFile.getCommitterKey());
+ }
+ final PipeEventCollector collector =
+ new PipeEventCollector(
+ pendingQueue,
+ TEST_PIPE_CREATION_TIME,
+ dataRegion1,
+ extractor.getSourceId(),
+ false,
+ false,
+ false);
+ firstTsFile.consumeTabletInsertionEventsWithRetry(
+ parsedTablet -> {
+
Assert.assertTrue(parsedTablet.increaseReferenceCount(TEST_REFERENCE_HOLDER));
+ parsedTablets.add(parsedTablet);
+ // A processor may produce multiple outputs from the same parsed
tablet.
+ for (int i = 0; i < 2; i++) {
+ for (final TabletInsertionEvent processedTablet :
+ parsedTablet.processRowByRow(
+ (row, rowCollector) -> {
+ try {
+ rowCollector.collectRow(row);
+ } catch (final IOException e) {
+ throw new RuntimeException(e);
+ }
+ })) {
+ collector.collect(processedTablet);
+ }
+ }
+ },
+ TEST_REFERENCE_HOLDER);
+ Assert.assertEquals(1, parsedTablets.size());
+ Assert.assertEquals(2, pendingQueue.size());
+ Assert.assertFalse(collector.isFailedToIncreaseReferenceCount());
+ while (!pendingQueue.isEmpty()) {
+ final PipeRawTabletInsertionEvent generatedTablet =
+ (PipeRawTabletInsertionEvent) pendingQueue.directPoll();
+ Assert.assertSame(parsedTablets.get(0),
generatedTablet.getSourceEvent());
+ Assert.assertNotNull(generatedTablet.getCommitterKey());
+ generatedTablets.add(generatedTablet);
+ }
+
+
Assert.assertTrue(firstTsFile.decreaseReferenceCount(TEST_REFERENCE_HOLDER,
false));
+ Assert.assertTrue(firstTsFile.isReleased());
+ Assert.assertEquals(0, completionCount.get());
+ Assert.assertNull(extractor.supply());
+ for (int i = 0; i < generatedTablets.size(); i++) {
+ final PipeRawTabletInsertionEvent generatedTablet =
generatedTablets.get(i);
+ if (discardTablets) {
+
Assert.assertTrue(generatedTablet.clearReferenceCount(TEST_REFERENCE_HOLDER));
+ } else {
+
Assert.assertTrue(generatedTablet.decreaseReferenceCount(TEST_REFERENCE_HOLDER,
true));
+ }
+
Assert.assertFalse(generatedTablet.clearReferenceCount(TEST_REFERENCE_HOLDER));
+ if (i == 0 || originalCommitPending) {
+ Assert.assertEquals(0, completionCount.get());
+ Assert.assertNull(extractor.supply());
+ }
+ }
+ if (originalCommitPending) {
+
Assert.assertTrue(earlierEvent.decreaseReferenceCount(TEST_REFERENCE_HOLDER,
true));
+ }
+ Assert.assertEquals(1, completionCount.get());
+ final Event secondTsFile = extractor.supply();
+ Assert.assertSame(events[1].getEvent(), secondTsFile);
+ generatedTablets.forEach(tablet ->
tablet.clearReferenceCount(TEST_REFERENCE_HOLDER));
+ Assert.assertEquals(1, completionCount.get());
+ Assert.assertNull(extractor.supply());
+ releaseSuppliedEvent(secondTsFile);
+ } finally {
+ pendingQueue.discardAllEvents();
+ generatedTablets.forEach(tablet ->
tablet.clearReferenceCount(TEST_REFERENCE_HOLDER));
+ parsedTablets.forEach(tablet ->
tablet.clearReferenceCount(TEST_REFERENCE_HOLDER));
+ for (final PipeRealtimeEvent event : events) {
+ if (event != null) {
+ event.clearReferenceCount(TEST_REFERENCE_HOLDER);
+ }
+ }
+ earlierEvent.clearReferenceCount(TEST_REFERENCE_HOLDER);
+ commitManager.deregister(TEST_PIPE_NAME, TEST_PIPE_CREATION_TIME,
dataRegion1);
+ }
+ }
+
@Test
public void
testTsFileSourceRegionLevelDowngradingCompensatesForDiscardedEvents()
throws Exception {
@@ -1762,6 +2013,62 @@ public class PipeRealtimeExtractTest {
return createTsFileResource(dataRegionId, "0", fileName);
}
+ private TsFileResource createTreeTsFileResource(final String fileName)
throws Exception {
+ final File dataRegionDir = new File(tsFileDir, dataRegion1 +
File.separator + "0");
+ Assert.assertTrue(dataRegionDir.mkdirs() || dataRegionDir.isDirectory());
+ final File tsFile = new File(dataRegionDir, fileName);
+ final Tablet tablet =
+ new Tablet(
+ "root.sg.d",
+ Collections.singletonList(new MeasurementSchema("s0",
TSDataType.INT64)),
+ 1);
+ tablet.addTimestamp(0, 1);
+ tablet.addValue(0, 0, 1L);
+ try (final TsFileWriter writer = new TsFileWriter(tsFile)) {
+ writer.registerTimeseries(new PartialPath("root.sg.d"),
tablet.getSchemas());
+ writer.writeTree(tablet);
+ }
+ final TsFileResource resource = new TsFileResource(tsFile);
+ final IDeviceID deviceId =
IDeviceID.Factory.DEFAULT_FACTORY.create("root.sg.d");
+ resource.updateStartTime(deviceId, 1);
+ resource.updateEndTime(deviceId, 1);
+ resource.close();
+ return resource;
+ }
+
+ private TsFileResource createTableTsFileResource(final String fileName)
throws Exception {
+ final File dataRegionDir = new File(tsFileDir, dataRegion1 +
File.separator + "0");
+ Assert.assertTrue(dataRegionDir.mkdirs() || dataRegionDir.isDirectory());
+ final File tsFile = new File(dataRegionDir, fileName);
+ final List<ColumnCategory> categories = List.of(ColumnCategory.TAG,
ColumnCategory.FIELD);
+ try (final TsFileWriter writer = new TsFileWriter(tsFile)) {
+ writer.registerTableSchema(
+ new TableSchema(
+ "t0",
+ List.of(
+ new MeasurementSchema("device_id", TSDataType.STRING),
+ new MeasurementSchema("s0", TSDataType.INT64)),
+ categories));
+ final Tablet tablet =
+ new Tablet(
+ "t0",
+ List.of("device_id", "s0"),
+ List.of(TSDataType.STRING, TSDataType.INT64),
+ categories,
+ 1);
+ tablet.addTimestamp(0, 1);
+ tablet.addValue(0, 0, "d0");
+ tablet.addValue(0, 1, 1L);
+ writer.writeTable(tablet);
+ }
+ final TsFileResource resource = new TsFileResource(tsFile);
+ final IDeviceID deviceId = IDeviceID.Factory.DEFAULT_FACTORY.create(new
String[] {"t0", "d0"});
+ resource.updateStartTime(deviceId, 1);
+ resource.updateEndTime(deviceId, 1);
+ resource.close();
+ return resource;
+ }
+
private TsFileResource createTsFileResource(
final int dataRegionId, final String timePartition, final String
fileName)
throws IOException {