jt2594838 commented on code in PR #18726:
URL: https://github.com/apache/iotdb/pull/18726#discussion_r4522523444
##########
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/subtask/sink/PipeRealtimePriorityBlockingQueue.java:
##########
@@ -249,71 +262,59 @@ public synchronized void replace(
eventsToBeAddedGroupByCommitterKey.put(committerKey, newEvents);
}
- // Handling new events
final Set<PipeTsFileInsertionEvent> successfullyReferenceIncreasedEvents =
new HashSet<>();
- final AtomicBoolean
- allSuccess = // To track if all events successfully increased the
reference count
- new AtomicBoolean(true);
+ boolean allSuccess = true;
outerLoop:
- for (final Map.Entry<CommitterKey, Set<PipeTsFileInsertionEvent>>
committerKeySetEntry :
- eventsToBeAddedGroupByCommitterKey.entrySet()) {
- for (final PipeTsFileInsertionEvent event :
committerKeySetEntry.getValue()) {
- if (event != null) {
- try {
- if
(!event.increaseReferenceCount(PipeRealtimePriorityBlockingQueue.class.getName()))
{
- allSuccess.set(false);
- break outerLoop;
- } else {
- successfullyReferenceIncreasedEvents.add(event);
- }
- } catch (final Exception e) {
- allSuccess.set(false);
+ for (final Set<PipeTsFileInsertionEvent> events :
eventsToBeAddedGroupByCommitterKey.values()) {
+ for (final PipeTsFileInsertionEvent event : events) {
+ try {
+ if
(!event.increaseReferenceCount(PipeRealtimePriorityBlockingQueue.class.getName()))
{
+ allSuccess = false;
break outerLoop;
}
+ successfullyReferenceIncreasedEvents.add(event);
+ } catch (final Exception e) {
+ allSuccess = false;
+ break outerLoop;
}
}
}
- if (!allSuccess.get()) {
- // If any event failed to increase the reference count,
- // we need to decrease the reference count for all successfully
increased events
+ if (!allSuccess) {
for (final PipeTsFileInsertionEvent event :
successfullyReferenceIncreasedEvents) {
try {
event.decreaseReferenceCount(PipeRealtimePriorityBlockingQueue.class.getName(),
false);
} catch (final Exception e) {
LOGGER.warn(DataNodePipeMessages.FAILED_TO_DECREASE_REFERENCE_COUNT_FOR_EVENT,
event, e);
}
}
- return; // Exit early if any event failed to increase the reference count
- } else {
- // If all events successfully increased reference count,
- // we can proceed to add them to the deque
- for (final PipeTsFileInsertionEvent event :
successfullyReferenceIncreasedEvents) {
- tsfileInsertEventDeque.add(event);
- eventCounter.increaseEventCount(event);
- }
+ return;
}
- // Handling old events
- for (final Map.Entry<CommitterKey, Set<PipeTsFileInsertionEvent>> entry :
- eventsToBeRemovedGroupByCommitterKey.entrySet()) {
- for (final PipeTsFileInsertionEvent event : entry.getValue()) {
- if (event != null) {
- try {
-
event.decreaseReferenceCount(PipeRealtimePriorityBlockingQueue.class.getName(),
false);
- } catch (final Exception e) {
- LOGGER.warn(
-
DataNodePipeMessages.FAILED_TO_DECREASE_REFERENCE_COUNT_FOR_EVENT, event, e);
- }
- eventCounter.decreaseEventCount(event);
- }
- }
- }
final Set<PipeTsFileInsertionEvent> eventsToRemove = new HashSet<>();
- for (Set<PipeTsFileInsertionEvent> pipeTsFileInsertionEvents :
+ for (final Set<PipeTsFileInsertionEvent> events :
eventsToBeRemovedGroupByCommitterKey.values()) {
- eventsToRemove.addAll(pipeTsFileInsertionEvents);
+ eventsToRemove.addAll(events);
+ }
+
+ // The deque is concurrent, so a source event may be polled while
replacement is in progress.
+ // Only release events that are still owned by this queue. If a poll wins
the race, both the
+ // source event and its compacted replacement may be delivered; this
deliberately favors the
+ // hot-path throughput over strict de-duplication.
+ final Set<PipeTsFileInsertionEvent> removedEvents =
+
eventsToRemove.stream().filter(tsfileInsertEventDeque::remove).collect(Collectors.toSet());
+ for (final PipeTsFileInsertionEvent event :
successfullyReferenceIncreasedEvents) {
+ tsfileInsertEventDeque.add(event);
+ eventCounter.increaseEventCount(event);
+ }
Review Comment:
Will this break the order of the files?
For example, we have f0, f1, f2, f3, f4, f5, and f0, f1, f2 are replaced
with f6, then the queue becomes f4, f5, f6.
If some data points of f0 are overwritten in f4, the new transfer order may
cause some trouble.
##########
integration-test/src/test/java/org/apache/iotdb/pipe/it/single/IoTDBPipeCompactionSinkParameterIT.java:
##########
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.pipe.it.single;
+
+import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient;
+import org.apache.iotdb.commons.pipe.config.constant.PipeSinkConstant;
+import org.apache.iotdb.confignode.rpc.thrift.TCreatePipeReq;
+import org.apache.iotdb.confignode.rpc.thrift.TShowPipeInfo;
+import org.apache.iotdb.confignode.rpc.thrift.TShowPipeReq;
+import org.apache.iotdb.it.framework.IoTDBTestRunner;
+import org.apache.iotdb.itbase.category.MultiClusterIT1;
+import org.apache.iotdb.rpc.TSStatusCode;
+
+import org.junit.Assert;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.junit.runner.RunWith;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+@RunWith(IoTDBTestRunner.class)
+@Category({MultiClusterIT1.class})
+public class IoTDBPipeCompactionSinkParameterIT extends AbstractPipeSingleIT {
Review Comment:
Is it possible to put this into an existing class?
##########
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/subtask/sink/PipeRealtimePriorityBlockingQueue.java:
##########
@@ -203,22 +199,39 @@ public Event peek() {
public synchronized void replace(
String dataRegionId, Set<TsFileResource> sourceFiles,
List<TsFileResource> targetFiles) {
+ if (!enableCompaction) {
+ return;
+ }
final int regionId = Integer.parseInt(dataRegionId);
+ final Set<Path> sourceFilePaths =
+ sourceFiles.stream()
+ .map(TsFileResource::getTsFile)
+ .map(PipeRealtimePriorityBlockingQueue::normalizePath)
+ .collect(Collectors.toSet());
+
final Map<CommitterKey, Set<PipeTsFileInsertionEvent>>
eventsToBeRemovedGroupByCommitterKey =
tsfileInsertEventDeque.stream()
.filter(
event ->
event instanceof PipeTsFileInsertionEvent
&& ((PipeTsFileInsertionEvent) event).getRegionId() ==
regionId)
.map(event -> (PipeTsFileInsertionEvent) event)
+ .filter(event ->
sourceFilePaths.contains(normalizePath(event.getSourceTsFile())))
.collect(
Collectors.groupingBy(
PipeTsFileInsertionEvent::getCommitterKey,
Collectors.toSet()))
.entrySet()
.stream()
- // Replace if all source files are present in the queue
- .filter(entry -> entry.getValue().size() == sourceFiles.size())
+ // Replace only when every source file is present exactly once for
this pipe.
+ .filter(entry -> entry.getValue().size() == sourceFilePaths.size())
+ .filter(
+ entry ->
+ entry.getValue().stream()
+ .map(PipeTsFileInsertionEvent::getSourceTsFile)
+ .map(PipeRealtimePriorityBlockingQueue::normalizePath)
+ .collect(Collectors.toSet())
+ .equals(sourceFilePaths))
Review Comment:
Is it possible to ease the condition to allow more files to be replaced when
possible?
One suggestion is to allow a suffix with a total size over twice the sum of
the targets to be replaced.
For example, the compaction involves f0, f1, f2, f3, f4 and results in f5
and f6.
However, f0 is already sent by Pipe and only f1-f4 remain in the queue.
If sum_size(f1, f2, f3, f4) > 2 * sum_size(f5, f6), then we still admit the
replacement since it is expected to reduce the IO amount significantly.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]