This is an automated email from the ASF dual-hosted git repository. Caideyipi pushed a commit to branch fix/pipe-compaction-queue-replacement in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 23cee2563ab7b2bab8dd615b7aa2ca0e3114a27e Merge: 5667d06f96c 9ab8a435014 Author: Caideyipi <[email protected]> AuthorDate: Mon Nov 16 16:39:01 2026 +0800 Merge origin/master and finish concurrent compaction test setup .../treemodel/auto/basic/IoTDBPipeSourceIT.java | 7 +- .../operator/process/TransformOperator.java | 8 +- .../operator/TransformOperatorResourceTest.java | 154 +++++++++++++++++++++ 3 files changed, 163 insertions(+), 6 deletions(-) diff --cc integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeSourceIT.java index 8566835ac5a,3db7bff4f8b..09bc959cc1f --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeSourceIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeSourceIT.java @@@ -29,7 -28,6 +29,7 @@@ import org.apache.iotdb.db.i18n.DataNod import org.apache.iotdb.db.it.utils.TestUtils; import org.apache.iotdb.isession.ISession; import org.apache.iotdb.isession.SessionConfig; - import org.apache.iotdb.it.env.cluster.config.MppDataNodeConfig; ++import org.apache.iotdb.it.env.cluster.config.MppCommonConfig; import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; import org.apache.iotdb.it.framework.IoTDBTestRunner; import org.apache.iotdb.itbase.category.MultiClusterIT2DualTreeAutoBasic; @@@ -81,15 -68,7 +81,15 @@@ public class IoTDBPipeSourceIT extends senderEnv .getConfig() .getDataNodeConfig() - .setMetricReporterType(Collections.singletonList("JMX")); + .setMetricReporterType(Collections.singletonList("JMX")) + .setCompactionScheduleInterval(100); + // Keep a backlog at the oldest end while one file is being sent. Compaction stays disabled + // for the other source tests; only the concurrent-compaction scenario enables it temporarily. + final Properties queueProperties = new Properties(); - queueProperties.setProperty("pipe_realtime_queue_max_waiting_tsfile_size", "1"); ++ queueProperties.setProperty("pipe_realTime_queue_max_waiting_tsFile_size", "1"); + queueProperties.setProperty("pipe_realtime_queue_poll_historical_tsfile_threshold", "1000000"); - ((MppDataNodeConfig) senderEnv.getConfig().getDataNodeConfig()) ++ ((MppCommonConfig) senderEnv.getConfig().getDataNodeCommonConfig()) + .updateProperties(queueProperties); senderEnv .getConfig() .getCommonConfig() @@@ -1136,135 -1114,6 +1136,136 @@@ } } + @Test + public void testConcurrentSendingAndCompaction() throws Exception { + final String pipeName = "compaction_correctness"; + final String device = "root.compaction_correctness.d1"; + final String query = "select s1 from " + device; + final String header = "Time," + device + ".s1,"; + final Set<String> expectedRows = new HashSet<>(); + final DataNodeWrapper receiver = receiverEnv.getDataNodeWrapper(0); + try (final Connection connection = senderEnv.getConnection(); + final Statement statement = connection.createStatement(); + final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + statement.execute("create timeseries " + device + ".s1 with datatype=INT64,encoding=RLE"); + for (int file = 0; file < 24; file++) { + final StringBuilder insert = + new StringBuilder("insert into " + device + "(time,s1) values "); + for (int point = 0; point < 64; point++) { + final int value = file * 64 + point; + if (point > 0) { + insert.append(','); + } + insert.append('(').append(value).append(',').append(value).append(')'); + expectedRows.add(value + "," + value + ","); + } + statement.execute(insert.toString()); + statement.execute("flush"); + } + + // The newest files occupy the send slot in a separate time partition. This leaves every + // source of the older compaction queued, including while the first send finishes. + final long blockerTime = TimeUnit.DAYS.toMillis(7); + statement.execute( + "create timeseries " + device + ".payload with datatype=TEXT,encoding=PLAIN"); + final byte[] payload = new byte[65536]; + final Random random = new Random(0); + for (int blocker = 0; blocker < 2; blocker++) { + random.nextBytes(payload); + final long time = blockerTime + blocker; + final int value = 1536 + blocker; + statement.execute( + "insert into " + + device + + "(time,s1,payload) values (" + + time + + "," + + value + + ",'" + + Base64.getEncoder().encodeToString(payload) + + "')"); + statement.execute("flush"); + expectedRows.add(time + "," + value + ","); + } + + final Map<String, String> sinkAttributes = new HashMap<>(); + sinkAttributes.put("sink", "iotdb-thrift-sink"); + sinkAttributes.put("sink.ip", receiver.getIp()); + sinkAttributes.put("sink.port", Integer.toString(receiver.getPort())); + sinkAttributes.put("sink.enable-compaction", "true"); ++ sinkAttributes.put("sink.parallel.tasks", "1"); + // Keep a send in flight while the remaining historical files are compacted. + sinkAttributes.put("sink.rate-limit-bytes-per-second", "8192"); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), + client + .createPipe( + new TCreatePipeReq(pipeName, sinkAttributes) + .setExtractorAttributes( + Map.of( + "source.path", "root.compaction_correctness.**", + "source.inclusion", "data.insert", + "source.realtime.mode", "batch"))) + .getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe(pipeName).getCode()); + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + query + " where time = " + (blockerTime + 1), + header, + Collections.singleton((blockerTime + 1) + ",1537,")); + + try { + statement.execute("set configuration 'enable_seq_space_compaction'='true'"); + final Pattern replacementMessage = + Pattern.compile( + Pattern.quote(DataNodePipeMessages.REGION_REPLACED_TSFILEINSERTIONEVENTS_WITH) + .replace("{}", "\\E.*\\Q")); + await() + .alias("A real queued compaction replacement for " + pipeName) + .atMost(60, TimeUnit.SECONDS) + .until(() -> hasCompactionReplacement(pipeName, replacementMessage)); + + PipeTestUtils.awaitDataPipeCompletion(senderEnv, pipeName); + TestUtils.assertDataEventuallyOnEnv(senderEnv, query, header, expectedRows); + TestUtils.assertDataEventuallyOnEnv(receiverEnv, query, header, expectedRows); + + // A later overwrite must also arrive and commit after the compacted backlog is drained. + statement.execute("insert into " + device + "(time,s1) values (0,-1),(1536,1536)"); + statement.execute("flush"); + expectedRows.remove("0,0,"); + expectedRows.add("0,-1,"); + expectedRows.add("1536,1536,"); + PipeTestUtils.awaitDataPipeCompletion(senderEnv, pipeName); + TestUtils.assertDataEventuallyOnEnv(senderEnv, query, header, expectedRows); + TestUtils.assertDataEventuallyOnEnv(receiverEnv, query, header, expectedRows); + } finally { + statement.execute("set configuration 'enable_seq_space_compaction'='false'"); + } + } + } + + private boolean hasCompactionReplacement(final String pipeName, final Pattern replacementMessage) + throws Exception { + for (final DataNodeWrapper node : senderEnv.getDataNodeWrapperList()) { + try (final Stream<Path> logs = Files.walk(Path.of(node.getLogDirPath()))) { + for (final Path log : + logs.filter(Files::isRegularFile) + .filter(path -> path.toString().endsWith(".log")) + .collect(Collectors.toList())) { + try (final Stream<String> lines = Files.lines(log)) { + if (lines.anyMatch( + line -> line.contains(pipeName) && replacementMessage.matcher(line).find())) { + return true; + } + } + } + } + } + return false; + } + private void assertTimeseriesCountOnReceiver(BaseEnv receiverEnv, int count) { TestUtils.assertDataEventuallyOnEnv( receiverEnv,
