(iotdb) branch master updated: Pipe: Support table pattern matching when sync data between clusters (#13817 following fixup) (#13830)

[rong]

This is an automated email from the ASF dual-hosted git repository.

rong pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git


The following commit(s) were added to refs/heads/master by this push:
     new 0d8ad0c6790 Pipe: Support table pattern matching when sync data 
between clusters (#13817 following fixup) (#13830)
0d8ad0c6790 is described below

commit 0d8ad0c67907f23411812d8c438b93cbfee6de34
Author: Steve Yurong Su <r...@apache.org>
AuthorDate: Sat Oct 19 02:25:13 2024 +0800

    Pipe: Support table pattern matching when sync data between clusters 
(#13817 following fixup) (#13830)
---
 .../db/pipe/agent/task/connection/PipeEventCollector.java    |  6 +++++-
 .../payload/evolvable/batch/PipeTabletEventPlainBatch.java   |  4 ++++
 .../table/TsFileInsertionEventTableParserTabletIterator.java | 12 +++++++++++-
 pom.xml                                                      |  2 +-
 4 files changed, 21 insertions(+), 3 deletions(-)

diff --git 
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/connection/PipeEventCollector.java
 
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/connection/PipeEventCollector.java
index 5ff7a87bc2c..fa02b6e8f2a 100644
--- 
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/connection/PipeEventCollector.java
+++ 
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/connection/PipeEventCollector.java
@@ -120,7 +120,11 @@ public class PipeEventCollector implements EventCollector {
       return;
     }
 
-    if (!forceTabletFormat && !sourceEvent.shouldParseTimeOrPattern()) {
+    if (!forceTabletFormat
+        && (!sourceEvent.shouldParseTimeOrPattern()
+            || (sourceEvent.isTableModelEvent()
+                && sourceEvent.getTablePattern() == null
+                && !sourceEvent.shouldParseTime()))) {
       collectEvent(sourceEvent);
       return;
     }
diff --git 
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/payload/evolvable/batch/PipeTabletEventPlainBatch.java
 
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/payload/evolvable/batch/PipeTabletEventPlainBatch.java
index c3fb4b9068e..fae2338a215 100644
--- 
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/payload/evolvable/batch/PipeTabletEventPlainBatch.java
+++ 
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/payload/evolvable/batch/PipeTabletEventPlainBatch.java
@@ -110,6 +110,10 @@ public class PipeTabletEventPlainBatch extends 
PipeTabletEventBatch {
     insertNodeBuffers.clear();
     tabletBuffers.clear();
 
+    binaryDataBases.clear();
+    inertNodeDataBases.clear();
+    tabletDataBases.clear();
+
     pipe2BytesAccumulated.clear();
   }
 
diff --git 
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/table/TsFileInsertionEventTableParserTabletIterator.java
 
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/table/TsFileInsertionEventTableParserTabletIterator.java
index 5ae2cf6188a..dc65963980a 100644
--- 
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/table/TsFileInsertionEventTableParserTabletIterator.java
+++ 
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/table/TsFileInsertionEventTableParserTabletIterator.java
@@ -102,6 +102,8 @@ public class TsFileInsertionEventTableParserTabletIterator 
implements Iterator<T
         new Tablet(tableName, columnSchemas, columnTypes, 
tsBlock.getPositionCount());
     tablet.initBitMaps();
 
+    boolean isAllNull = true;
+
     final TsBlock.TsBlockRowIterator rowIterator = 
tsBlock.getTsBlockRowIterator();
     while (rowIterator.hasNext()) {
       final Object[] row = rowIterator.next();
@@ -114,11 +116,19 @@ public class 
TsFileInsertionEventTableParserTabletIterator implements Iterator<T
       final int rowIndex = tablet.rowSize;
       tablet.addTimestamp(rowIndex, timestamp);
       for (int i = 0, fieldSize = row.length - 1; i < fieldSize; i++) {
-        tablet.addValue(columnNames.get(i), rowIndex, row[i]);
+        final Object value = row[i];
+        tablet.addValue(columnNames.get(i), rowIndex, value);
+        if (value != null && columnTypes.get(i) == 
Tablet.ColumnType.MEASUREMENT) {
+          isAllNull = false;
+        }
       }
       tablet.rowSize++;
     }
 
+    if (isAllNull) {
+      tablet.rowSize = 0;
+    }
+
     return tablet;
   }
 }
diff --git a/pom.xml b/pom.xml
index 2ea7ee7e908..b10fd9521e6 100644
--- a/pom.xml
+++ b/pom.xml
@@ -166,7 +166,7 @@
         <thrift.version>0.14.1</thrift.version>
         <xz.version>1.9</xz.version>
         <zstd-jni.version>1.5.6-3</zstd-jni.version>
-        <tsfile.version>1.2.0-241018-SNAPSHOT</tsfile.version>
+        <tsfile.version>1.2.0-4e902e0-SNAPSHOT</tsfile.version>
     </properties>
     <!--
     if we claim dependencies in dependencyManagement, then we do not claim