This is an automated email from the ASF dual-hosted git repository.
rong pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new 22cf9449c40 Pipe: Improve performance when syncing table data between
clusters with param 'table-name' = '.*' or with param 'database-name' = '.*' by
reducing unnecessary tsfile parse & Fix data filter with both tree and table
patterns (#14150)
22cf9449c40 is described below
commit 22cf9449c40dbd5376f6b4a52f64897b41544065
Author: Zhenyu Luo <[email protected]>
AuthorDate: Fri Nov 22 00:50:23 2024 +0800
Pipe: Improve performance when syncing table data between clusters with
param 'table-name' = '.*' or with param 'database-name' = '.*' by reducing
unnecessary tsfile parse & Fix data filter with both tree and table patterns
(#14150)
Co-authored-by: Steve Yurong Su <[email protected]>
---
...oricalDataRegionTsFileAndDeletionExtractor.java | 5 ++--
.../realtime/PipeRealtimeDataRegionExtractor.java | 5 ++--
.../config/constant/PipeExtractorConstant.java | 2 ++
.../pipe/datastructure/pattern/TablePattern.java | 28 +++++++++++++++++-----
4 files changed, 30 insertions(+), 10 deletions(-)
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/historical/PipeHistoricalDataRegionTsFileAndDeletionExtractor.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/historical/PipeHistoricalDataRegionTsFileAndDeletionExtractor.java
index 61b721f8877..8ad6bda37c4 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/historical/PipeHistoricalDataRegionTsFileAndDeletionExtractor.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/historical/PipeHistoricalDataRegionTsFileAndDeletionExtractor.java
@@ -336,9 +336,10 @@ public class
PipeHistoricalDataRegionTsFileAndDeletionExtractor
final String databaseName = dataRegion.getDatabaseName();
if (Objects.nonNull(databaseName)) {
isDbNameCoveredByPattern =
- treePattern.coversDb(databaseName)
+ treePattern.isTreeModelDataAllowedToBeCaptured() &&
treePattern.coversDb(databaseName)
// The database name is prefixed with "root."
- && tablePattern.coversDb(databaseName.substring(5));
+ || tablePattern.isTableModelDataAllowedToBeCaptured()
+ && tablePattern.coversDb(databaseName.substring(5));
}
}
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/realtime/PipeRealtimeDataRegionExtractor.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/realtime/PipeRealtimeDataRegionExtractor.java
index 568e863acf4..45661ad1246 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/realtime/PipeRealtimeDataRegionExtractor.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/realtime/PipeRealtimeDataRegionExtractor.java
@@ -221,9 +221,10 @@ public abstract class PipeRealtimeDataRegionExtractor
implements PipeExtractor {
final String databaseName = dataRegion.getDatabaseName();
if (databaseName != null) {
isDbNameCoveredByPattern =
- treePattern.coversDb(databaseName)
+ treePattern.isTreeModelDataAllowedToBeCaptured() &&
treePattern.coversDb(databaseName)
// The database name is prefixed with "root."
- && tablePattern.coversDb(databaseName.substring(5));
+ || tablePattern.isTableModelDataAllowedToBeCaptured()
+ && tablePattern.coversDb(databaseName.substring(5));
}
}
diff --git
a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/config/constant/PipeExtractorConstant.java
b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/config/constant/PipeExtractorConstant.java
index e3df4f73934..bc57308f1e8 100644
---
a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/config/constant/PipeExtractorConstant.java
+++
b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/config/constant/PipeExtractorConstant.java
@@ -59,6 +59,8 @@ public class PipeExtractorConstant {
public static final String SOURCE_DATABASE_NAME_KEY = "source.database-name";
public static final String EXTRACTOR_TABLE_NAME_KEY = "extractor.table-name";
public static final String SOURCE_TABLE_NAME_KEY = "source.table-name";
+ public static final String EXTRACTOR_DATABASE_NAME_DEFAULT_VALUE = ".*";
+ public static final String EXTRACTOR_TABLE_NAME_DEFAULT_VALUE = ".*";
public static final String EXTRACTOR_FORWARDING_PIPE_REQUESTS_KEY =
"extractor.forwarding-pipe-requests";
diff --git
a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/datastructure/pattern/TablePattern.java
b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/datastructure/pattern/TablePattern.java
index 0e3bf6e9bf9..c5031c44f2d 100644
---
a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/datastructure/pattern/TablePattern.java
+++
b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/datastructure/pattern/TablePattern.java
@@ -27,7 +27,9 @@ import org.apache.iotdb.pipe.api.exception.PipeException;
import java.util.Arrays;
import java.util.regex.Pattern;
+import static
org.apache.iotdb.commons.pipe.config.constant.PipeExtractorConstant.EXTRACTOR_DATABASE_NAME_DEFAULT_VALUE;
import static
org.apache.iotdb.commons.pipe.config.constant.PipeExtractorConstant.EXTRACTOR_DATABASE_NAME_KEY;
+import static
org.apache.iotdb.commons.pipe.config.constant.PipeExtractorConstant.EXTRACTOR_TABLE_NAME_DEFAULT_VALUE;
import static
org.apache.iotdb.commons.pipe.config.constant.PipeExtractorConstant.EXTRACTOR_TABLE_NAME_KEY;
import static
org.apache.iotdb.commons.pipe.config.constant.PipeExtractorConstant.SOURCE_DATABASE_NAME_KEY;
import static
org.apache.iotdb.commons.pipe.config.constant.PipeExtractorConstant.SOURCE_TABLE_NAME_KEY;
@@ -44,8 +46,16 @@ public class TablePattern {
final String databasePatternString,
final String tablePatternString) {
this.isTableModelDataAllowedToBeCaptured =
isTableModelDataAllowedToBeCaptured;
- databasePattern = databasePatternString == null ? null :
Pattern.compile(databasePatternString);
- tablePattern = tablePatternString == null ? null :
Pattern.compile(tablePatternString);
+ databasePattern =
+ databasePatternString == null
+ ||
databasePatternString.trim().equals(EXTRACTOR_DATABASE_NAME_DEFAULT_VALUE)
+ ? null
+ : Pattern.compile(databasePatternString);
+ tablePattern =
+ tablePatternString == null
+ ||
tablePatternString.trim().equals(EXTRACTOR_TABLE_NAME_DEFAULT_VALUE)
+ ? null
+ : Pattern.compile(tablePatternString);
}
public boolean isTableModelDataAllowedToBeCaptured() {
@@ -72,11 +82,13 @@ public class TablePattern {
}
public String getDatabasePattern() {
- return databasePattern == null ? ".*" : databasePattern.pattern();
+ return databasePattern == null
+ ? EXTRACTOR_DATABASE_NAME_DEFAULT_VALUE
+ : databasePattern.pattern();
}
public String getTablePattern() {
- return tablePattern == null ? ".*" : tablePattern.pattern();
+ return tablePattern == null ? EXTRACTOR_TABLE_NAME_DEFAULT_VALUE :
tablePattern.pattern();
}
/**
@@ -96,9 +108,13 @@ public class TablePattern {
SystemConstant.SQL_DIALECT_KEY,
SystemConstant.SQL_DIALECT_TREE_VALUE)
.equals(SystemConstant.SQL_DIALECT_TREE_VALUE));
final String databaseNamePattern =
- sourceParameters.getStringByKeys(EXTRACTOR_DATABASE_NAME_KEY,
SOURCE_DATABASE_NAME_KEY);
+ sourceParameters.getStringOrDefault(
+ Arrays.asList(EXTRACTOR_DATABASE_NAME_KEY,
SOURCE_DATABASE_NAME_KEY),
+ EXTRACTOR_DATABASE_NAME_DEFAULT_VALUE);
final String tableNamePattern =
- sourceParameters.getStringByKeys(EXTRACTOR_TABLE_NAME_KEY,
SOURCE_TABLE_NAME_KEY);
+ sourceParameters.getStringOrDefault(
+ Arrays.asList(EXTRACTOR_TABLE_NAME_KEY, SOURCE_TABLE_NAME_KEY),
+ EXTRACTOR_TABLE_NAME_DEFAULT_VALUE);
try {
return new TablePattern(
isTableModelDataAllowedToBeCaptured, databaseNamePattern,
tableNamePattern);
