This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 23fd525be8f [chore](fe) Add ATTN logs about partition range (#37338)
23fd525be8f is described below
commit 23fd525be8fd9b4bd519326ebc27c6cb3b021b76
Author: walter <[email protected]>
AuthorDate: Fri Jul 5 19:16:36 2024 +0800
[chore](fe) Add ATTN logs about partition range (#37338)
The CCR syncer relies on the value of the partition range to determine
the mapping relationship between the upstream partition and the
downstream partition. This PR records this behavior in the corresponding
code.
---
.../java/org/apache/doris/catalog/ListPartitionItem.java | 16 +++++++++++++---
.../java/org/apache/doris/catalog/PartitionInfo.java | 10 ++++++++++
.../java/org/apache/doris/catalog/PartitionItem.java | 3 +++
.../main/java/org/apache/doris/catalog/PartitionKey.java | 5 +++++
.../org/apache/doris/catalog/RangePartitionItem.java | 9 +++++++++
.../apache/doris/transaction/DatabaseTransactionMgr.java | 16 ++++------------
6 files changed, 44 insertions(+), 15 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/catalog/ListPartitionItem.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/ListPartitionItem.java
index a7ffb96756f..491bcc74c6d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/ListPartitionItem.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/ListPartitionItem.java
@@ -60,6 +60,10 @@ public class ListPartitionItem extends PartitionItem {
return partitionKeys;
}
+ public String getItemsString() {
+ return toString();
+ }
+
@Override
public boolean isDefaultPartition() {
return isDefaultPartition;
@@ -113,9 +117,10 @@ public class ListPartitionItem extends PartitionItem {
partitionKey.toString(),
pos));
}
- if (!isDefaultPartition() &&
MTMVUtil.getExprTimeSec(partitionKey.getKeys().get(pos), dateFormatOptional)
- >= nowTruncSubSec) {
- // As long as one of the partitionKeys meets the requirements,
this partition needs to be retained
+ if (!isDefaultPartition()
+ &&
MTMVUtil.getExprTimeSec(partitionKey.getKeys().get(pos), dateFormatOptional) >=
nowTruncSubSec) {
+ // As long as one of the partitionKeys meets the requirements,
this partition
+ // needs to be retained
return true;
}
}
@@ -164,6 +169,11 @@ public class ListPartitionItem extends PartitionItem {
@Override
public String toString() {
+ // ATTN: DO NOT EDIT unless unless you explicitly guarantee
compatibility
+ // between different versions.
+ //
+ // the ccr syncer depends on this string to identify partitions
between two
+ // clusters (cluster versions may be different).
StringBuilder builder = new StringBuilder();
builder.append("partitionKeys: [");
for (PartitionKey partitionKey : partitionKeys) {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionInfo.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionInfo.java
index 8deaa6d5f41..448c03f1b3a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionInfo.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionInfo.java
@@ -154,6 +154,16 @@ public class PartitionInfo {
return item;
}
+ // Get the unique string of the partition range.
+ public String getPartitionRangeString(long partitionId) {
+ String partitionRange = "";
+ if (getType() == PartitionType.RANGE || getType() ==
PartitionType.LIST) {
+ PartitionItem item = getItem(partitionId);
+ partitionRange = item.getItemsString();
+ }
+ return partitionRange;
+ }
+
public PartitionItem getItemOrAnalysisException(long partitionId) throws
AnalysisException {
PartitionItem item = idToItem.get(partitionId);
if (item == null) {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionItem.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionItem.java
index dec52093807..5b24b1651c3 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionItem.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionItem.java
@@ -28,6 +28,9 @@ public abstract class PartitionItem implements
Comparable<PartitionItem> {
public static final Comparator<Map.Entry<Long, PartitionItem>>
ITEM_MAP_ENTRY_COMPARATOR =
Comparator.comparing(o -> ((ListPartitionItem)
o.getValue()).getItems().iterator().next());
+ // get the unique string of the partition item.
+ public abstract String getItemsString();
+
public abstract <T> T getItems();
public abstract PartitionItem getIntersect(PartitionItem newItem);
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionKey.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionKey.java
index 1f38ce045ee..45960c20b83 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionKey.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionKey.java
@@ -334,6 +334,11 @@ public class PartitionKey implements
Comparable<PartitionKey>, Writable {
@Override
public String toString() {
+ // ATTN: DO NOT EDIT unless unless you explicitly guarantee
compatibility
+ // between different versions.
+ //
+ // the ccr syncer depends on this string to identify partitions
between two
+ // clusters (cluster versions may be different).
StringBuilder builder = new StringBuilder();
builder.append("types: [");
builder.append(Joiner.on(", ").join(types));
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/catalog/RangePartitionItem.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/RangePartitionItem.java
index a28f9553a74..760e3111b6d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/RangePartitionItem.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/RangePartitionItem.java
@@ -45,6 +45,10 @@ public class RangePartitionItem extends PartitionItem {
return partitionKeyRange;
}
+ public String getItemsString() {
+ return toString();
+ }
+
@Override
public boolean isDefaultPartition() {
return false;
@@ -112,6 +116,11 @@ public class RangePartitionItem extends PartitionItem {
@Override
public String toString() {
+ // ATTN: DO NOT EDIT unless unless you explicitly guarantee
compatibility
+ // between different versions.
+ //
+ // the ccr syncer depends on this string to identify partitions
between two
+ // clusters (cluster versions may be different).
return partitionKeyRange.toString();
}
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/transaction/DatabaseTransactionMgr.java
b/fe/fe-core/src/main/java/org/apache/doris/transaction/DatabaseTransactionMgr.java
index 92249f8defc..df9802f5eaa 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/transaction/DatabaseTransactionMgr.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/transaction/DatabaseTransactionMgr.java
@@ -25,7 +25,6 @@ import org.apache.doris.catalog.MaterializedIndex;
import org.apache.doris.catalog.OlapTable;
import org.apache.doris.catalog.Partition;
import org.apache.doris.catalog.PartitionInfo;
-import org.apache.doris.catalog.PartitionType;
import org.apache.doris.catalog.Replica;
import org.apache.doris.catalog.Table;
import org.apache.doris.catalog.TableIf;
@@ -1466,12 +1465,9 @@ public class DatabaseTransactionMgr {
TableCommitInfo tableCommitInfo = new TableCommitInfo(tableId);
PartitionInfo tblPartitionInfo = table.getPartitionInfo();
for (long partitionId : tableToPartition.get(tableId)) {
- String partitionRange = "";
- if (tblPartitionInfo.getType() == PartitionType.RANGE
- || tblPartitionInfo.getType() == PartitionType.LIST) {
- partitionRange =
tblPartitionInfo.getItem(partitionId).getItems().toString();
- }
- PartitionCommitInfo partitionCommitInfo = new
PartitionCommitInfo(partitionId, partitionRange, -1, -1,
+ String partitionRange =
tblPartitionInfo.getPartitionRangeString(partitionId);
+ PartitionCommitInfo partitionCommitInfo = new
PartitionCommitInfo(
+ partitionId, partitionRange, -1, -1,
table.isTemporaryPartition(partitionId));
tableCommitInfo.addPartitionCommitInfo(partitionCommitInfo);
}
@@ -1489,11 +1485,7 @@ public class DatabaseTransactionMgr {
private PartitionCommitInfo generatePartitionCommitInfo(OlapTable table,
long partitionId, long partitionVersion) {
PartitionInfo tblPartitionInfo = table.getPartitionInfo();
- String partitionRange = "";
- if (tblPartitionInfo.getType() == PartitionType.RANGE
- || tblPartitionInfo.getType() == PartitionType.LIST) {
- partitionRange =
tblPartitionInfo.getItem(partitionId).getItems().toString();
- }
+ String partitionRange =
tblPartitionInfo.getPartitionRangeString(partitionId);
return new PartitionCommitInfo(partitionId, partitionRange,
partitionVersion, System.currentTimeMillis() /* use as
partition visible time */,
table.isTemporaryPartition(partitionId));
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
