This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.0 by this push:
new cb401018f67 Revert "[fix](cache) Follower FE sql cache not invalidated
on table metadata replay (#63612)" (#63872)
cb401018f67 is described below
commit cb401018f67bc7608bf979f27c42c5098151bc30
Author: Mingyu Chen (Rayner) <[email protected]>
AuthorDate: Fri May 29 14:37:41 2026 +0800
Revert "[fix](cache) Follower FE sql cache not invalidated on table
metadata replay (#63612)" (#63872)
Reverts apache/doris#63657
---
.../main/java/org/apache/doris/common/Config.java | 12 --
.../main/java/org/apache/doris/alter/Alter.java | 4 +-
.../main/java/org/apache/doris/catalog/Env.java | 42 +-----
.../doris/common/cache/NereidsSqlCacheManager.java | 28 ++--
.../org/apache/doris/journal/JournalEntity.java | 6 -
.../org/apache/doris/nereids/SqlCacheContext.java | 8 +-
.../java/org/apache/doris/persist/EditLog.java | 9 --
.../org/apache/doris/persist/OperationType.java | 5 -
.../org/apache/doris/persist/TableMetaChange.java | 153 ---------------------
9 files changed, 18 insertions(+), 249 deletions(-)
diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
index 6a9694c5469..63d8134757f 100644
--- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
+++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
@@ -2413,18 +2413,6 @@ public class Config extends ConfigBase {
)
public static int sql_cache_manage_num = 100;
- @ConfField(
- mutable = true,
- description = {
- "是否在 DDL 时写入 OP_TABLE_META_CHANGE edit log 通知 follower FE
清理 sql cache。"
- + "默认 false，开启后 master DDL 会广播表元数据变更信号到所有
follower",
- "Whether to write OP_TABLE_META_CHANGE edit log on DDL to
notify follower FEs "
- + "to invalidate sql cache. Default false. When
enabled, master DDL broadcasts "
- + "table metadata change signal to all followers"
- }
- )
- public static boolean enable_write_op_table_meta_change = false;
-
@ConfField(
mutable = true,
callbackClassString =
"org.apache.doris.common.cache.NereidsSortedPartitionsCacheManager$UpdateConfig",
diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java
b/fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java
index 4383fda5df5..53681831b34 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java
@@ -72,6 +72,7 @@ import org.apache.doris.common.Config;
import org.apache.doris.common.DdlException;
import org.apache.doris.common.MetaNotFoundException;
import org.apache.doris.common.UserException;
+import org.apache.doris.common.cache.NereidsSqlCacheManager;
import org.apache.doris.common.util.DynamicPartitionUtil;
import org.apache.doris.common.util.MetaLockUtils;
import org.apache.doris.common.util.PropertyAnalyzer;
@@ -362,7 +363,8 @@ public class Alter {
olapTable.writeLock();
try {
- Env.getCurrentEnv().notifyTableMetaChange(olapTable);
+ NereidsSqlCacheManager sqlCacheManager =
Env.getCurrentEnv().getSqlCacheManager();
+ sqlCacheManager.invalidateAboutTable(olapTable);
} finally {
olapTable.writeUnlock();
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
index 082394c7f11..a6108e2d1b2 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
@@ -220,7 +220,6 @@ import org.apache.doris.persist.SetTableStatusOperationLog;
import org.apache.doris.persist.Storage;
import org.apache.doris.persist.StorageInfo;
import org.apache.doris.persist.TableInfo;
-import org.apache.doris.persist.TableMetaChange;
import org.apache.doris.persist.TablePropertyInfo;
import org.apache.doris.persist.TableRenameColumnInfo;
import org.apache.doris.persist.TruncateTableInfo;
@@ -6935,9 +6934,7 @@ public class Env {
LOG.warn("ignore set same state {} for table {}. is replay:
{}.",
olapTable.getState(), tableName, isReplay);
}
- if (!isReplay) {
- notifyTableMetaChange(olapTable);
- }
+
Env.getCurrentEnv().getSqlCacheManager().invalidateAboutTable(olapTable);
} finally {
olapTable.writeUnlock();
}
@@ -7045,9 +7042,7 @@ public class Env {
LOG.info("set replica {} of tablet {} on backend {} as version
{}, last success version {}, "
+ "last failed version {}, update time {}. is replay:
{}", replica.getId(), tabletId,
backendId, version, lastSuccessVersion,
lastFailedVersion, updateTime, isReplay);
- if (!isReplay) {
- notifyTableMetaChange(table);
- }
+
Env.getCurrentEnv().getSqlCacheManager().invalidateAboutTable(table);
} finally {
table.writeUnlock();
}
@@ -7128,9 +7123,7 @@ public class Env {
+ " {}.", partitionId, oldVersion, visibleVersion,
database, table, isReplay);
}
- if (!isReplay) {
- notifyTableMetaChange(olapTable);
- }
+
Env.getCurrentEnv().getSqlCacheManager().invalidateAboutTable(olapTable);
} finally {
olapTable.writeUnlock();
}
@@ -7359,35 +7352,6 @@ public class Env {
return sortedPartitionsCacheManager;
}
- public void notifyTableMetaChange(TableIf table) {
- if (table == null) {
- return;
- }
- TableMetaChange change =
- TableMetaChange.fromTable(table);
- fanOutTableMetaChange(change);
- if (isMaster() && editLog != null &&
Config.enable_write_op_table_meta_change) {
- editLog.logTableMetaChange(change);
- }
- }
-
- public void replayTableMetaChange(TableMetaChange change) {
- if (change == null) {
- return;
- }
- fanOutTableMetaChange(change);
- }
-
- private void fanOutTableMetaChange(TableMetaChange change) {
- if (sqlCacheManager != null) {
- sqlCacheManager.invalidateAboutTable(change);
- }
- if (sortedPartitionsCacheManager != null) {
- sortedPartitionsCacheManager.invalidateTable(
- change.getCatalogName(), change.getDbName(),
change.getTableName());
- }
- }
-
public SplitSourceManager getSplitSourceManager() {
return splitSourceManager;
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/common/cache/NereidsSqlCacheManager.java
b/fe/fe-core/src/main/java/org/apache/doris/common/cache/NereidsSqlCacheManager.java
index af44c008b2e..fe8f01b7254 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/common/cache/NereidsSqlCacheManager.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/common/cache/NereidsSqlCacheManager.java
@@ -59,7 +59,6 @@ import org.apache.doris.nereids.trees.plans.RelationId;
import org.apache.doris.nereids.trees.plans.logical.LogicalEmptyRelation;
import org.apache.doris.nereids.trees.plans.logical.LogicalSqlCache;
import org.apache.doris.nereids.util.Utils;
-import org.apache.doris.persist.TableMetaChange;
import org.apache.doris.proto.InternalService;
import org.apache.doris.proto.Types.PUniqueId;
import org.apache.doris.qe.ConnectContext;
@@ -113,31 +112,29 @@ public class NereidsSqlCacheManager {
}
public void invalidateAboutTable(TableIf tableIf) {
- invalidateAboutTable(TableMetaChange.fromTable(tableIf));
- }
-
- public void invalidateAboutTable(TableMetaChange event) {
+ Set<String> invalidateKeys = new LinkedHashSet<>();
FullTableName invalidateTableName = null;
- if (event.getCatalogName() != null && event.getDbName() != null &&
event.getTableName() != null) {
- invalidateTableName = new FullTableName(
- event.getCatalogName(), event.getDbName(),
event.getTableName());
+ DatabaseIf database = tableIf.getDatabase();
+ if (database != null) {
+ CatalogIf catalog = database.getCatalog();
+ if (catalog != null) {
+ invalidateTableName = new FullTableName(
+ database.getCatalog().getName(),
database.getFullName(), tableIf.getName()
+ );
+ }
}
- Set<String> invalidateKeys = new LinkedHashSet<>();
for (Entry<String, SqlCacheContext> kv : sqlCaches.asMap().entrySet())
{
String key = kv.getKey();
SqlCacheContext context = kv.getValue();
- if (context == null) {
- continue;
- }
for (Entry<FullTableName, TableVersion> nameToVersion :
context.getUsedTables().entrySet()) {
FullTableName tableName = nameToVersion.getKey();
TableVersion tableVersion = nameToVersion.getValue();
- if (tableVersion.id == event.getTableId()) {
+ if (tableVersion.id == tableIf.getId()) {
invalidateKeys.add(key);
break;
}
- if (invalidateTableName != null &&
tableName.equals(invalidateTableName)) {
+ if (tableName.equals(invalidateTableName)) {
invalidateKeys.add(key);
break;
}
@@ -468,9 +465,6 @@ public class NereidsSqlCacheManager {
if (currentTableVersion != cacheTableVersion) {
return IsChanged.CHANGED_AND_INVALIDATE_CACHE;
}
- if (olapTable.getBaseSchemaVersion() !=
tableVersion.schemaVersion) {
- return IsChanged.CHANGED_AND_INVALIDATE_CACHE;
- }
if (tableIf instanceof MTMV) {
// mtmv maybe access old data when grace_period > 0, we
should disable cache at this case
long gracePeriod = ((MTMV) tableIf).getGracePeriod();
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/journal/JournalEntity.java
b/fe/fe-core/src/main/java/org/apache/doris/journal/JournalEntity.java
index 69e27b65e78..cb6dee000de 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/journal/JournalEntity.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/journal/JournalEntity.java
@@ -124,7 +124,6 @@ import org.apache.doris.persist.TableAddOrDropColumnsInfo;
import org.apache.doris.persist.TableAddOrDropInvertedIndicesInfo;
import org.apache.doris.persist.TableBranchOrTagInfo;
import org.apache.doris.persist.TableInfo;
-import org.apache.doris.persist.TableMetaChange;
import org.apache.doris.persist.TablePropertyInfo;
import org.apache.doris.persist.TableRenameColumnInfo;
import org.apache.doris.persist.TableStatsDeletionLog;
@@ -993,11 +992,6 @@ public class JournalEntity implements Writable {
isRead = true;
break;
}
- case OperationType.OP_TABLE_META_CHANGE: {
- data = TableMetaChange.read(in);
- isRead = true;
- break;
- }
default: {
IOException e = new IOException();
LOG.error("UNKNOWN Operation Type {}", opCode, e);
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/SqlCacheContext.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/SqlCacheContext.java
index 8828c5b711b..e532ce611fe 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/SqlCacheContext.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/SqlCacheContext.java
@@ -203,17 +203,12 @@ public class SqlCacheContext {
LOG.warn("table {}, can not get version", tableIf.getName(), e);
}
- int schemaVersion = 0;
- if (tableIf instanceof OlapTable) {
- schemaVersion = ((OlapTable) tableIf).getBaseSchemaVersion();
- }
usedTables.put(
new FullTableName(database.getCatalog().getName(),
database.getFullName(), tableIf.getName()),
new TableVersion(
tableIf.getId(),
version,
- tableIf.getType(),
- schemaVersion
+ tableIf.getType()
)
);
}
@@ -598,7 +593,6 @@ public class SqlCacheContext {
public final long id;
public final long version;
public final TableType type;
- public final int schemaVersion;
}
/** CacheKeyType */
diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java
b/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java
index 0aad4b44ea6..95885f92286 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java
@@ -1418,11 +1418,6 @@ public class EditLog {
// TODO: implement
break;
}
- case OperationType.OP_TABLE_META_CHANGE: {
- TableMetaChange op = (TableMetaChange) journal.getData();
- env.replayTableMetaChange(op);
- break;
- }
default: {
IOException e = new IOException();
LOG.error("UNKNOWN Operation Type {}, log id: {}", opCode,
logId, e);
@@ -2509,8 +2504,4 @@ public class EditLog {
public long logBeginSnapshot(SnapshotState snapshotState) {
return logEdit(OperationType.OP_BEGIN_SNAPSHOT, snapshotState);
}
-
- public void logTableMetaChange(TableMetaChange op) {
- logEdit(OperationType.OP_TABLE_META_CHANGE, op);
- }
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/persist/OperationType.java
b/fe/fe-core/src/main/java/org/apache/doris/persist/OperationType.java
index e4f009274f3..fc42ce65163 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/persist/OperationType.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/persist/OperationType.java
@@ -425,11 +425,6 @@ public class OperationType {
public static final short OP_BEGIN_SNAPSHOT = 1100;
- // Generic "an operation modified this table's metadata" signal broadcast
from
- // master to followers so that every FE-local cache keyed by table can be
- // invalidated (NereidsSqlCacheManager,
NereidsSortedPartitionsCacheManager, …).
- public static final short OP_TABLE_META_CHANGE = 1102;
-
/**
* Get opcode name by op code.
**/
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/persist/TableMetaChange.java
b/fe/fe-core/src/main/java/org/apache/doris/persist/TableMetaChange.java
deleted file mode 100644
index cc62417d92f..00000000000
--- a/fe/fe-core/src/main/java/org/apache/doris/persist/TableMetaChange.java
+++ /dev/null
@@ -1,153 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.persist;
-
-import org.apache.doris.catalog.DatabaseIf;
-import org.apache.doris.catalog.TableIf;
-import org.apache.doris.common.io.Text;
-import org.apache.doris.common.io.Writable;
-import org.apache.doris.datasource.CatalogIf;
-import org.apache.doris.persist.gson.GsonUtils;
-
-import com.google.gson.annotations.SerializedName;
-
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-
-/**
- * Persist payload for {@link OperationType#OP_TABLE_META_OPERATION}.
- * Generic "an operation modified this table's metadata" signal that follower
- * FEs use to invalidate any local FE caches keyed by the table (sql cache,
- * sorted partition cache, and future per-table caches). This is about
- * metadata mutations (schema/properties/partitions/etc.), not data writes.
- * Carries both ids and names of catalog / database / table so each subscriber
- * can match by id (preferred) or by full name (fallback, e.g. when the table
- * has been concurrently dropped/recreated and the id no longer matches but
- * the name still does). Also carries the master-side timestamp so subscribers
- * and audit tooling can correlate the event with the originating DDL.
- */
-public class TableMetaChange implements Writable {
- @SerializedName("ci")
- private long catalogId;
- @SerializedName("cn")
- private String catalogName;
- @SerializedName("di")
- private long dbId;
- @SerializedName("dn")
- private String dbName;
- @SerializedName("ti")
- private long tableId;
- @SerializedName("tn")
- private String tableName;
- // master-side millis-since-epoch when this event was emitted
- @SerializedName("ts")
- private long eventTimeMs;
-
- public TableMetaChange() {
- // for persist
- }
-
- /** Build a TableMetaChange from a TableIf (master-side helper). */
- public static TableMetaChange fromTable(TableIf table) {
- long catalogId = -1L;
- String catalogName = "";
- long dbId = -1L;
- String dbName = "";
- DatabaseIf<?> db = table.getDatabase();
- if (db != null) {
- dbId = db.getId();
- dbName = db.getFullName();
- CatalogIf<?> catalog = db.getCatalog();
- if (catalog != null) {
- catalogId = catalog.getId();
- catalogName = catalog.getName();
- }
- }
- return new TableMetaChange(catalogId, catalogName, dbId, dbName,
- table.getId(), table.getName());
- }
-
- public TableMetaChange(long catalogId, String catalogName,
- long dbId, String dbName,
- long tableId, String tableName) {
- this(catalogId, catalogName, dbId, dbName, tableId, tableName,
System.currentTimeMillis());
- }
-
- public TableMetaChange(long catalogId, String catalogName,
- long dbId, String dbName,
- long tableId, String tableName,
- long eventTimeMs) {
- this.catalogId = catalogId;
- this.catalogName = catalogName;
- this.dbId = dbId;
- this.dbName = dbName;
- this.tableId = tableId;
- this.tableName = tableName;
- this.eventTimeMs = eventTimeMs;
- }
-
- public long getCatalogId() {
- return catalogId;
- }
-
- public String getCatalogName() {
- return catalogName;
- }
-
- public long getDbId() {
- return dbId;
- }
-
- public String getDbName() {
- return dbName;
- }
-
- public long getTableId() {
- return tableId;
- }
-
- public String getTableName() {
- return tableName;
- }
-
- public long getEventTimeMs() {
- return eventTimeMs;
- }
-
- @Override
- public void write(DataOutput out) throws IOException {
- Text.writeString(out, GsonUtils.GSON.toJson(this));
- }
-
- public static TableMetaChange read(DataInput in) throws IOException {
- return GsonUtils.GSON.fromJson(Text.readString(in),
TableMetaChange.class);
- }
-
- @Override
- public String toString() {
- return "TableMetaChange{catalogId=" + catalogId
- + ", catalogName='" + catalogName + '\''
- + ", dbId=" + dbId
- + ", dbName='" + dbName + '\''
- + ", tableId=" + tableId
- + ", tableName='" + tableName + '\''
- + ", eventTimeMs=" + eventTimeMs
- + '}';
- }
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
