This is an automated email from the ASF dual-hosted git repository.
hello-stephen pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new fe8b92b347b [fix](fe) Preserve recycled databases when recovery times
out (#70581)
fe8b92b347b is described below
commit fe8b92b347b8ffb2d1f412f50a41dc9100b45fce
Author: linrrarity <[email protected]>
AuthorDate: Mon Nov 16 22:59:56 2026 -0800
[fix](fe) Preserve recycled databases when recovery times out (#70581)
### Background
DROP DATABASE retains recoverable metadata in the catalog recycle bin.
RECOVER DATABASE restores its tables and publishes the database in the
internal catalog.
### What problem does this PR solve?
Issue Number: None
Related PR: None
PR Type: Bugfix
Problem Summary:
RECOVER DATABASE removed recycled metadata before attempting to acquire
the catalog lock. A lock timeout left the database absent from both the
active catalog and the recycle bin, so subsequent queries and recovery
attempts failed. Acquire the catalog lock and validate the target name
before restoring recycled metadata.
### Details
- **Frequency:** Triggered when RECOVER DATABASE times out waiting for a
catalog lock held by another DDL operation.
- **Root cause:** Recovery consumed the recycle-bin entry before
catalog-lock acquisition could fail.
- **Why existing tests missed it:** Existing recovery tests did not
exercise catalog-lock contention during database recovery.
- **Fix:** Hold the existing catalog lock across name validation and
metadata recovery, and release it from an outer finally block.
- **Other paths and branches:** Unknown-database failures release the
catalog lock. Recovery with a new database name follows the same
protected path.
### Testing
- Added a regression test using real catalog and recycle-bin objects
with a competing thread holding the catalog lock. It fails before the
fix because the recycled database has disappeared, and passes after the
fix.
- Ran RecoverDatabaseLockTest, DropDbTest, RecoverTest and
CatalogRecycleBinTest through run-fe-ut.sh: 38 tests passed.
- Completed a full FE/BE build with Checkstyle through the workspace
wrapper, which invokes build.sh --be --fe -j50.
- Verified a live single-FE/single-BE cluster with ordinary OLAP tables:
concurrent EXPLAIN, DROP DATABASE and RECOVER DATABASE produced the
expected lock timeout while preserving the recycled database and table.
Retrying recovery returned the original row (1, 5). Recovery with a new
name and DDL after an unknown-database recovery failure also succeeded.
### Release note
RECOVER DATABASE leaves the database available in the recycle bin when
catalog-lock acquisition times out, allowing a subsequent recovery
attempt.
### Check List (For Author)
- Behavior changed: Yes (lock-timeout failures preserve recycled
database metadata)
- Does this need documentation: No
- Pre-submission status: Ready
### AI Disclosure
- Tool: Codex
- Generated: Java change, unit tests and this description
- Human review: Pending
---
.../apache/doris/datasource/InternalCatalog.java | 66 +++++--------
.../doris/datasource/RecoverDatabaseLockTest.java | 106 +++++++++++++++++++++
2 files changed, 131 insertions(+), 41 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
index 53312b13e7f..1d654066324 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
@@ -610,56 +610,40 @@ public class InternalCatalog implements
CatalogIf<Database> {
}
public void recoverDatabase(String dbName, long dbId, String newDbName)
throws DdlException {
- // check is new db with same name already exist
- if (!Strings.isNullOrEmpty(newDbName)) {
- if (getDb(newDbName).isPresent()) {
- throw new DdlException("Database[" + newDbName + "] already
exist.");
- }
- } else {
- if (getDb(dbName).isPresent()) {
- throw new DdlException("Database[" + dbName + "] already
exist.");
- }
- }
-
- Database db = Env.getCurrentRecycleBin().recoverDatabase(dbName, dbId);
-
- // add db to catalog
if (!tryLock(false)) {
throw new DdlException("Failed to acquire catalog lock. Try
again");
}
- db.writeLock();
- List<Table> tableList = db.getTablesOnIdOrder();
- MetaLockUtils.writeLockTables(tableList);
try {
- if (!Strings.isNullOrEmpty(newDbName)) {
- if (fullNameToDb.containsKey(newDbName)) {
- throw new DdlException("Database[" + newDbName + "]
already exist.");
- // it's ok that we do not put db back to CatalogRecycleBin
- // cause this db cannot recover any more
- }
- } else {
- if (fullNameToDb.containsKey(db.getFullName())) {
- throw new DdlException("Database[" + db.getFullName() + "]
already exist.");
- // it's ok that we do not put db back to CatalogRecycleBin
- // cause this db cannot recover any more
- }
+ // check is new db with same name already exist
+ String targetDbName = Strings.isNullOrEmpty(newDbName) ? dbName :
newDbName;
+ if (getDb(targetDbName).isPresent()) {
+ throw new DdlException("Database[" + targetDbName + "] already
exist.");
}
- if (!Strings.isNullOrEmpty(newDbName)) {
- db.setNameWithLock(newDbName);
+
+ Database db = Env.getCurrentRecycleBin().recoverDatabase(dbName,
dbId);
+ db.writeLock();
+ List<Table> tableList = db.getTablesOnIdOrder();
+ MetaLockUtils.writeLockTables(tableList);
+ try {
+ if (!Strings.isNullOrEmpty(newDbName)) {
+ db.setNameWithLock(newDbName);
+ }
+ // add db to catalog
+ fullNameToDb.put(db.getFullName(), db);
+ idToDb.put(db.getId(), db);
+ // log
+ RecoverInfo recoverInfo = new RecoverInfo(db.getId(), -1L,
-1L, newDbName, "", "", "", "");
+ Env.getCurrentEnv().getEditLog().logRecoverDb(recoverInfo);
+ db.unmarkDropped();
+ registerDbFunctionsToNereids(db);
+ } finally {
+ MetaLockUtils.writeUnlockTables(tableList);
+ db.writeUnlock();
}
- fullNameToDb.put(db.getFullName(), db);
- idToDb.put(db.getId(), db);
- // log
- RecoverInfo recoverInfo = new RecoverInfo(db.getId(), -1L, -1L,
newDbName, "", "", "", "");
- Env.getCurrentEnv().getEditLog().logRecoverDb(recoverInfo);
- db.unmarkDropped();
- registerDbFunctionsToNereids(db);
+ LOG.info("recover database[{}]", db.getId());
} finally {
- MetaLockUtils.writeUnlockTables(tableList);
- db.writeUnlock();
unlock();
}
- LOG.info("recover database[{}]", db.getId());
}
public void recoverTable(String dbName, String tableName, String
newTableName, long tableId) throws DdlException {
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/datasource/RecoverDatabaseLockTest.java
b/fe/fe-core/src/test/java/org/apache/doris/datasource/RecoverDatabaseLockTest.java
new file mode 100644
index 00000000000..cfafad29e5a
--- /dev/null
+++
b/fe/fe-core/src/test/java/org/apache/doris/datasource/RecoverDatabaseLockTest.java
@@ -0,0 +1,106 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.datasource;
+
+import org.apache.doris.catalog.CatalogRecycleBin;
+import org.apache.doris.catalog.Column;
+import org.apache.doris.catalog.Database;
+import org.apache.doris.catalog.Env;
+import org.apache.doris.catalog.HashDistributionInfo;
+import org.apache.doris.catalog.KeysType;
+import org.apache.doris.catalog.OlapTable;
+import org.apache.doris.catalog.SinglePartitionInfo;
+import org.apache.doris.catalog.Type;
+import org.apache.doris.common.Config;
+import org.apache.doris.common.DdlException;
+
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+import java.lang.reflect.Field;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.locks.ReentrantLock;
+
+public class RecoverDatabaseLockTest {
+ @Test
+ public void testLockTimeoutKeepsDatabaseAndTableInRecycleBin() throws
Exception {
+ InternalCatalog catalog = Env.getCurrentInternalCatalog();
+ CatalogRecycleBin recycleBin = Env.getCurrentRecycleBin();
+ Database db = new Database(9100000L, "recover_catalog_lock_timeout");
+ Column column = new Column("k", Type.INT);
+ OlapTable table = new OlapTable(9100001L, "t", List.of(column),
KeysType.DUP_KEYS,
+ new SinglePartitionInfo(), new HashDistributionInfo(1,
List.of(column)));
+ Assertions.assertTrue(recycleBin.recycleTable(db.getId(), table,
false, false, 0));
+ Assertions.assertTrue(recycleBin.recycleDatabase(db,
Set.of(table.getName()), Set.of(table.getId()),
+ false, false, 0));
+ long recycleTime = recycleBin.getRecycleTimeById(db.getId());
+ long oldTimeout = Config.catalog_try_lock_timeout_ms;
+ ReentrantLock catalogLock = getCatalogLock(catalog);
+ ExecutorService executor = Executors.newSingleThreadExecutor();
+ catalogLock.lock();
+ try {
+ Config.catalog_try_lock_timeout_ms = 50;
+ DdlException error = executor.submit(() ->
Assertions.assertThrows(DdlException.class,
+ () -> catalog.recoverDatabase(db.getFullName(),
db.getId(), "")))
+ .get(5, TimeUnit.SECONDS);
+ Assertions.assertTrue(error.getMessage().contains("Failed to
acquire catalog lock"));
+ Assertions.assertNull(catalog.getDbNullable(db.getId()));
+ Assertions.assertTrue(recycleBin.isRecycleDatabase(db.getId()));
+ Assertions.assertTrue(recycleBin.isRecycleTable(db.getId(),
table.getId()));
+ Assertions.assertEquals(recycleTime,
recycleBin.getRecycleTimeById(db.getId()));
+ Assertions.assertTrue(db.getTables().isEmpty());
+ } finally {
+ Config.catalog_try_lock_timeout_ms = oldTimeout;
+ catalogLock.unlock();
+ executor.shutdownNow();
+ }
+
+ Assertions.assertSame(db, recycleBin.recoverDatabase(db.getFullName(),
db.getId()));
+ Assertions.assertSame(table, db.getTableNullable(table.getId()));
+ Assertions.assertFalse(recycleBin.isRecycleDatabase(db.getId()));
+ Assertions.assertFalse(recycleBin.isRecycleTable(db.getId(),
table.getId()));
+ }
+
+ @Test
+ public void testUnknownDatabaseReleasesCatalogLock() throws Exception {
+ InternalCatalog catalog = Env.getCurrentInternalCatalog();
+ Assertions.assertThrows(DdlException.class,
+ () ->
catalog.recoverDatabase("unknown_recover_catalog_database", -1, ""));
+ ReentrantLock catalogLock = getCatalogLock(catalog);
+ ExecutorService executor = Executors.newSingleThreadExecutor();
+ try {
+ executor.submit(() -> {
+ Assertions.assertTrue(catalogLock.tryLock(5,
TimeUnit.SECONDS));
+ catalogLock.unlock();
+ return null;
+ }).get(10, TimeUnit.SECONDS);
+ } finally {
+ executor.shutdownNow();
+ }
+ }
+
+ private ReentrantLock getCatalogLock(InternalCatalog catalog) throws
Exception {
+ Field field = InternalCatalog.class.getDeclaredField("lock");
+ field.setAccessible(true);
+ return (ReentrantLock) field.get(catalog);
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]