This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
The following commit(s) were added to refs/heads/master by this push:
new 8398fa9 [Bug] Fix bug that DbTxnMgr does not create for db in
CatalogRecycleBin
8398fa9 is described below
commit 8398fa9b755ad54e1ae8e0d150b19302ea84e5b7
Author: Mingyu Chen <[email protected]>
AuthorDate: Mon May 18 14:49:49 2020 +0800
[Bug] Fix bug that DbTxnMgr does not create for db in CatalogRecycleBin
Fix #3589
The reason is that:
When load meta from image, we will create `DatabaseTransactionMgr` for each
database
loaded from `loadDb()` method. But we forget to create
`DatabaseTransactionMgr` for
database in the catalog recycle bin.
NOTICE(#3622):
This is a "revert of revert pull request".
This pr is mainly used to synthesize the PRs whose commits were
scattered and submitted due to the wrong merge method into a complete
single commit.
---
.../java/org/apache/doris/catalog/Catalog.java | 9 ++++++--
.../apache/doris/catalog/CatalogRecycleBin.java | 9 ++++++++
.../org/apache/doris/qe/AuditEventProcessor.java | 1 +
.../doris/transaction/GlobalTransactionMgr.java | 26 +++++++++++++---------
4 files changed, 32 insertions(+), 13 deletions(-)
diff --git a/fe/src/main/java/org/apache/doris/catalog/Catalog.java
b/fe/src/main/java/org/apache/doris/catalog/Catalog.java
index b68c0d9..75df268 100755
--- a/fe/src/main/java/org/apache/doris/catalog/Catalog.java
+++ b/fe/src/main/java/org/apache/doris/catalog/Catalog.java
@@ -1803,10 +1803,15 @@ public class Catalog {
public long loadRecycleBin(DataInputStream dis, long checksum) throws
IOException {
if (Catalog.getCurrentCatalogJournalVersion() >=
FeMetaVersion.VERSION_10) {
- Catalog.getCurrentRecycleBin().readFields(dis);
+ recycleBin.readFields(dis);
if (!isCheckpointThread()) {
// add tablet in Recycle bin to TabletInvertedIndex
- Catalog.getCurrentRecycleBin().addTabletToInvertedIndex();
+ recycleBin.addTabletToInvertedIndex();
+ }
+ // create DatabaseTransactionMgr for db in recycle bin.
+ // these dbs do not exist in `idToDb` of the catalog.
+ for (Long dbId : recycleBin.getAllDbIds()) {
+ globalTransactionMgr.addDatabaseTransactionMgr(dbId);
}
}
return checksum;
diff --git a/fe/src/main/java/org/apache/doris/catalog/CatalogRecycleBin.java
b/fe/src/main/java/org/apache/doris/catalog/CatalogRecycleBin.java
index fae59a0..2307908 100644
--- a/fe/src/main/java/org/apache/doris/catalog/CatalogRecycleBin.java
+++ b/fe/src/main/java/org/apache/doris/catalog/CatalogRecycleBin.java
@@ -36,6 +36,7 @@ import org.apache.doris.task.DropReplicaTask;
import org.apache.doris.thrift.TStorageMedium;
import com.google.common.base.Preconditions;
+import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Range;
import com.google.common.collect.Sets;
@@ -172,6 +173,9 @@ public class CatalogRecycleBin extends MasterDaemon
implements Writable {
iterator.remove();
idToRecycleTime.remove(entry.getKey());
+ // remove database transaction manager
+
Catalog.getCurrentCatalog().getGlobalTransactionMgr().removeDatabaseTransactionMgr(db.getId());
+
LOG.info("erase database[{}] name: {}", db.getId(), dbName);
}
}
@@ -887,4 +891,9 @@ public class CatalogRecycleBin extends MasterDaemon
implements Writable {
}
}
}
+
+ // currently only used when loading image. So no synchronized protected.
+ public List<Long> getAllDbIds() {
+ return Lists.newArrayList(idToDatabase.keySet());
+ }
}
diff --git a/fe/src/main/java/org/apache/doris/qe/AuditEventProcessor.java
b/fe/src/main/java/org/apache/doris/qe/AuditEventProcessor.java
index bc621e7..a648dc9 100644
--- a/fe/src/main/java/org/apache/doris/qe/AuditEventProcessor.java
+++ b/fe/src/main/java/org/apache/doris/qe/AuditEventProcessor.java
@@ -56,6 +56,7 @@ public class AuditEventProcessor {
public void start() {
workerThread = new Thread(new Worker(), "AuditEventProcessor");
+ workerThread.setDaemon(true);
workerThread.start();
}
diff --git
a/fe/src/main/java/org/apache/doris/transaction/GlobalTransactionMgr.java
b/fe/src/main/java/org/apache/doris/transaction/GlobalTransactionMgr.java
index 5f8fd92..456c927 100644
--- a/fe/src/main/java/org/apache/doris/transaction/GlobalTransactionMgr.java
+++ b/fe/src/main/java/org/apache/doris/transaction/GlobalTransactionMgr.java
@@ -79,11 +79,15 @@ public class GlobalTransactionMgr implements Writable {
}
public void addDatabaseTransactionMgr(Long dbId) {
- dbIdToDatabaseTransactionMgrs.putIfAbsent(dbId, new
DatabaseTransactionMgr(dbId, catalog, idGenerator));
+ if (dbIdToDatabaseTransactionMgrs.putIfAbsent(dbId, new
DatabaseTransactionMgr(dbId, catalog, idGenerator)) == null) {
+ LOG.debug("add database transaction manager for db {}", dbId);
+ }
}
public void removeDatabaseTransactionMgr(Long dbId) {
- dbIdToDatabaseTransactionMgrs.remove(dbId);
+ if (dbIdToDatabaseTransactionMgrs.remove(dbId) != null) {
+ LOG.debug("remove database transaction manager for db {}", dbId);
+ }
}
public long beginTransaction(long dbId, List<Long> tableIdList, String
label, TxnCoordinator coordinator, LoadJobSourceType sourceType,
@@ -355,19 +359,19 @@ public class GlobalTransactionMgr implements Writable {
}
public void readFields(DataInput in) throws IOException {
- try {
- int numTransactions = in.readInt();
- for (int i = 0; i < numTransactions; ++i) {
- TransactionState transactionState = new TransactionState();
- transactionState.readFields(in);
+ int numTransactions = in.readInt();
+ for (int i = 0; i < numTransactions; ++i) {
+ TransactionState transactionState = new TransactionState();
+ transactionState.readFields(in);
+ try {
DatabaseTransactionMgr dbTransactionMgr =
getDatabaseTransactionMgr(transactionState.getDbId());
dbTransactionMgr.unprotectUpsertTransactionState(transactionState, true);
+ } catch (AnalysisException e) {
+ LOG.warn("failed to get db transaction manager for txn: {}",
transactionState);
+ throw new IOException("Read transaction states failed", e);
}
- idGenerator.readFields(in);
- } catch (AnalysisException e) {
- throw new IOException("Read transaction states failed", e);
}
-
+ idGenerator.readFields(in);
}
public TransactionState getTransactionStateByCallbackIdAndStatus(long
dbId, long callbackId, Set<TransactionStatus> status) {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
