This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new cb5a5a5c9be [enhance](catalog)Allow parallel running of insert
overwrite on the e… (#41742)
cb5a5a5c9be is described below
commit cb5a5a5c9be84320c0d9198b0a5c87711ddfb1c4
Author: zhangdong <[email protected]>
AuthorDate: Sat Oct 12 17:19:23 2024 +0800
[enhance](catalog)Allow parallel running of insert overwrite on the e…
(#41742)
…xternal table (#41575)
pick: https://github.com/apache/doris/pull/41575
---
.../insertoverwrite/InsertOverwriteManager.java | 8 +++++
.../InsertOverwriteManagerTest.java | 34 ++++++++++++++++------
2 files changed, 33 insertions(+), 9 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/insertoverwrite/InsertOverwriteManager.java
b/fe/fe-core/src/main/java/org/apache/doris/insertoverwrite/InsertOverwriteManager.java
index a00107c76a7..df16b8f1be2 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/insertoverwrite/InsertOverwriteManager.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/insertoverwrite/InsertOverwriteManager.java
@@ -289,6 +289,14 @@ public class InsertOverwriteManager extends MasterDaemon
implements Writable {
* @param table Run the table for insert overwrite
*/
public void recordRunningTableOrException(DatabaseIf db, TableIf table) {
+ // The logic of OlapTable executing insert overwrite is to create
temporary partitions,
+ // replace partitions, etc.
+ // If executed in parallel, it may cause problems such as not being
able to find temporary partitions.
+ // But in terms of external table, we don't care the internal logic of
execution,
+ // so there's no need to keep records
+ if (!(table instanceof OlapTable)) {
+ return;
+ }
long dbId = db.getId();
long tableId = table.getId();
runningLock.writeLock().lock();
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/insertoverwrite/InsertOverwriteManagerTest.java
b/fe/fe-core/src/test/java/org/apache/doris/insertoverwrite/InsertOverwriteManagerTest.java
index 4bf6c9f12d5..026f8213522 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/insertoverwrite/InsertOverwriteManagerTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/insertoverwrite/InsertOverwriteManagerTest.java
@@ -18,23 +18,27 @@
package org.apache.doris.insertoverwrite;
import org.apache.doris.catalog.DatabaseIf;
-import org.apache.doris.catalog.TableIf;
+import org.apache.doris.catalog.OlapTable;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.DdlException;
import org.apache.doris.common.MetaNotFoundException;
+import org.apache.doris.datasource.hive.HMSExternalTable;
import mockit.Expectations;
import mockit.Mocked;
-import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
+import org.junit.jupiter.api.Assertions;
public class InsertOverwriteManagerTest {
@Mocked
private DatabaseIf db;
@Mocked
- private TableIf table;
+ private OlapTable table;
+
+ @Mocked
+ private HMSExternalTable hmsExternalTable;
@Before
public void setUp()
@@ -57,6 +61,14 @@ public class InsertOverwriteManagerTest {
table.getName();
minTimes = 0;
result = "table1";
+
+ hmsExternalTable.getId();
+ minTimes = 0;
+ result = 3L;
+
+ hmsExternalTable.getName();
+ minTimes = 0;
+ result = "hmsTable";
}
};
}
@@ -65,13 +77,17 @@ public class InsertOverwriteManagerTest {
public void testParallel() {
InsertOverwriteManager manager = new InsertOverwriteManager();
manager.recordRunningTableOrException(db, table);
- try {
- manager.recordRunningTableOrException(db, table);
- } catch (Exception e) {
- Assert.assertTrue(e.getMessage().contains("Not allowed"));
- }
+
Assertions.assertThrows(org.apache.doris.nereids.exceptions.AnalysisException.class,
+ () -> manager.recordRunningTableOrException(db, table));
manager.dropRunningRecord(db.getId(), table.getId());
- manager.recordRunningTableOrException(db, table);
+ Assertions.assertDoesNotThrow(() ->
manager.recordRunningTableOrException(db, table));
}
+ @Test
+ public void testHmsTableParallel() {
+ InsertOverwriteManager manager = new InsertOverwriteManager();
+ manager.recordRunningTableOrException(db, hmsExternalTable);
+ Assertions.assertDoesNotThrow(() ->
manager.recordRunningTableOrException(db, hmsExternalTable));
+ manager.dropRunningRecord(db.getId(), hmsExternalTable.getId());
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
