This is an automated email from the ASF dual-hosted git repository.
danny0405 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new ea59a08f4d5 [HUDI-6133] Eliminate one deletion operation if state is
not COMPLETED (#8563)
ea59a08f4d5 is described below
commit ea59a08f4d5ab346526afff77e67b0e29792bf6c
Author: Brisk Wong <[email protected]>
AuthorDate: Tue Jul 4 14:07:28 2023 +0800
[HUDI-6133] Eliminate one deletion operation if state is not COMPLETED
(#8563)
1. Eliminate one deletion operation if state is not COMPLETED;
2. Enable it to run unit tests repeatedly.
---
.../hudi/metaserver/service/TableService.java | 2 +-
.../hudi/metaserver/service/TimelineService.java | 8 ++------
.../hudi/metaserver/store/MetaserverStorage.java | 10 ++++++++++
.../metaserver/store/RelationalDBBasedStorage.java | 21 +++++++++++++++++++++
.../src/main/resources/mybatis/DDLMapper.xml | 16 ++++++++--------
.../src/main/resources/mybatis/TableMapper.xml | 16 ++++++++++++++++
.../src/main/resources/mybatis/TimelineMapper.xml | 8 ++++++++
.../store/TestRelationalDBBasedStore.java | 19 +++++++++++++++++--
8 files changed, 83 insertions(+), 17 deletions(-)
diff --git
a/hudi-platform-service/hudi-metaserver/hudi-metaserver-server/src/main/java/org/apache/hudi/metaserver/service/TableService.java
b/hudi-platform-service/hudi-metaserver/hudi-metaserver-server/src/main/java/org/apache/hudi/metaserver/service/TableService.java
index 23325e9582e..450abc574ca 100644
---
a/hudi-platform-service/hudi-metaserver/hudi-metaserver-server/src/main/java/org/apache/hudi/metaserver/service/TableService.java
+++
b/hudi-platform-service/hudi-metaserver/hudi-metaserver-server/src/main/java/org/apache/hudi/metaserver/service/TableService.java
@@ -32,7 +32,7 @@ import java.io.Serializable;
* Handle all database / table related requests.
*/
public class TableService implements Serializable {
- private MetaserverStorage store;
+ private final MetaserverStorage store;
public TableService(MetaserverStorage metaserverStorage) {
this.store = metaserverStorage;
diff --git
a/hudi-platform-service/hudi-metaserver/hudi-metaserver-server/src/main/java/org/apache/hudi/metaserver/service/TimelineService.java
b/hudi-platform-service/hudi-metaserver/hudi-metaserver-server/src/main/java/org/apache/hudi/metaserver/service/TimelineService.java
index b31ef50fdba..2ba98e86464 100644
---
a/hudi-platform-service/hudi-metaserver/hudi-metaserver-server/src/main/java/org/apache/hudi/metaserver/service/TimelineService.java
+++
b/hudi-platform-service/hudi-metaserver/hudi-metaserver-server/src/main/java/org/apache/hudi/metaserver/service/TimelineService.java
@@ -139,12 +139,8 @@ public class TimelineService implements Serializable {
Long tableId = MetaserverTableUtils.getTableId(store, db, tb);
HoodieInstantChangeResult result = new HoodieInstantChangeResult();
if (store.instantExists(tableId, instant)) {
- switch (instant.getState()) {
- case COMPLETED:
- store.deleteInstantAllMeta(tableId, instant.getTimestamp());
- break;
- default:
- store.deleteInstant(tableId, instant);
+ if (instant.getState() == TState.COMPLETED) {
+ store.deleteInstantAllMeta(tableId, instant.getTimestamp());
}
store.deleteInstant(tableId, instant);
} else {
diff --git
a/hudi-platform-service/hudi-metaserver/hudi-metaserver-server/src/main/java/org/apache/hudi/metaserver/store/MetaserverStorage.java
b/hudi-platform-service/hudi-metaserver/hudi-metaserver-server/src/main/java/org/apache/hudi/metaserver/store/MetaserverStorage.java
index fdd8d9546a7..e545b89ecc3 100644
---
a/hudi-platform-service/hudi-metaserver/hudi-metaserver-server/src/main/java/org/apache/hudi/metaserver/store/MetaserverStorage.java
+++
b/hudi-platform-service/hudi-metaserver/hudi-metaserver-server/src/main/java/org/apache/hudi/metaserver/store/MetaserverStorage.java
@@ -21,6 +21,7 @@ package org.apache.hudi.metaserver.store;
import org.apache.hudi.ApiMaturityLevel;
import org.apache.hudi.PublicAPIClass;
import org.apache.hudi.common.util.Option;
+import org.apache.hudi.common.util.VisibleForTesting;
import org.apache.hudi.metaserver.thrift.MetaserverStorageException;
import org.apache.hudi.metaserver.thrift.THoodieInstant;
import org.apache.hudi.metaserver.thrift.TState;
@@ -41,15 +42,24 @@ public interface MetaserverStorage extends AutoCloseable {
Long getDatabaseId(String db) throws MetaserverStorageException;
+ @VisibleForTesting
+ boolean deleteDatabase(Long dbId) throws MetaserverStorageException;
+
boolean createTable(Long dbId, Table table) throws
MetaserverStorageException;
Table getTable(String db, String tb) throws MetaserverStorageException;
Long getTableId(String db, String tb) throws MetaserverStorageException;
+ @VisibleForTesting
+ boolean deleteTable(Long tableId) throws MetaserverStorageException;
+
// timeline related
String createNewTimestamp(long tableId) throws MetaserverStorageException;
+ @VisibleForTesting
+ boolean deleteTableTimestamp(Long tableId) throws MetaserverStorageException;
+
boolean createInstant(long tableId, THoodieInstant instant) throws
MetaserverStorageException;
boolean updateInstant(long tableId, THoodieInstant fromInstant,
THoodieInstant toInstant) throws MetaserverStorageException;
diff --git
a/hudi-platform-service/hudi-metaserver/hudi-metaserver-server/src/main/java/org/apache/hudi/metaserver/store/RelationalDBBasedStorage.java
b/hudi-platform-service/hudi-metaserver/hudi-metaserver-server/src/main/java/org/apache/hudi/metaserver/store/RelationalDBBasedStorage.java
index cccb2a04534..e8ed65a47c0 100644
---
a/hudi-platform-service/hudi-metaserver/hudi-metaserver-server/src/main/java/org/apache/hudi/metaserver/store/RelationalDBBasedStorage.java
+++
b/hudi-platform-service/hudi-metaserver/hudi-metaserver-server/src/main/java/org/apache/hudi/metaserver/store/RelationalDBBasedStorage.java
@@ -75,6 +75,13 @@ public class RelationalDBBasedStorage implements
MetaserverStorage, Serializable
return ids.isEmpty() ? null : ids.get(0);
}
+ @Override
+ public boolean deleteDatabase(Long dbId) throws MetaserverStorageException {
+ Map<String, Object> params = new HashMap<>();
+ params.put("dbId", dbId);
+ return tableDao.deleteBySql("deleteDB", params) == 1;
+ }
+
@Override
public boolean createTable(Long dbId, Table table) throws
MetaserverStorageException {
Map<String, Object> params = new HashMap<>();
@@ -104,6 +111,13 @@ public class RelationalDBBasedStorage implements
MetaserverStorage, Serializable
return ids.isEmpty() ? null : ids.get(0);
}
+ @Override
+ public boolean deleteTable(Long tableId) throws MetaserverStorageException {
+ Map<String, Object> params = new HashMap<>();
+ params.put("tableId", tableId);
+ return tableDao.deleteBySql("deleteTable", params) == 1;
+ }
+
@Override
public String createNewTimestamp(long tableId) throws
MetaserverStorageException {
String oldTimestamp;
@@ -134,6 +148,13 @@ public class RelationalDBBasedStorage implements
MetaserverStorage, Serializable
return timestamps.isEmpty() ? null : timestamps.get(0);
}
+ @Override
+ public boolean deleteTableTimestamp(Long tableId) throws
MetaserverStorageException {
+ Map<String, Object> params = new HashMap<>();
+ params.put("tableId", tableId);
+ return timelineDao.deleteBySql("deleteTimestamp", params) == 1;
+ }
+
@Override
public boolean createInstant(long tableId, THoodieInstant instant) throws
MetaserverStorageException {
InstantBean instantBean = new InstantBean(tableId, instant);
diff --git
a/hudi-platform-service/hudi-metaserver/hudi-metaserver-server/src/main/resources/mybatis/DDLMapper.xml
b/hudi-platform-service/hudi-metaserver/hudi-metaserver-server/src/main/resources/mybatis/DDLMapper.xml
index 938328130f9..edaa933116f 100644
---
a/hudi-platform-service/hudi-metaserver/hudi-metaserver-server/src/main/resources/mybatis/DDLMapper.xml
+++
b/hudi-platform-service/hudi-metaserver/hudi-metaserver-server/src/main/resources/mybatis/DDLMapper.xml
@@ -20,7 +20,7 @@
<mapper namespace="DDLMapper">
<update id="createDBs">
- CREATE TABLE dbs
+ CREATE TABLE IF NOT EXISTS dbs
(
db_id BIGINT UNSIGNED PRIMARY KEY AUTO_INCREMENT COMMENT 'uuid',
description VARCHAR(512) COMMENT 'database description',
@@ -34,7 +34,7 @@
</update>
<update id="createTables">
- CREATE TABLE tbls
+ CREATE TABLE IF NOT EXISTS tbls
(
tbl_id BIGINT UNSIGNED PRIMARY KEY AUTO_INCREMENT COMMENT 'uuid',
db_id BIGINT COMMENT 'database id',
@@ -48,7 +48,7 @@
</update>
<update id="createTableParams">
- CREATE TABLE tbl_params
+ CREATE TABLE IF NOT EXISTS tbl_params
(
tbl_id BIGINT UNSIGNED COMMENT 'tbl id',
param_key VARCHAR(256) COMMENT 'param_key',
@@ -60,7 +60,7 @@
</update>
<update id="createPartitions">
- CREATE TABLE partitions
+ CREATE TABLE IF NOT EXISTS partitions
(
part_id BIGINT UNSIGNED PRIMARY KEY AUTO_INCREMENT COMMENT 'uuid',
tbl_id BIGINT COMMENT 'table id',
@@ -73,7 +73,7 @@
</update>
<update id="createTableTimestamp">
- CREATE TABLE tbl_timestamp
+ CREATE TABLE IF NOT EXISTS tbl_timestamp
(
tbl_id BIGINT UNSIGNED PRIMARY KEY COMMENT 'uuid',
ts VARCHAR(17) COMMENT 'instant timestamp'
@@ -81,7 +81,7 @@
</update>
<update id="createInstant">
- CREATE TABLE instant
+ CREATE TABLE IF NOT EXISTS instant
(
instant_id BIGINT UNSIGNED PRIMARY KEY AUTO_INCREMENT COMMENT
'uuid',
tbl_id BIGINT COMMENT 'table id',
@@ -96,7 +96,7 @@
</update>
<update id="createInstantMetadata">
- CREATE TABLE instant_meta_data
+ CREATE TABLE IF NOT EXISTS instant_meta_data
(
commit_id BIGINT UNSIGNED PRIMARY KEY AUTO_INCREMENT COMMENT
'uuid',
tbl_id BIGINT COMMENT 'table id',
@@ -110,7 +110,7 @@
</update>
<update id="createFiles">
- CREATE TABLE files
+ CREATE TABLE IF NOT EXISTS files
(
id BIGINT UNSIGNED PRIMARY KEY AUTO_INCREMENT COMMENT
'uuid',
tbl_id BIGINT COMMENT 'table id',
diff --git
a/hudi-platform-service/hudi-metaserver/hudi-metaserver-server/src/main/resources/mybatis/TableMapper.xml
b/hudi-platform-service/hudi-metaserver/hudi-metaserver-server/src/main/resources/mybatis/TableMapper.xml
index 130b9e3428f..b676c8eaad5 100644
---
a/hudi-platform-service/hudi-metaserver/hudi-metaserver-server/src/main/resources/mybatis/TableMapper.xml
+++
b/hudi-platform-service/hudi-metaserver/hudi-metaserver-server/src/main/resources/mybatis/TableMapper.xml
@@ -31,6 +31,14 @@
WHERE name=#{databaseName}
</select>
+ <delete id="deleteDB" parameterType="java.util.HashMap">
+ DELETE
+ FROM
+ dbs
+ WHERE
+ db_id = #{dbId}
+ </delete>
+
<!-- table related -->
<insert id="insertTable" parameterType="java.util.Map"
useGeneratedKeys="true" keyProperty="tbl_id">
INSERT INTO tbls (db_id, name, owner_name, location)
@@ -57,4 +65,12 @@
AND dbs.NAME = #{databaseName}
</select>
+ <delete id="deleteTable" parameterType="java.util.HashMap">
+ DELETE
+ FROM
+ tbls
+ WHERE
+ tbl_id = #{tableId}
+ </delete>
+
</mapper>
diff --git
a/hudi-platform-service/hudi-metaserver/hudi-metaserver-server/src/main/resources/mybatis/TimelineMapper.xml
b/hudi-platform-service/hudi-metaserver/hudi-metaserver-server/src/main/resources/mybatis/TimelineMapper.xml
index db36a47d628..ba81fb6c5bb 100644
---
a/hudi-platform-service/hudi-metaserver/hudi-metaserver-server/src/main/resources/mybatis/TimelineMapper.xml
+++
b/hudi-platform-service/hudi-metaserver/hudi-metaserver-server/src/main/resources/mybatis/TimelineMapper.xml
@@ -43,6 +43,14 @@
tbl_id = #{tableId}
</select>
+ <delete id="deleteTimestamp" parameterType="java.util.HashMap">
+ DELETE
+ FROM
+ tbl_timestamp
+ WHERE
+ tbl_id = #{tableId}
+ </delete>
+
<!-- instant related -->
<insert id="insertInstant" parameterType="java.util.HashMap"
useGeneratedKeys="true" keyProperty="instant_id">
INSERT INTO instant (tbl_id, action, ts, state, duration, start_ts)
diff --git
a/hudi-platform-service/hudi-metaserver/hudi-metaserver-server/src/test/java/org/apache/hudi/metaserver/store/TestRelationalDBBasedStore.java
b/hudi-platform-service/hudi-metaserver/hudi-metaserver-server/src/test/java/org/apache/hudi/metaserver/store/TestRelationalDBBasedStore.java
index 41944d46527..8f13498f41b 100644
---
a/hudi-platform-service/hudi-metaserver/hudi-metaserver-server/src/test/java/org/apache/hudi/metaserver/store/TestRelationalDBBasedStore.java
+++
b/hudi-platform-service/hudi-metaserver/hudi-metaserver-server/src/test/java/org/apache/hudi/metaserver/store/TestRelationalDBBasedStore.java
@@ -25,6 +25,8 @@ import org.apache.hudi.metaserver.thrift.THoodieInstant;
import org.apache.hudi.metaserver.thrift.TState;
import org.apache.hudi.metaserver.thrift.Table;
import org.apache.thrift.TException;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import java.nio.charset.StandardCharsets;
@@ -46,10 +48,14 @@ public class TestRelationalDBBasedStore {
private final String db = "test_db";
private final String tb = "test_tb";
- @Test
- public void testAPIs() throws TException {
+ @BeforeEach
+ public void setUp() {
HoodieMetaserver.getEmbeddedMetaserver();
store = HoodieMetaserver.getMetaserverStorage();
+ }
+
+ @Test
+ public void testAPIs() throws TException {
testTableRelatedAPIs();
testTimelineRelatedAPIs();
}
@@ -109,4 +115,13 @@ public class TestRelationalDBBasedStore {
assertFalse(store.getInstantMetadata(tableId, inflight).isPresent());
}
+ @AfterEach
+ public void tearDown() throws MetaserverStorageException {
+ Long tableId = store.getTableId(db, tb);
+ store.deleteTableTimestamp(tableId);
+ store.deleteTable(tableId);
+ Long dbId = store.getDatabaseId(db);
+ store.deleteDatabase(dbId);
+ }
+
}
