This is an automated email from the ASF dual-hosted git repository.
jshao pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gravitino.git
The following commit(s) were added to refs/heads/main by this push:
new 27ca87535 [#3968] improvement(core): Disable KV entity store and
optimize CI (#3975)
27ca87535 is described below
commit 27ca87535e30535df2dc8f4108570671cd9e80bd
Author: Qi Yu <[email protected]>
AuthorDate: Mon Jul 8 19:49:18 2024 +0800
[#3968] improvement(core): Disable KV entity store and optimize CI (#3975)
### What changes were proposed in this pull request?
- Disabling support for the KV entity store and adjusting the tests
accordingly.
- Change CI about backend option `jdbcBackend`
### Why are the changes needed?
We are going to deprecate kv entity store
Fix: #3968
### Why are the changes needed?
N/A
### How was this patch tested?
Existing test.
---
.github/workflows/backend-integration-test.yml | 2 +-
build.gradle.kts | 5 +-
catalogs/catalog-hadoop/build.gradle.kts | 10 +
.../hadoop/TestHadoopCatalogOperations.java | 81 ++++++--
catalogs/catalog-kafka/build.gradle.kts | 10 +
.../catalog/kafka/TestKafkaCatalogOperations.java | 69 ++++++-
.../lakehouse/paimon/TestGravitinoPaimonTable.java | 11 ++
.../lakehouse/paimon/TestPaimonCatalog.java | 13 +-
.../catalog/lakehouse/paimon/TestPaimonSchema.java | 13 ++
.../gravitino/filesystem/hadoop/TestGvfsBase.java | 8 +
.../java/com/datastrato/gravitino/Configs.java | 2 +-
.../datastrato/gravitino/EntityStoreFactory.java | 9 +-
.../gravitino/storage/TestEntityStorage.java | 35 +---
.../storage/kv/TestEntityKeyEncoding.java | 2 +
.../gravitino/storage/kv/TestKvEntityStorage.java | 1 +
.../storage/kv/TestKvGarbageCollector.java | 2 +
.../storage/kv/TestKvNameMappingService.java | 2 +
.../gravitino/storage/kv/TestRocksDBKvBackend.java | 2 +
.../gravitino/storage/kv/TestStorageVersion.java | 2 +
.../storage/kv/TestTransactionIdGenerator.java | 1 +
.../storage/kv/TestTransactionalKvBackend.java | 1 +
.../storage/relational/TestJDBCBackend.java | 9 +-
.../storage/relational/session/TestSqlSession.java | 8 +-
core/src/test/resources/h2/schema-h2.sql | 218 ---------------------
docs/gravitino-server-config.md | 2 +-
gradle/libs.versions.toml | 1 +
.../integration/test/util/AbstractIT.java | 2 +-
.../relational/service/FilesetMetaServiceIT.java | 4 +-
28 files changed, 242 insertions(+), 283 deletions(-)
diff --git a/.github/workflows/backend-integration-test.yml
b/.github/workflows/backend-integration-test.yml
index da534f467..437acbd02 100644
--- a/.github/workflows/backend-integration-test.yml
+++ b/.github/workflows/backend-integration-test.yml
@@ -61,7 +61,7 @@ jobs:
architecture: [linux/amd64]
java-version: [ 8, 11, 17 ]
test-mode: [ embedded, deploy ]
- backend: [ jdbcBackend, kvBackend]
+ backend: [ mysql, h2]
env:
PLATFORM: ${{ matrix.architecture }}
steps:
diff --git a/build.gradle.kts b/build.gradle.kts
index ec5898921..a29b0405d 100644
--- a/build.gradle.kts
+++ b/build.gradle.kts
@@ -178,9 +178,8 @@ allprojects {
// Change poll image pause time from 30s to 60s
param.environment("TESTCONTAINERS_PULL_PAUSE_TIMEOUT", "60")
- if (project.hasProperty("jdbcBackend")) {
- param.environment("jdbcBackend", "true")
- }
+ val jdbcDatabase = project.properties["jdbcBackend"] as? String ?: "h2"
+ param.environment("jdbcBackend", jdbcDatabase)
val testMode = project.properties["testMode"] as? String ?: "embedded"
param.systemProperty("gravitino.log.path", project.buildDir.path +
"/${project.name}-integration-test.log")
diff --git a/catalogs/catalog-hadoop/build.gradle.kts
b/catalogs/catalog-hadoop/build.gradle.kts
index ccdf7c996..0dfa23154 100644
--- a/catalogs/catalog-hadoop/build.gradle.kts
+++ b/catalogs/catalog-hadoop/build.gradle.kts
@@ -53,6 +53,7 @@ dependencies {
testImplementation(libs.bundles.log4j)
testImplementation(libs.mockito.core)
+ testImplementation(libs.mockito.inline)
testImplementation(libs.mysql.driver)
testImplementation(libs.junit.jupiter.api)
testImplementation(libs.junit.jupiter.params)
@@ -101,6 +102,15 @@ tasks {
}
tasks.test {
+ doFirst {
+ val testMode = project.properties["testMode"] as? String ?: "embedded"
+ if (testMode == "deploy") {
+ environment("GRAVITINO_HOME", project.rootDir.path +
"/distribution/package")
+ } else if (testMode == "embedded") {
+ environment("GRAVITINO_HOME", project.rootDir.path)
+ }
+ }
+
val skipUTs = project.hasProperty("skipTests")
if (skipUTs) {
// Only run integration tests
diff --git
a/catalogs/catalog-hadoop/src/test/java/com/datastrato/gravitino/catalog/hadoop/TestHadoopCatalogOperations.java
b/catalogs/catalog-hadoop/src/test/java/com/datastrato/gravitino/catalog/hadoop/TestHadoopCatalogOperations.java
index 6fa41cd95..7023ad96e 100644
---
a/catalogs/catalog-hadoop/src/test/java/com/datastrato/gravitino/catalog/hadoop/TestHadoopCatalogOperations.java
+++
b/catalogs/catalog-hadoop/src/test/java/com/datastrato/gravitino/catalog/hadoop/TestHadoopCatalogOperations.java
@@ -18,20 +18,27 @@
*/
package com.datastrato.gravitino.catalog.hadoop;
-import static com.datastrato.gravitino.Configs.DEFAULT_ENTITY_KV_STORE;
-import static com.datastrato.gravitino.Configs.ENTITY_KV_ROCKSDB_BACKEND_PATH;
-import static com.datastrato.gravitino.Configs.ENTITY_KV_STORE;
+import static com.datastrato.gravitino.Configs.DEFAULT_ENTITY_RELATIONAL_STORE;
+import static
com.datastrato.gravitino.Configs.ENTITY_RELATIONAL_JDBC_BACKEND_DRIVER;
+import static
com.datastrato.gravitino.Configs.ENTITY_RELATIONAL_JDBC_BACKEND_PASSWORD;
+import static
com.datastrato.gravitino.Configs.ENTITY_RELATIONAL_JDBC_BACKEND_PATH;
+import static
com.datastrato.gravitino.Configs.ENTITY_RELATIONAL_JDBC_BACKEND_URL;
+import static
com.datastrato.gravitino.Configs.ENTITY_RELATIONAL_JDBC_BACKEND_USER;
+import static com.datastrato.gravitino.Configs.ENTITY_RELATIONAL_STORE;
+import static com.datastrato.gravitino.Configs.ENTITY_SERDE;
import static com.datastrato.gravitino.Configs.ENTITY_STORE;
+import static com.datastrato.gravitino.Configs.RELATIONAL_ENTITY_STORE;
import static com.datastrato.gravitino.Configs.STORE_DELETE_AFTER_TIME;
import static com.datastrato.gravitino.Configs.STORE_TRANSACTION_MAX_SKEW_TIME;
+import static com.datastrato.gravitino.Configs.VERSION_RETENTION_COUNT;
import static
com.datastrato.gravitino.catalog.hadoop.HadoopCatalog.CATALOG_PROPERTIES_META;
import static
com.datastrato.gravitino.catalog.hadoop.HadoopCatalog.FILESET_PROPERTIES_META;
import static
com.datastrato.gravitino.catalog.hadoop.HadoopCatalog.SCHEMA_PROPERTIES_META;
import static
com.datastrato.gravitino.connector.BaseCatalog.CATALOG_BYPASS_PREFIX;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.when;
import com.datastrato.gravitino.Config;
-import com.datastrato.gravitino.Configs;
-import com.datastrato.gravitino.EntitySerDeFactory;
import com.datastrato.gravitino.EntityStore;
import com.datastrato.gravitino.EntityStoreFactory;
import com.datastrato.gravitino.NameIdentifier;
@@ -49,9 +56,12 @@ import com.datastrato.gravitino.file.Fileset;
import com.datastrato.gravitino.file.FilesetChange;
import com.datastrato.gravitino.storage.IdGenerator;
import com.datastrato.gravitino.storage.RandomIdGenerator;
+import com.datastrato.gravitino.storage.relational.service.CatalogMetaService;
+import com.datastrato.gravitino.storage.relational.service.MetalakeMetaService;
import com.datastrato.gravitino.utils.NameIdentifierUtil;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Maps;
+import java.io.File;
import java.io.IOException;
import java.nio.file.Paths;
import java.util.Arrays;
@@ -71,13 +81,15 @@ import org.junit.jupiter.api.Test;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.Arguments;
import org.junit.jupiter.params.provider.MethodSource;
+import org.mockito.MockedStatic;
import org.mockito.Mockito;
public class TestHadoopCatalogOperations {
- private static final String ROCKS_DB_STORE_PATH =
+ private static final String STORE_PATH =
"/tmp/gravitino_test_entityStore_" +
UUID.randomUUID().toString().replace("-", "");
+ private static final String H2_file = STORE_PATH + ".mv.db";
private static final String UNFORMALIZED_TEST_ROOT_PATH =
"/tmp/gravitino_test_catalog_" +
UUID.randomUUID().toString().replace("-", "");
@@ -118,28 +130,65 @@ public class TestHadoopCatalogOperations {
@BeforeAll
public static void setUp() {
Config config = Mockito.mock(Config.class);
- Mockito.when(config.get(ENTITY_STORE)).thenReturn("kv");
-
Mockito.when(config.get(ENTITY_KV_STORE)).thenReturn(DEFAULT_ENTITY_KV_STORE);
- Mockito.when(config.get(Configs.ENTITY_SERDE)).thenReturn("proto");
-
Mockito.when(config.get(ENTITY_KV_ROCKSDB_BACKEND_PATH)).thenReturn(ROCKS_DB_STORE_PATH);
-
- Assertions.assertEquals(ROCKS_DB_STORE_PATH,
config.get(ENTITY_KV_ROCKSDB_BACKEND_PATH));
-
Mockito.when(config.get(STORE_TRANSACTION_MAX_SKEW_TIME)).thenReturn(1000L);
- Mockito.when(config.get(STORE_DELETE_AFTER_TIME)).thenReturn(20 * 60 *
1000L);
+ when(config.get(ENTITY_STORE)).thenReturn(RELATIONAL_ENTITY_STORE);
+
when(config.get(ENTITY_RELATIONAL_STORE)).thenReturn(DEFAULT_ENTITY_RELATIONAL_STORE);
+
when(config.get(ENTITY_RELATIONAL_JDBC_BACKEND_PATH)).thenReturn(STORE_PATH);
+
+ // The following properties are used to create the JDBC connection; they
are just for test, in
+ // the real world,
+ // they will be set automatically by the configuration file if you set
ENTITY_RELATIONAL_STORE
+ // as EMBEDDED_ENTITY_RELATIONAL_STORE.
+ when(config.get(ENTITY_RELATIONAL_JDBC_BACKEND_URL))
+ .thenReturn(String.format("jdbc:h2:%s;DB_CLOSE_DELAY=-1;MODE=MYSQL",
STORE_PATH));
+
when(config.get(ENTITY_RELATIONAL_JDBC_BACKEND_USER)).thenReturn("gravitino");
+
when(config.get(ENTITY_RELATIONAL_JDBC_BACKEND_PASSWORD)).thenReturn("gravitino");
+
when(config.get(ENTITY_RELATIONAL_JDBC_BACKEND_DRIVER)).thenReturn("org.h2.Driver");
+
+ File f = FileUtils.getFile(STORE_PATH);
+ f.deleteOnExit();
+
+ when(config.get(VERSION_RETENTION_COUNT)).thenReturn(1L);
+ when(config.get(STORE_TRANSACTION_MAX_SKEW_TIME)).thenReturn(1000L);
+ when(config.get(STORE_DELETE_AFTER_TIME)).thenReturn(20 * 60 * 1000L);
+ when(config.get(ENTITY_SERDE)).thenReturn("proto");
store = EntityStoreFactory.createEntityStore(config);
store.initialize(config);
- store.setSerDe(EntitySerDeFactory.createEntitySerDe(config));
idGenerator = new RandomIdGenerator();
+
+ // Mock
+ MetalakeMetaService metalakeMetaService =
MetalakeMetaService.getInstance();
+ MetalakeMetaService spyMetaservice = Mockito.spy(metalakeMetaService);
+ doReturn(1L).when(spyMetaservice).getMetalakeIdByName(Mockito.anyString());
+
+ CatalogMetaService catalogMetaService = CatalogMetaService.getInstance();
+ CatalogMetaService spyCatalogMetaService = Mockito.spy(catalogMetaService);
+ doReturn(1L)
+ .when(spyCatalogMetaService)
+ .getCatalogIdByMetalakeIdAndName(Mockito.anyLong(),
Mockito.anyString());
+
+ MockedStatic<MetalakeMetaService> metalakeMetaServiceMockedStatic =
+ Mockito.mockStatic(MetalakeMetaService.class);
+ MockedStatic<CatalogMetaService> catalogMetaServiceMockedStatic =
+ Mockito.mockStatic(CatalogMetaService.class);
+
+ metalakeMetaServiceMockedStatic
+ .when(MetalakeMetaService::getInstance)
+ .thenReturn(spyMetaservice);
+ catalogMetaServiceMockedStatic
+ .when(CatalogMetaService::getInstance)
+ .thenReturn(spyCatalogMetaService);
}
@AfterAll
public static void tearDown() throws IOException {
store.close();
- FileUtils.deleteDirectory(FileUtils.getFile(ROCKS_DB_STORE_PATH));
new Path(TEST_ROOT_PATH)
.getFileSystem(new Configuration())
.delete(new Path(TEST_ROOT_PATH), true);
+
+ File f = FileUtils.getFile(H2_file);
+ f.delete();
}
@Test
diff --git a/catalogs/catalog-kafka/build.gradle.kts
b/catalogs/catalog-kafka/build.gradle.kts
index 790383433..8ce72eeba 100644
--- a/catalogs/catalog-kafka/build.gradle.kts
+++ b/catalogs/catalog-kafka/build.gradle.kts
@@ -43,6 +43,7 @@ dependencies {
testImplementation(libs.junit.jupiter.api)
testImplementation(libs.kafka)
testImplementation(libs.mockito.core)
+ testImplementation(libs.mockito.inline)
testImplementation(libs.mysql.driver)
testImplementation(libs.testcontainers)
testImplementation(libs.testcontainers.mysql)
@@ -83,6 +84,15 @@
tasks.getByName("generateMetadataFileForMavenJavaPublication") {
}
tasks.test {
+ doFirst {
+ val testMode = project.properties["testMode"] as? String ?: "embedded"
+ if (testMode == "deploy") {
+ environment("GRAVITINO_HOME", project.rootDir.path +
"/distribution/package")
+ } else if (testMode == "embedded") {
+ environment("GRAVITINO_HOME", project.rootDir.path)
+ }
+ }
+
val skipUTs = project.hasProperty("skipTests")
if (skipUTs) {
// Only run integration tests
diff --git
a/catalogs/catalog-kafka/src/test/java/com/datastrato/gravitino/catalog/kafka/TestKafkaCatalogOperations.java
b/catalogs/catalog-kafka/src/test/java/com/datastrato/gravitino/catalog/kafka/TestKafkaCatalogOperations.java
index 2d29b2f68..93cee22d2 100644
---
a/catalogs/catalog-kafka/src/test/java/com/datastrato/gravitino/catalog/kafka/TestKafkaCatalogOperations.java
+++
b/catalogs/catalog-kafka/src/test/java/com/datastrato/gravitino/catalog/kafka/TestKafkaCatalogOperations.java
@@ -20,11 +20,21 @@ package com.datastrato.gravitino.catalog.kafka;
import static com.datastrato.gravitino.Catalog.Type.MESSAGING;
import static com.datastrato.gravitino.Configs.DEFAULT_ENTITY_KV_STORE;
+import static com.datastrato.gravitino.Configs.DEFAULT_ENTITY_RELATIONAL_STORE;
import static com.datastrato.gravitino.Configs.ENTITY_KV_ROCKSDB_BACKEND_PATH;
import static com.datastrato.gravitino.Configs.ENTITY_KV_STORE;
+import static
com.datastrato.gravitino.Configs.ENTITY_RELATIONAL_JDBC_BACKEND_DRIVER;
+import static
com.datastrato.gravitino.Configs.ENTITY_RELATIONAL_JDBC_BACKEND_PASSWORD;
+import static
com.datastrato.gravitino.Configs.ENTITY_RELATIONAL_JDBC_BACKEND_PATH;
+import static
com.datastrato.gravitino.Configs.ENTITY_RELATIONAL_JDBC_BACKEND_URL;
+import static
com.datastrato.gravitino.Configs.ENTITY_RELATIONAL_JDBC_BACKEND_USER;
+import static com.datastrato.gravitino.Configs.ENTITY_RELATIONAL_STORE;
+import static com.datastrato.gravitino.Configs.ENTITY_SERDE;
import static com.datastrato.gravitino.Configs.ENTITY_STORE;
+import static com.datastrato.gravitino.Configs.RELATIONAL_ENTITY_STORE;
import static com.datastrato.gravitino.Configs.STORE_DELETE_AFTER_TIME;
import static com.datastrato.gravitino.Configs.STORE_TRANSACTION_MAX_SKEW_TIME;
+import static com.datastrato.gravitino.Configs.VERSION_RETENTION_COUNT;
import static com.datastrato.gravitino.StringIdentifier.ID_KEY;
import static
com.datastrato.gravitino.catalog.kafka.KafkaCatalog.CATALOG_PROPERTIES_METADATA;
import static
com.datastrato.gravitino.catalog.kafka.KafkaCatalog.SCHEMA_PROPERTIES_METADATA;
@@ -33,10 +43,11 @@ import static
com.datastrato.gravitino.catalog.kafka.KafkaCatalogOperations.CLIE
import static
com.datastrato.gravitino.catalog.kafka.KafkaCatalogPropertiesMetadata.BOOTSTRAP_SERVERS;
import static
com.datastrato.gravitino.catalog.kafka.KafkaTopicPropertiesMetadata.PARTITION_COUNT;
import static
com.datastrato.gravitino.catalog.kafka.KafkaTopicPropertiesMetadata.REPLICATION_FACTOR;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.when;
import com.datastrato.gravitino.Config;
import com.datastrato.gravitino.Configs;
-import com.datastrato.gravitino.EntitySerDeFactory;
import com.datastrato.gravitino.EntityStore;
import com.datastrato.gravitino.EntityStoreFactory;
import com.datastrato.gravitino.NameIdentifier;
@@ -55,7 +66,10 @@ import com.datastrato.gravitino.meta.AuditInfo;
import com.datastrato.gravitino.meta.CatalogEntity;
import com.datastrato.gravitino.storage.IdGenerator;
import com.datastrato.gravitino.storage.RandomIdGenerator;
+import com.datastrato.gravitino.storage.relational.service.CatalogMetaService;
+import com.datastrato.gravitino.storage.relational.service.MetalakeMetaService;
import com.google.common.collect.ImmutableMap;
+import java.io.File;
import java.io.IOException;
import java.time.Instant;
import java.util.Map;
@@ -65,12 +79,13 @@ import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
+import org.mockito.MockedStatic;
import org.mockito.Mockito;
public class TestKafkaCatalogOperations extends KafkaClusterEmbedded {
- private static final String ROCKS_DB_STORE_PATH =
- "/tmp/gravitino_test_entityStore_" + genRandomString();
+ private static final String STORE_PATH = "/tmp/gravitino_test_entityStore_"
+ genRandomString();
+ private static final String H2_FILE = STORE_PATH + ".mv.db";
private static final String METALAKE_NAME = "metalake";
private static final String CATALOG_NAME = "test_kafka_catalog";
private static final String DEFAULT_SCHEMA_NAME = "default";
@@ -114,15 +129,55 @@ public class TestKafkaCatalogOperations extends
KafkaClusterEmbedded {
Mockito.when(config.get(ENTITY_STORE)).thenReturn("kv");
Mockito.when(config.get(ENTITY_KV_STORE)).thenReturn(DEFAULT_ENTITY_KV_STORE);
Mockito.when(config.get(Configs.ENTITY_SERDE)).thenReturn("proto");
-
Mockito.when(config.get(ENTITY_KV_ROCKSDB_BACKEND_PATH)).thenReturn(ROCKS_DB_STORE_PATH);
+
Mockito.when(config.get(ENTITY_KV_ROCKSDB_BACKEND_PATH)).thenReturn(STORE_PATH);
- Assertions.assertEquals(ROCKS_DB_STORE_PATH,
config.get(ENTITY_KV_ROCKSDB_BACKEND_PATH));
+ Assertions.assertEquals(STORE_PATH,
config.get(ENTITY_KV_ROCKSDB_BACKEND_PATH));
Mockito.when(config.get(STORE_TRANSACTION_MAX_SKEW_TIME)).thenReturn(1000L);
Mockito.when(config.get(STORE_DELETE_AFTER_TIME)).thenReturn(20 * 60 *
1000L);
+ when(config.get(ENTITY_STORE)).thenReturn(RELATIONAL_ENTITY_STORE);
+
when(config.get(ENTITY_RELATIONAL_STORE)).thenReturn(DEFAULT_ENTITY_RELATIONAL_STORE);
+
when(config.get(ENTITY_RELATIONAL_JDBC_BACKEND_PATH)).thenReturn(STORE_PATH);
+
+ when(config.get(ENTITY_RELATIONAL_JDBC_BACKEND_URL))
+ .thenReturn(String.format("jdbc:h2:%s;DB_CLOSE_DELAY=-1;MODE=MYSQL",
STORE_PATH));
+
when(config.get(ENTITY_RELATIONAL_JDBC_BACKEND_USER)).thenReturn("gravitino");
+
when(config.get(ENTITY_RELATIONAL_JDBC_BACKEND_PASSWORD)).thenReturn("gravitino");
+
when(config.get(ENTITY_RELATIONAL_JDBC_BACKEND_DRIVER)).thenReturn("org.h2.Driver");
+
+ File f = FileUtils.getFile(STORE_PATH);
+ f.deleteOnExit();
+
+ when(config.get(VERSION_RETENTION_COUNT)).thenReturn(1L);
+ when(config.get(STORE_TRANSACTION_MAX_SKEW_TIME)).thenReturn(1000L);
+ when(config.get(STORE_DELETE_AFTER_TIME)).thenReturn(20 * 60 * 1000L);
+ when(config.get(ENTITY_SERDE)).thenReturn("proto");
+
+ // Mock
+ MetalakeMetaService metalakeMetaService =
MetalakeMetaService.getInstance();
+ MetalakeMetaService spyMetaservice = Mockito.spy(metalakeMetaService);
+ doReturn(1L).when(spyMetaservice).getMetalakeIdByName(Mockito.anyString());
+
+ CatalogMetaService catalogMetaService = CatalogMetaService.getInstance();
+ CatalogMetaService spyCatalogMetaService = Mockito.spy(catalogMetaService);
+ doReturn(1L)
+ .when(spyCatalogMetaService)
+ .getCatalogIdByMetalakeIdAndName(Mockito.anyLong(),
Mockito.anyString());
+
+ MockedStatic<MetalakeMetaService> metalakeMetaServiceMockedStatic =
+ Mockito.mockStatic(MetalakeMetaService.class);
+ MockedStatic<CatalogMetaService> catalogMetaServiceMockedStatic =
+ Mockito.mockStatic(CatalogMetaService.class);
+
+ metalakeMetaServiceMockedStatic
+ .when(MetalakeMetaService::getInstance)
+ .thenReturn(spyMetaservice);
+ catalogMetaServiceMockedStatic
+ .when(CatalogMetaService::getInstance)
+ .thenReturn(spyCatalogMetaService);
+
store = EntityStoreFactory.createEntityStore(config);
store.initialize(config);
- store.setSerDe(EntitySerDeFactory.createEntitySerDe(config));
idGenerator = new RandomIdGenerator();
kafkaCatalogEntity =
CatalogEntity.builder()
@@ -147,7 +202,7 @@ public class TestKafkaCatalogOperations extends
KafkaClusterEmbedded {
public static void tearDown() throws IOException {
if (store != null) {
store.close();
- FileUtils.deleteDirectory(FileUtils.getFile(ROCKS_DB_STORE_PATH));
+ FileUtils.deleteQuietly(FileUtils.getFile(H2_FILE));
}
}
diff --git
a/catalogs/catalog-lakehouse-paimon/src/test/java/com/datastrato/gravitino/catalog/lakehouse/paimon/TestGravitinoPaimonTable.java
b/catalogs/catalog-lakehouse-paimon/src/test/java/com/datastrato/gravitino/catalog/lakehouse/paimon/TestGravitinoPaimonTable.java
index eba62d4de..1c8b40fe6 100644
---
a/catalogs/catalog-lakehouse-paimon/src/test/java/com/datastrato/gravitino/catalog/lakehouse/paimon/TestGravitinoPaimonTable.java
+++
b/catalogs/catalog-lakehouse-paimon/src/test/java/com/datastrato/gravitino/catalog/lakehouse/paimon/TestGravitinoPaimonTable.java
@@ -38,11 +38,15 @@ import
com.datastrato.gravitino.rel.expressions.sorts.SortOrder;
import com.datastrato.gravitino.rel.expressions.transforms.Transform;
import com.datastrato.gravitino.rel.types.Types;
import com.google.common.collect.Maps;
+import java.io.File;
+import java.nio.file.Files;
+import java.nio.file.Paths;
import java.time.Instant;
import java.util.Arrays;
import java.util.HashMap;
import java.util.Map;
import java.util.UUID;
+import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.ArrayUtils;
import org.apache.paimon.schema.Schema;
import org.apache.paimon.types.DataField;
@@ -98,6 +102,13 @@ public class TestGravitinoPaimonTable {
@AfterAll
static void cleanUp() {
paimonCatalogOperations.dropSchema(schemaIdent, true);
+ String warehousePath = "/tmp/paimon_catalog_warehouse";
+ try {
+ FileUtils.deleteDirectory(new File(warehousePath));
+ Files.delete(Paths.get(warehousePath));
+ } catch (Exception e) {
+ // Ignore
+ }
}
private static CatalogEntity createDefaultCatalogEntity() {
diff --git
a/catalogs/catalog-lakehouse-paimon/src/test/java/com/datastrato/gravitino/catalog/lakehouse/paimon/TestPaimonCatalog.java
b/catalogs/catalog-lakehouse-paimon/src/test/java/com/datastrato/gravitino/catalog/lakehouse/paimon/TestPaimonCatalog.java
index fb8cb2bd6..3ba2dde0c 100644
---
a/catalogs/catalog-lakehouse-paimon/src/test/java/com/datastrato/gravitino/catalog/lakehouse/paimon/TestPaimonCatalog.java
+++
b/catalogs/catalog-lakehouse-paimon/src/test/java/com/datastrato/gravitino/catalog/lakehouse/paimon/TestPaimonCatalog.java
@@ -35,6 +35,8 @@ import java.io.File;
import java.io.IOException;
import java.time.Instant;
import java.util.Map;
+import org.apache.commons.io.FileUtils;
+import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
@@ -69,9 +71,18 @@ public class TestPaimonCatalog {
}
};
- private String tempDir =
+ private static String tempDir =
String.join(File.separator, System.getProperty("java.io.tmpdir"),
"paimon_catalog_warehouse");
+ @AfterAll
+ public static void clean() {
+ try {
+ FileUtils.deleteDirectory(new File(tempDir));
+ } catch (Exception e) {
+ // Ignore
+ }
+ }
+
@Test
public void testCatalogOperation() {
AuditInfo auditInfo =
diff --git
a/catalogs/catalog-lakehouse-paimon/src/test/java/com/datastrato/gravitino/catalog/lakehouse/paimon/TestPaimonSchema.java
b/catalogs/catalog-lakehouse-paimon/src/test/java/com/datastrato/gravitino/catalog/lakehouse/paimon/TestPaimonSchema.java
index f492c30ab..3fe4f563c 100644
---
a/catalogs/catalog-lakehouse-paimon/src/test/java/com/datastrato/gravitino/catalog/lakehouse/paimon/TestPaimonSchema.java
+++
b/catalogs/catalog-lakehouse-paimon/src/test/java/com/datastrato/gravitino/catalog/lakehouse/paimon/TestPaimonSchema.java
@@ -28,11 +28,14 @@ import com.datastrato.gravitino.meta.AuditInfo;
import com.datastrato.gravitino.meta.CatalogEntity;
import com.google.common.collect.Maps;
import java.io.File;
+import java.io.IOException;
import java.time.Instant;
import java.util.Arrays;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
+import org.apache.commons.io.FileUtils;
+import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
@@ -51,6 +54,16 @@ public class TestPaimonSchema {
NoSuchSchemaException.class, () ->
paimonCatalogOperations.loadSchema(ident));
}
+ @AfterAll
+ static void cleanUpFile() {
+ try {
+ FileUtils.deleteDirectory(
+ new File(System.getProperty("java.io.tmpdir") +
"/paimon_catalog_warehouse"));
+ } catch (IOException e) {
+ // Ignore
+ }
+ }
+
@Test
public void testCreatePaimonSchema() {
PaimonCatalog paimonCatalog = initPaimonCatalog("testCreatePaimonSchema");
diff --git
a/clients/filesystem-hadoop3/src/test/java/com/datastrato/gravitino/filesystem/hadoop/TestGvfsBase.java
b/clients/filesystem-hadoop3/src/test/java/com/datastrato/gravitino/filesystem/hadoop/TestGvfsBase.java
index dfe489421..2c6af2ef5 100644
---
a/clients/filesystem-hadoop3/src/test/java/com/datastrato/gravitino/filesystem/hadoop/TestGvfsBase.java
+++
b/clients/filesystem-hadoop3/src/test/java/com/datastrato/gravitino/filesystem/hadoop/TestGvfsBase.java
@@ -27,6 +27,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue;
import com.datastrato.gravitino.NameIdentifier;
import com.datastrato.gravitino.file.Fileset;
+import java.io.File;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
@@ -37,6 +38,7 @@ import java.util.Comparator;
import java.util.List;
import java.util.Objects;
import java.util.concurrent.TimeUnit;
+import org.apache.commons.io.FileUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
@@ -80,6 +82,12 @@ public class TestGvfsBase extends GravitinoMockServerBase {
@AfterAll
public static void tearDown() {
GravitinoMockServerBase.tearDown();
+ String fileName = FileSystemTestUtils.localRootPrefix().replace("file:",
"");
+ try {
+ FileUtils.deleteDirectory(new File(fileName));
+ } catch (Exception e) {
+ // Ignore
+ }
}
@BeforeEach
diff --git a/core/src/main/java/com/datastrato/gravitino/Configs.java
b/core/src/main/java/com/datastrato/gravitino/Configs.java
index 5ee2232ca..6c7fa3fb5 100644
--- a/core/src/main/java/com/datastrato/gravitino/Configs.java
+++ b/core/src/main/java/com/datastrato/gravitino/Configs.java
@@ -30,7 +30,7 @@ public class Configs {
private Configs() {}
- public static final String DEFAULT_ENTITY_STORE = "kv";
+ public static final String KV_STORE_KEY = "kv";
public static final String RELATIONAL_ENTITY_STORE = "relational";
public static final String ENTITY_STORE_KEY = "gravitino.entity.store";
diff --git
a/core/src/main/java/com/datastrato/gravitino/EntityStoreFactory.java
b/core/src/main/java/com/datastrato/gravitino/EntityStoreFactory.java
index 249a0a9e9..bfd74efc7 100644
--- a/core/src/main/java/com/datastrato/gravitino/EntityStoreFactory.java
+++ b/core/src/main/java/com/datastrato/gravitino/EntityStoreFactory.java
@@ -18,6 +18,8 @@
*/
package com.datastrato.gravitino;
+import static com.datastrato.gravitino.Configs.KV_STORE_KEY;
+
import com.datastrato.gravitino.storage.kv.KvEntityStore;
import com.datastrato.gravitino.storage.relational.RelationalEntityStore;
import com.google.common.collect.ImmutableMap;
@@ -36,7 +38,7 @@ public class EntityStoreFactory {
// doesn't need to specify the full qualified class name when creating an
EntityStore.
public static final ImmutableMap<String, String> ENTITY_STORES =
ImmutableMap.of(
- Configs.DEFAULT_ENTITY_STORE,
+ KV_STORE_KEY,
KvEntityStore.class.getCanonicalName(),
Configs.RELATIONAL_ENTITY_STORE,
RelationalEntityStore.class.getCanonicalName());
@@ -54,6 +56,11 @@ public class EntityStoreFactory {
String name = config.get(Configs.ENTITY_STORE);
String className = ENTITY_STORES.getOrDefault(name, name);
+ if (KV_STORE_KEY.equals(name)) {
+ throw new UnsupportedOperationException(
+ "KvEntityStore is not supported since version 0.6.0. Please use
RelationalEntityStore instead.");
+ }
+
try {
return (EntityStore)
Class.forName(className).getDeclaredConstructor().newInstance();
} catch (Exception e) {
diff --git
a/core/src/test/java/com/datastrato/gravitino/storage/TestEntityStorage.java
b/core/src/test/java/com/datastrato/gravitino/storage/TestEntityStorage.java
index b38dc0ef5..09a14682c 100644
--- a/core/src/test/java/com/datastrato/gravitino/storage/TestEntityStorage.java
+++ b/core/src/test/java/com/datastrato/gravitino/storage/TestEntityStorage.java
@@ -63,14 +63,12 @@ import com.datastrato.gravitino.meta.SchemaVersion;
import com.datastrato.gravitino.meta.TableEntity;
import com.datastrato.gravitino.meta.TopicEntity;
import com.datastrato.gravitino.meta.UserEntity;
-import com.datastrato.gravitino.storage.relational.RelationalEntityStore;
import
com.datastrato.gravitino.storage.relational.session.SqlSessionFactoryHelper;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Lists;
import java.io.File;
import java.io.IOException;
-import java.nio.charset.StandardCharsets;
import java.sql.Connection;
import java.sql.ResultSet;
import java.sql.SQLException;
@@ -78,10 +76,8 @@ import java.sql.Statement;
import java.time.Instant;
import java.util.ArrayList;
import java.util.List;
-import java.util.Objects;
import java.util.UUID;
import org.apache.commons.io.FileUtils;
-import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.ibatis.session.SqlSession;
import org.junit.jupiter.api.Assertions;
@@ -96,14 +92,15 @@ public class TestEntityStorage {
private static final String JDBC_STORE_PATH =
"/tmp/gravitino_jdbc_entityStore_" +
UUID.randomUUID().toString().replace("-", "");
private static final String DB_DIR = JDBC_STORE_PATH + "/testdb";
+ private static final String H2_FILE = DB_DIR + ".mv.db";
static Object[] storageProvider() {
- return new Object[] {Configs.DEFAULT_ENTITY_STORE,
Configs.RELATIONAL_ENTITY_STORE};
+ return new Object[] {Configs.RELATIONAL_ENTITY_STORE};
}
private void init(String type, Config config) {
Preconditions.checkArgument(StringUtils.isNotBlank(type));
- if (type.equals(Configs.DEFAULT_ENTITY_STORE)) {
+ if (type.equals(Configs.KV_STORE_KEY)) {
try {
FileUtils.deleteDirectory(FileUtils.getFile(KV_STORE_PATH));
} catch (Exception e) {
@@ -145,28 +142,9 @@ public class TestEntityStorage {
}
}
- private void prepareJdbcTable() {
- // Read the ddl sql to create table
- String scriptPath = "h2/schema-h2.sql";
- try (SqlSession sqlSession =
-
SqlSessionFactoryHelper.getInstance().getSqlSessionFactory().openSession(true);
- Connection connection = sqlSession.getConnection();
- Statement statement = connection.createStatement()) {
- StringBuilder ddlBuilder = new StringBuilder();
- IOUtils.readLines(
- Objects.requireNonNull(
-
this.getClass().getClassLoader().getResourceAsStream(scriptPath)),
- StandardCharsets.UTF_8)
- .forEach(line -> ddlBuilder.append(line).append("\n"));
- statement.execute(ddlBuilder.toString());
- } catch (Exception e) {
- throw new IllegalStateException("Create tables failed", e);
- }
- }
-
private void destroy(String type) {
Preconditions.checkArgument(StringUtils.isNotBlank(type));
- if (type.equals(Configs.DEFAULT_ENTITY_STORE)) {
+ if (type.equals(Configs.KV_STORE_KEY)) {
try {
FileUtils.deleteDirectory(FileUtils.getFile(KV_STORE_PATH));
} catch (Exception e) {
@@ -178,6 +156,8 @@ public class TestEntityStorage {
if (dir.exists()) {
dir.delete();
}
+
+ FileUtils.deleteQuietly(new File(H2_FILE));
} else {
throw new UnsupportedOperationException("Unsupported entity store type:
" + type);
}
@@ -2135,9 +2115,6 @@ public class TestEntityStorage {
try (EntityStore store = EntityStoreFactory.createEntityStore(config)) {
store.initialize(config);
- if (store instanceof RelationalEntityStore) {
- prepareJdbcTable();
- }
BaseMetalake metalake = createBaseMakeLake(1L, "metalake", auditInfo);
CatalogEntity catalog = createCatalog(1L, Namespace.of("metalake"),
"catalog", auditInfo);
diff --git
a/core/src/test/java/com/datastrato/gravitino/storage/kv/TestEntityKeyEncoding.java
b/core/src/test/java/com/datastrato/gravitino/storage/kv/TestEntityKeyEncoding.java
index 588bde8e6..a8c7a2ac3 100644
---
a/core/src/test/java/com/datastrato/gravitino/storage/kv/TestEntityKeyEncoding.java
+++
b/core/src/test/java/com/datastrato/gravitino/storage/kv/TestEntityKeyEncoding.java
@@ -46,6 +46,7 @@ import java.util.Random;
import java.util.stream.Stream;
import org.apache.commons.lang3.tuple.Pair;
import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.TestInstance;
import org.junit.jupiter.api.TestInstance.Lifecycle;
@@ -55,6 +56,7 @@ import org.junit.jupiter.params.provider.MethodSource;
import org.mockito.Mockito;
@TestInstance(Lifecycle.PER_CLASS)
+@Disabled("Gravitino will not support KV entity store since 0.6.0, so we
disable this test.")
public class TestEntityKeyEncoding {
private Config getConfig() throws IOException {
File baseDir = new File(System.getProperty("java.io.tmpdir"));
diff --git
a/core/src/test/java/com/datastrato/gravitino/storage/kv/TestKvEntityStorage.java
b/core/src/test/java/com/datastrato/gravitino/storage/kv/TestKvEntityStorage.java
index f8d41f351..28985c31e 100644
---
a/core/src/test/java/com/datastrato/gravitino/storage/kv/TestKvEntityStorage.java
+++
b/core/src/test/java/com/datastrato/gravitino/storage/kv/TestKvEntityStorage.java
@@ -63,6 +63,7 @@ import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
import org.mockito.Mockito;
+@Disabled("Gravitino will not support KV entity store since 0.6.0, so we
disable this test.")
public class TestKvEntityStorage extends TestEntityStorage {
@BeforeEach
@AfterEach
diff --git
a/core/src/test/java/com/datastrato/gravitino/storage/kv/TestKvGarbageCollector.java
b/core/src/test/java/com/datastrato/gravitino/storage/kv/TestKvGarbageCollector.java
index 8a58f8525..cef6e2e40 100644
---
a/core/src/test/java/com/datastrato/gravitino/storage/kv/TestKvGarbageCollector.java
+++
b/core/src/test/java/com/datastrato/gravitino/storage/kv/TestKvGarbageCollector.java
@@ -59,10 +59,12 @@ import java.time.Instant;
import java.util.List;
import org.apache.commons.lang3.tuple.Pair;
import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
import org.mockito.Mockito;
@SuppressWarnings("DefaultCharset")
+@Disabled("Gravitino will not support KV entity store since 0.6.0, so we
disable this test.")
class TestKvGarbageCollector {
public Config getConfig() throws IOException {
Config config = Mockito.mock(Config.class);
diff --git
a/core/src/test/java/com/datastrato/gravitino/storage/kv/TestKvNameMappingService.java
b/core/src/test/java/com/datastrato/gravitino/storage/kv/TestKvNameMappingService.java
index 40a6bacc8..f8cdd091b 100644
---
a/core/src/test/java/com/datastrato/gravitino/storage/kv/TestKvNameMappingService.java
+++
b/core/src/test/java/com/datastrato/gravitino/storage/kv/TestKvNameMappingService.java
@@ -37,9 +37,11 @@ import java.io.IOException;
import java.lang.reflect.Field;
import java.nio.file.Files;
import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
import org.mockito.Mockito;
+@Disabled("Gravitino will not support KV entity store since 0.6.0, so we
disable this test.")
public class TestKvNameMappingService {
private Config getConfig() throws IOException {
File baseDir = new File(System.getProperty("java.io.tmpdir"));
diff --git
a/core/src/test/java/com/datastrato/gravitino/storage/kv/TestRocksDBKvBackend.java
b/core/src/test/java/com/datastrato/gravitino/storage/kv/TestRocksDBKvBackend.java
index 588908ba2..e1d3d80b0 100644
---
a/core/src/test/java/com/datastrato/gravitino/storage/kv/TestRocksDBKvBackend.java
+++
b/core/src/test/java/com/datastrato/gravitino/storage/kv/TestRocksDBKvBackend.java
@@ -30,11 +30,13 @@ import java.nio.file.Files;
import java.util.List;
import org.apache.commons.lang3.tuple.Pair;
import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
import org.mockito.Mockito;
import org.rocksdb.RocksDB;
import org.rocksdb.RocksDBException;
+@Disabled("Gravitino will not support KV entity store since 0.6.0, so we
disable this test.")
public class TestRocksDBKvBackend {
private KvBackend getKvBackEnd() throws IOException {
diff --git
a/core/src/test/java/com/datastrato/gravitino/storage/kv/TestStorageVersion.java
b/core/src/test/java/com/datastrato/gravitino/storage/kv/TestStorageVersion.java
index 3f8bd230b..fcf1eb03d 100644
---
a/core/src/test/java/com/datastrato/gravitino/storage/kv/TestStorageVersion.java
+++
b/core/src/test/java/com/datastrato/gravitino/storage/kv/TestStorageVersion.java
@@ -37,9 +37,11 @@ import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
import org.mockito.Mockito;
+@Disabled("Gravitino will not support KV entity store since 0.6.0, so we
disable this test.")
class TestStorageVersion {
@Test
diff --git
a/core/src/test/java/com/datastrato/gravitino/storage/kv/TestTransactionIdGenerator.java
b/core/src/test/java/com/datastrato/gravitino/storage/kv/TestTransactionIdGenerator.java
index a2903bc4a..22e6b9f5e 100644
---
a/core/src/test/java/com/datastrato/gravitino/storage/kv/TestTransactionIdGenerator.java
+++
b/core/src/test/java/com/datastrato/gravitino/storage/kv/TestTransactionIdGenerator.java
@@ -42,6 +42,7 @@ import org.mockito.Mockito;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+@Disabled("Gravitino will not support KV entity store since 0.6.0, so we
disable this test.")
public class TestTransactionIdGenerator {
private static final Logger LOGGER =
LoggerFactory.getLogger(TestTransactionalKvBackend.class);
diff --git
a/core/src/test/java/com/datastrato/gravitino/storage/kv/TestTransactionalKvBackend.java
b/core/src/test/java/com/datastrato/gravitino/storage/kv/TestTransactionalKvBackend.java
index 936fbc492..7514c290c 100644
---
a/core/src/test/java/com/datastrato/gravitino/storage/kv/TestTransactionalKvBackend.java
+++
b/core/src/test/java/com/datastrato/gravitino/storage/kv/TestTransactionalKvBackend.java
@@ -54,6 +54,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@SuppressWarnings("DefaultCharset")
+@Disabled("Gravitino will not support KV entity store since 0.6.0, so we
disable this test.")
class TestTransactionalKvBackend {
private static final Logger LOGGER =
LoggerFactory.getLogger(TestTransactionalKvBackend.class);
diff --git
a/core/src/test/java/com/datastrato/gravitino/storage/relational/TestJDBCBackend.java
b/core/src/test/java/com/datastrato/gravitino/storage/relational/TestJDBCBackend.java
index ffb7f9dc4..7a62eacab 100644
---
a/core/src/test/java/com/datastrato/gravitino/storage/relational/TestJDBCBackend.java
+++
b/core/src/test/java/com/datastrato/gravitino/storage/relational/TestJDBCBackend.java
@@ -65,6 +65,8 @@ import com.google.common.collect.Lists;
import java.io.File;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Paths;
import java.sql.Connection;
import java.sql.ResultSet;
import java.sql.SQLException;
@@ -88,6 +90,7 @@ public class TestJDBCBackend {
private static final String JDBC_STORE_PATH =
"/tmp/gravitino_jdbc_entityStore_" +
UUID.randomUUID().toString().replace("-", "");
private static final String DB_DIR = JDBC_STORE_PATH + "/testdb";
+ private static final String H2_FILE = DB_DIR + ".mv.db";
private static final Config config = Mockito.mock(Config.class);
public static final ImmutableMap<String, String> RELATIONAL_BACKENDS =
ImmutableMap.of(
@@ -97,6 +100,7 @@ public class TestJDBCBackend {
@BeforeAll
public static void setup() {
File dir = new File(DB_DIR);
+ dir.deleteOnExit();
if (dir.exists() || !dir.isDirectory()) {
dir.delete();
}
@@ -127,8 +131,11 @@ public class TestJDBCBackend {
dropAllTables();
File dir = new File(DB_DIR);
if (dir.exists()) {
- dir.delete();
+ Files.delete(Paths.get(DB_DIR));
}
+
+ Files.delete(Paths.get(H2_FILE));
+ Files.delete(Paths.get(JDBC_STORE_PATH));
backend.close();
}
diff --git
a/core/src/test/java/com/datastrato/gravitino/storage/relational/session/TestSqlSession.java
b/core/src/test/java/com/datastrato/gravitino/storage/relational/session/TestSqlSession.java
index e64e454c1..03e9904ed 100644
---
a/core/src/test/java/com/datastrato/gravitino/storage/relational/session/TestSqlSession.java
+++
b/core/src/test/java/com/datastrato/gravitino/storage/relational/session/TestSqlSession.java
@@ -35,9 +35,12 @@ import static org.junit.jupiter.api.Assertions.assertThrows;
import com.datastrato.gravitino.Config;
import java.io.File;
import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Paths;
import java.sql.SQLException;
import java.util.UUID;
import org.apache.commons.dbcp2.BasicDataSource;
+import org.apache.commons.io.FileUtils;
import org.apache.ibatis.session.SqlSession;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.AfterEach;
@@ -85,8 +88,11 @@ public class TestSqlSession {
public static void tearDown() throws IOException {
File dir = new File(DB_DIR);
if (dir.exists()) {
- dir.delete();
+ FileUtils.deleteDirectory(FileUtils.getFile(DB_DIR));
}
+
+ Files.delete(Paths.get(MYSQL_STORE_PATH));
+
SqlSessionFactoryHelper.getInstance().close();
}
diff --git a/core/src/test/resources/h2/schema-h2.sql
b/core/src/test/resources/h2/schema-h2.sql
deleted file mode 100644
index 8d29af599..000000000
--- a/core/src/test/resources/h2/schema-h2.sql
+++ /dev/null
@@ -1,218 +0,0 @@
---
--- Licensed to the Apache Software Foundation (ASF) under one
--- or more contributor license agreements. See the NOTICE file--
--- distributed with this work for additional information
--- regarding copyright ownership. The ASF licenses this file
--- to you under the Apache License, Version 2.0 (the
--- "License"). You may not use this file except in compliance
--- with the License. You may obtain a copy of the License at
---
--- http://www.apache.org/licenses/LICENSE-2.0
---
--- Unless required by applicable law or agreed to in writing,
--- software distributed under the License is distributed on an
--- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
--- KIND, either express or implied. See the License for the
--- specific language governing permissions and limitations
--- under the License.
---
-
-CREATE TABLE IF NOT EXISTS `metalake_meta` (
- `metalake_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'metalake id',
- `metalake_name` VARCHAR(128) NOT NULL COMMENT 'metalake name',
- `metalake_comment` VARCHAR(256) DEFAULT '' COMMENT 'metalake comment',
- `properties` MEDIUMTEXT DEFAULT NULL COMMENT 'metalake properties',
- `audit_info` MEDIUMTEXT NOT NULL COMMENT 'metalake audit info',
- `schema_version` MEDIUMTEXT NOT NULL COMMENT 'metalake schema version
info',
- `current_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'metalake
current version',
- `last_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'metalake last
version',
- `deleted_at` BIGINT(20) UNSIGNED NOT NULL DEFAULT 0 COMMENT 'metalake
deleted at',
- PRIMARY KEY (metalake_id),
- CONSTRAINT uk_mn_del UNIQUE (metalake_name, deleted_at)
-) ENGINE = InnoDB;
-
-
-CREATE TABLE IF NOT EXISTS `catalog_meta` (
- `catalog_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'catalog id',
- `catalog_name` VARCHAR(128) NOT NULL COMMENT 'catalog name',
- `metalake_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'metalake id',
- `type` VARCHAR(64) NOT NULL COMMENT 'catalog type',
- `provider` VARCHAR(64) NOT NULL COMMENT 'catalog provider',
- `catalog_comment` VARCHAR(256) DEFAULT '' COMMENT 'catalog comment',
- `properties` MEDIUMTEXT DEFAULT NULL COMMENT 'catalog properties',
- `audit_info` MEDIUMTEXT NOT NULL COMMENT 'catalog audit info',
- `current_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'catalog current
version',
- `last_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'catalog last
version',
- `deleted_at` BIGINT(20) UNSIGNED NOT NULL DEFAULT 0 COMMENT 'catalog
deleted at',
- PRIMARY KEY (catalog_id),
- CONSTRAINT uk_mid_cn_del UNIQUE (metalake_id, catalog_name, deleted_at)
-) ENGINE=InnoDB;
-
-
-CREATE TABLE IF NOT EXISTS `schema_meta` (
- `schema_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'schema id',
- `schema_name` VARCHAR(128) NOT NULL COMMENT 'schema name',
- `metalake_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'metalake id',
- `catalog_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'catalog id',
- `schema_comment` VARCHAR(256) DEFAULT '' COMMENT 'schema comment',
- `properties` MEDIUMTEXT DEFAULT NULL COMMENT 'schema properties',
- `audit_info` MEDIUMTEXT NOT NULL COMMENT 'schema audit info',
- `current_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'schema current
version',
- `last_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'schema last
version',
- `deleted_at` BIGINT(20) UNSIGNED NOT NULL DEFAULT 0 COMMENT 'schema
deleted at',
- PRIMARY KEY (schema_id),
- CONSTRAINT uk_cid_sn_del UNIQUE (catalog_id, schema_name, deleted_at),
- -- Aliases are used here, and indexes with the same name in H2 can only be
created once.
- KEY idx_smid (metalake_id)
-) ENGINE=InnoDB;
-
-CREATE TABLE IF NOT EXISTS `table_meta` (
- `table_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'table id',
- `table_name` VARCHAR(128) NOT NULL COMMENT 'table name',
- `metalake_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'metalake id',
- `catalog_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'catalog id',
- `schema_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'schema id',
- `audit_info` MEDIUMTEXT NOT NULL COMMENT 'table audit info',
- `current_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'table current
version',
- `last_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'table last
version',
- `deleted_at` BIGINT(20) UNSIGNED NOT NULL DEFAULT 0 COMMENT 'table deleted
at',
- PRIMARY KEY (table_id),
- CONSTRAINT uk_sid_tn_del UNIQUE (schema_id, table_name, deleted_at),
- -- Aliases are used here, and indexes with the same name in H2 can only be
created once.
- KEY idx_tmid (metalake_id),
- KEY idx_tcid (catalog_id)
-) ENGINE=InnoDB;
-
-
-CREATE TABLE IF NOT EXISTS `fileset_meta` (
- `fileset_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'fileset id',
- `fileset_name` VARCHAR(128) NOT NULL COMMENT 'fileset name',
- `metalake_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'metalake id',
- `catalog_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'catalog id',
- `schema_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'schema id',
- `type` VARCHAR(64) NOT NULL COMMENT 'fileset type',
- `audit_info` MEDIUMTEXT NOT NULL COMMENT 'fileset audit info',
- `current_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'fileset current
version',
- `last_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'fileset last
version',
- `deleted_at` BIGINT(20) UNSIGNED NOT NULL DEFAULT 0 COMMENT 'fileset
deleted at',
- PRIMARY KEY (fileset_id),
- CONSTRAINT uk_sid_fn_del UNIQUE (schema_id, fileset_name, deleted_at),
- -- Aliases are used here, and indexes with the same name in H2 can only be
created once.
- KEY idx_fmid (metalake_id),
- KEY idx_fcid (catalog_id)
-) ENGINE=InnoDB;
-
-
-CREATE TABLE IF NOT EXISTS `fileset_version_info` (
- `id` BIGINT(20) UNSIGNED NOT NULL AUTO_INCREMENT COMMENT 'auto increment
id',
- `metalake_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'metalake id',
- `catalog_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'catalog id',
- `schema_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'schema id',
- `fileset_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'fileset id',
- `version` INT UNSIGNED NOT NULL COMMENT 'fileset info version',
- `fileset_comment` VARCHAR(256) DEFAULT '' COMMENT 'fileset comment',
- `properties` MEDIUMTEXT DEFAULT NULL COMMENT 'fileset properties',
- `storage_location` MEDIUMTEXT DEFAULT NULL COMMENT 'fileset storage
location',
- `deleted_at` BIGINT(20) UNSIGNED NOT NULL DEFAULT 0 COMMENT 'fileset
deleted at',
- PRIMARY KEY (id),
- CONSTRAINT uk_fid_ver_del UNIQUE (fileset_id, version, deleted_at),
- -- Aliases are used here, and indexes with the same name in H2 can only be
created once.
- KEY idx_fvmid (metalake_id),
- KEY idx_fvcid (catalog_id)
-) ENGINE=InnoDB;
-
-CREATE TABLE IF NOT EXISTS `topic_meta` (
- `topic_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'topic id',
- `topic_name` VARCHAR(128) NOT NULL COMMENT 'topic name',
- `metalake_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'metalake id',
- `catalog_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'catalog id',
- `schema_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'schema id',
- `comment` VARCHAR(256) DEFAULT '' COMMENT 'topic comment',
- `properties` MEDIUMTEXT DEFAULT NULL COMMENT 'topic properties',
- `audit_info` MEDIUMTEXT NOT NULL COMMENT 'topic audit info',
- `current_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'topic current
version',
- `last_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'topic last
version',
- `deleted_at` BIGINT(20) UNSIGNED NOT NULL DEFAULT 0 COMMENT 'topic deleted
at',
- PRIMARY KEY (topic_id),
- CONSTRAINT uk_cid_tn_del UNIQUE (schema_id, topic_name, deleted_at),
- -- Aliases are used here, and indexes with the same name in H2 can only be
created once.
- KEY idx_tvmid (metalake_id),
- KEY idx_tvcid (catalog_id)
-) ENGINE=InnoDB;
-
-CREATE TABLE IF NOT EXISTS `user_meta` (
- `user_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'user id',
- `user_name` VARCHAR(128) NOT NULL COMMENT 'username',
- `metalake_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'metalake id',
- `audit_info` MEDIUMTEXT NOT NULL COMMENT 'user audit info',
- `current_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'user current
version',
- `last_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'user last version',
- `deleted_at` BIGINT(20) UNSIGNED NOT NULL DEFAULT 0 COMMENT 'user deleted
at',
- PRIMARY KEY (`user_id`),
- CONSTRAINT `uk_mid_us_del` UNIQUE (`metalake_id`, `user_name`,
`deleted_at`)
-) ENGINE=InnoDB;
-
-CREATE TABLE IF NOT EXISTS `role_meta` (
- `role_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'role id',
- `role_name` VARCHAR(128) NOT NULL COMMENT 'role name',
- `metalake_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'metalake id',
- `properties` MEDIUMTEXT DEFAULT NULL COMMENT 'schema properties',
- `audit_info` MEDIUMTEXT NOT NULL COMMENT 'role audit info',
- `current_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'role current
version',
- `last_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'role last version',
- `deleted_at` BIGINT(20) UNSIGNED NOT NULL DEFAULT 0 COMMENT 'role deleted
at',
- PRIMARY KEY (`role_id`),
- CONSTRAINT `uk_mid_rn_del` UNIQUE (`metalake_id`, `role_name`,
`deleted_at`)
-) ENGINE=InnoDB;
-
-CREATE TABLE IF NOT EXISTS `role_meta_securable_object` (
- `id` BIGINT(20) UNSIGNED NOT NULL AUTO_INCREMENT COMMENT 'auto increment
id',
- `role_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'role id',
- `entity_id` BIGINT(20) NOT NULL COMMENT 'securable object entity id',
- `type` VARCHAR(128) NOT NULL COMMENT 'securable object type',
- `privilege_names` VARCHAR(256) NOT NULL COMMENT 'securable object
privilege names',
- `privilege_conditions` VARCHAR(256) NOT NULL COMMENT 'securable object
privilege conditions',
- `current_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'securable
objectcurrent version',
- `last_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'securable object
last version',
- `deleted_at` BIGINT(20) UNSIGNED NOT NULL DEFAULT 0 COMMENT 'securable
object deleted at',
- PRIMARY KEY (`id`),
- KEY `idx_obj_rid` (`role_id`)
- ) ENGINE=InnoDB;
-
-CREATE TABLE IF NOT EXISTS `user_role_rel` (
- `id` BIGINT(20) UNSIGNED NOT NULL AUTO_INCREMENT COMMENT 'auto increment
id',
- `user_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'user id',
- `role_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'role id',
- `audit_info` MEDIUMTEXT NOT NULL COMMENT 'relation audit info',
- `current_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'relation
current version',
- `last_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'relation last
version',
- `deleted_at` BIGINT(20) UNSIGNED NOT NULL DEFAULT 0 COMMENT 'relation
deleted at',
- PRIMARY KEY (`id`),
- CONSTRAINT `uk_ui_ri_del` UNIQUE (`user_id`, `role_id`, `deleted_at`),
- KEY `idx_rid` (`role_id`)
-) ENGINE=InnoDB;
-
-CREATE TABLE IF NOT EXISTS `group_meta` (
- `group_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'group id',
- `group_name` VARCHAR(128) NOT NULL COMMENT 'group name',
- `metalake_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'metalake id',
- `audit_info` MEDIUMTEXT NOT NULL COMMENT 'group audit info',
- `current_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'group current
version',
- `last_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'group last
version',
- `deleted_at` BIGINT(20) UNSIGNED NOT NULL DEFAULT 0 COMMENT 'group deleted
at',
- PRIMARY KEY (`group_id`),
- CONSTRAINT `uk_mid_gr_del` UNIQUE (`metalake_id`, `group_name`,
`deleted_at`)
- ) ENGINE=InnoDB;
-
-CREATE TABLE IF NOT EXISTS `group_role_rel` (
- `id` BIGINT(20) UNSIGNED NOT NULL AUTO_INCREMENT COMMENT 'auto increment
id',
- `group_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'group id',
- `role_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'role id',
- `audit_info` MEDIUMTEXT NOT NULL COMMENT 'relation audit info',
- `current_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'relation
current version',
- `last_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'relation last
version',
- `deleted_at` BIGINT(20) UNSIGNED NOT NULL DEFAULT 0 COMMENT 'relation
deleted at',
- PRIMARY KEY (`id`),
- CONSTRAINT `uk_gi_ri_del` UNIQUE (`group_id`, `role_id`, `deleted_at`),
- KEY `idx_gid` (`group_id`)
- ) ENGINE=InnoDB;
\ No newline at end of file
diff --git a/docs/gravitino-server-config.md b/docs/gravitino-server-config.md
index 769f99958..cc815f9ee 100644
--- a/docs/gravitino-server-config.md
+++ b/docs/gravitino-server-config.md
@@ -44,7 +44,7 @@ You can also specify filter parameters by setting
configuration entries of the f
| Configuration item | Description
| Default value
| Required | Since
version |
|---------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------------------------------|--------------------------------------------------|---------------|
-| `gravitino.entity.store` | Which storage
implementation to use. Key-value pair storage and relational storage are
currently supported, the default value is `kv`, and the optional value is
`relational`.
| `kv` | No
| 0.1.0 |
+| `gravitino.entity.store` | Which entity storage
implementation to use. Only`relational` storage is currently supported.
| `relational`
| No | 0.1.0
|
| `gravitino.entity.store.kv` | Detailed implementation
of KV storage. `RocksDB` storage is currently supported, and the implementation
is `RocksDBKvBackend`.
|
`RocksDBKvBackend` | No
| 0.1.0 |
| `gravitino.entity.store.kv.rocksdbPath` | The storage path for
RocksDB storage implementation. It supports both absolute and relative path, if
the value is a relative path, the final path is
`${GRAVITINO_HOME}/${PATH_YOU_HAVA_SET}`, default value is
`${GRAVITINO_HOME}/data/rocksdb` | `${GRAVITINO_HOME}/data/rocksdb` | No
| 0.1.0 |
| `gravitino.entity.serde` | The
serialization/deserialization class used to support entity storage. `proto' is
currently supported.
| `proto` | No
| 0.1.0 |
diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml
index 59d89981d..7fa802ef6 100644
--- a/gradle/libs.versions.toml
+++ b/gradle/libs.versions.toml
@@ -124,6 +124,7 @@ jersey-test-framework-core = { group =
"org.glassfish.jersey.test-framework", na
jersey-test-framework-provider-jetty = { group =
"org.glassfish.jersey.test-framework.providers", name =
"jersey-test-framework-provider-jetty", version.ref = "jersey" }
jodd-core = { group = "org.jodd", name = "jodd-core", version.ref = "jodd" }
mockito-core = { group = "org.mockito", name = "mockito-core", version.ref =
"mockito" }
+mockito-inline = { group = "org.mockito", name = "mockito-inline", version.ref
= "mockito" }
hive2-metastore = { group = "org.apache.hive", name = "hive-metastore",
version.ref = "hive2"}
hive2-exec = { group = "org.apache.hive", name = "hive-exec", version.ref =
"hive2"}
hive2-common = { group = "org.apache.hive", name = "hive-common", version.ref
= "hive2"}
diff --git
a/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/util/AbstractIT.java
b/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/util/AbstractIT.java
index 8eacdc885..a763737e8 100644
---
a/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/util/AbstractIT.java
+++
b/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/util/AbstractIT.java
@@ -206,7 +206,7 @@ public class AbstractIT {
LOG.info("Running Gravitino Server in {} mode", testMode);
- if ("true".equals(System.getenv("jdbcBackend"))) {
+ if ("MySQL".equalsIgnoreCase(System.getenv("jdbcBackend"))) {
// Start MySQL docker instance.
META_DATA = TestDatabaseName.MYSQL_JDBC_BACKEND;
containerSuite.startMySQLContainer(META_DATA);
diff --git
a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/store/relational/service/FilesetMetaServiceIT.java
b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/store/relational/service/FilesetMetaServiceIT.java
index 95c8834e1..1222cbd19 100644
---
a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/store/relational/service/FilesetMetaServiceIT.java
+++
b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/store/relational/service/FilesetMetaServiceIT.java
@@ -76,7 +76,7 @@ public class FilesetMetaServiceIT {
@BeforeAll
public static void setup() {
- Assumptions.assumeTrue("true".equals(System.getenv("jdbcBackend")));
+ Assumptions.assumeTrue("MySQL".equals(System.getenv("jdbcBackend")));
TestDatabaseName META_DATA = TestDatabaseName.MYSQL_JDBC_BACKEND;
containerSuite.startMySQLContainer(META_DATA);
MySQLContainer MYSQL_CONTAINER = containerSuite.getMySQLContainer();
@@ -123,7 +123,7 @@ public class FilesetMetaServiceIT {
@Test
public void testDeleteFilesetVersionsByRetentionCount() throws IOException {
- Assumptions.assumeTrue("true".equals(System.getenv("jdbcBackend")));
+ Assumptions.assumeTrue("MySQL".equals(System.getenv("jdbcBackend")));
IdGenerator idGenerator = new RandomIdGenerator();
AuditInfo auditInfo =
AuditInfo.builder().withCreator("creator").withCreateTime(Instant.now()).build();
