This is an automated email from the ASF dual-hosted git repository.
qiaojialin pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new c58cc7e6b4 [IOTDB-2994] Separate RSchemaEngine into a separate module
(#5737)
c58cc7e6b4 is described below
commit c58cc7e6b45730fc7dcb0e9fb2bd244b73bdc614
Author: Liu Xuxin <[email protected]>
AuthorDate: Tue May 3 11:31:38 2022 +0800
[IOTDB-2994] Separate RSchemaEngine into a separate module (#5737)
---
.../Data-Modeling/SchemaRegion-rocksdb.md | 19 +++-
.../Data-Modeling/SchemaRegion-rocksdb.md | 12 ++-
pom.xml | 1 +
schema-engine-rocksdb/README.md | 38 +++++++
schema-engine-rocksdb/pom.xml | 80 +++++++++++++++
.../resources/conf/schema-rocksdb.properties | 0
.../src/assembly/schema-engine-rocksdb.xml | 44 ++++++++
.../schemaregion/rocksdb/CheckKeyResult.java | 0
.../schemaregion/rocksdb/RSchemaConfLoader.java | 0
.../schemaregion/rocksdb/RSchemaConstants.java | 0
.../schemaregion/rocksdb/RSchemaLogger.java | 0
.../rocksdb/RSchemaReadWriteHandler.java | 0
.../schemaregion/rocksdb/RSchemaRegion.java | 23 ++---
.../schemaregion/rocksdb/RSchemaUtils.java | 12 +--
.../schemaregion/rocksdb/mnode/REntityMNode.java | 0
.../schemaregion/rocksdb/mnode/RInternalMNode.java | 0
.../schemaregion/rocksdb/mnode/RMNode.java | 0
.../schemaregion/rocksdb/mnode/RMNodeType.java | 0
.../rocksdb/mnode/RMNodeValueType.java | 0
.../rocksdb/mnode/RMeasurementMNode.java | 0
.../rocksdb/mnode/RStorageGroupMNode.java | 0
.../metadata}/rocksdb/MRocksDBBenchmark.java | 0
.../metadata}/rocksdb/MRocksDBUnitTest.java | 0
.../rocksdb/RSchemaReadWriteHandlerTest.java | 0
.../rocksdb/RSchemaRegionAdvancedTest.java | 0
.../metadata}/rocksdb/RocksDBBenchmarkEngine.java | 0
.../metadata}/rocksdb/RocksDBBenchmarkTask.java | 0
.../metadata}/rocksdb/RocksDBTestUtils.java | 0
server/pom.xml | 5 -
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 14 ++-
.../metadata/schemaregion/RSchemaRegionLoader.java | 111 +++++++++++++++++++++
.../db/metadata/schemaregion/SchemaEngine.java | 14 +--
32 files changed, 329 insertions(+), 44 deletions(-)
diff --git a/docs/UserGuide/Data-Modeling/SchemaRegion-rocksdb.md
b/docs/UserGuide/Data-Modeling/SchemaRegion-rocksdb.md
index afa3daa150..4aca0fdaab 100644
--- a/docs/UserGuide/Data-Modeling/SchemaRegion-rocksdb.md
+++ b/docs/UserGuide/Data-Modeling/SchemaRegion-rocksdb.md
@@ -21,11 +21,25 @@
# Background
-When IoTDB service is started, metadata information is organized by loading
log file `mlog.bin` and the results are held in memory for a long time. As
metadata continues to grow, memory continues to grow. In order to support the
controllable fluctuation in the massive metadata scenario, we provide a
metadata storage type based on rocksDB.
+When IoTDB service is started, metadata information is organized by loading
log file `mlog.bin` and the results are held
+in memory for a long time. As metadata continues to grow, memory continues to
grow. In order to support the controllable
+fluctuation in the massive metadata scenario, we provide a metadata storage
type based on rocksDB.
# Usage
-In the system configuration file `iotdb-engine.properties`, change the
`schema_engine_mode` to `Rocksdb_based`, for example
+Firstly, you should package **schema-engine-rocksdb** by the following command:
+
+```shell
+mvn clean package -pl schema-engine-rocksdb -am -DskipTests
+```
+
+After that, you can get a **conf** directory and a **lib** directory in
+schema-engine-rocksdb/target/schema-engine-rocksdb. Copy the file in the conf
directory to the conf directory of server,
+and copy the files in the lib directory to the lib directory of server.
+
+Then, open the **iotdb-engine.properties** in the conf directory of server,
and set the `schema_engine_mode` to
+Rocksdb_based. Restart the IoTDB, the system will use `RSchemaRegion` to
manage the metadata.
+
```
####################
### Schema Engine Configuration
@@ -33,6 +47,7 @@ In the system configuration file `iotdb-engine.properties`,
change the `schema_e
# Choose the mode of schema engine. The value could be Memory,Schema_File and
Rocksdb_based. If the provided value doesn't match any pre-defined value,
Memory mode will be used as default.
# Datatype: string
schema_engine_mode=Rocksdb_based
+
```
When rocksdb is specified as the metadata storage type, configuration
parameters of rocksDB are open to the public as file. You can modify the
configuration file `schema-rocksdb.properties` to adjust parameters according
to your own requirements, such as block cache. If there is no special
requirement, use the default value.
diff --git a/docs/zh/UserGuide/Data-Modeling/SchemaRegion-rocksdb.md
b/docs/zh/UserGuide/Data-Modeling/SchemaRegion-rocksdb.md
index f4d3f833b2..413bd3d3b7 100644
--- a/docs/zh/UserGuide/Data-Modeling/SchemaRegion-rocksdb.md
+++ b/docs/zh/UserGuide/Data-Modeling/SchemaRegion-rocksdb.md
@@ -25,7 +25,17 @@
# 使用
-在系统配置文件`iotdb-engine.properties`中，将配置项`schema_engine_mode`修改为`Rocksdb_based`，如：
+首先使用下面的命令将 `schema-engine-rocksdb` 打包
+
+```shell
+mvn clean package -pl schema-engine-rocksdb -am -DskipTests
+```
+
+命令运行结束后，在其 target/schema-engine-rocksdb 中会有一个 lib 文件夹和 conf 文件夹。将 conf
文件夹下的文件拷贝到 server 的 conf 文件夹中，将 lib 文件夹下的文件也拷贝到
+server 的 lib 的文件夹中。
+
+在系统配置文件`iotdb-engine.properties`中，将配置项`schema_engine_mode`修改为`Rocksdb_based`，
如：
+
```
####################
### Schema Engine Configuration
diff --git a/pom.xml b/pom.xml
index 2840b10274..43f497a7a4 100644
--- a/pom.xml
+++ b/pom.xml
@@ -117,6 +117,7 @@
<module>integration</module>
<module>consensus</module>
<module>procedure</module>
+ <module>schema-engine-rocksdb</module>
<!-- <module>library-udf</module>-->
</modules>
<!-- Properties Management -->
diff --git a/schema-engine-rocksdb/README.md b/schema-engine-rocksdb/README.md
new file mode 100644
index 0000000000..6480b6a99f
--- /dev/null
+++ b/schema-engine-rocksdb/README.md
@@ -0,0 +1,38 @@
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+`RSchemaRegion` is an implementation of `SchemaRegion` based on
[RocksDB](http://rocksdb.org/). It performs better than
+pure in-memory schema management in the scenario that tens of billions of time
series is registered.
+
+# How To Use
+
+Firstly, you should package **schema-engine-rocksdb** by the following command:
+
+```shell
+mvn clean package -pl schema-engine-rocksdb -am -DskipTests
+```
+
+After that, you can get a **conf** directory and a **lib** directory in
+schema-engine-rocksdb/target/schema-engine-rocksdb. Copy the file in the conf
directory to the conf directory of server,
+and copy the files in the lib directory to the lib directory of server.
+
+Then, open the **iotdb-engine.properties** in the conf directory of server,
and set the `schema_engine_mode` to
+Rocksdb_based, set the `enable_last_cache` to false. Restart the IoTDB, the
system will use `RSchemaRegion` to manage
+the metadata.
\ No newline at end of file
diff --git a/schema-engine-rocksdb/pom.xml b/schema-engine-rocksdb/pom.xml
new file mode 100644
index 0000000000..90014f0fe5
--- /dev/null
+++ b/schema-engine-rocksdb/pom.xml
@@ -0,0 +1,80 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0";
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
http://maven.apache.org/xsd/maven-4.0.0.xsd";>
+ <parent>
+ <artifactId>iotdb-parent</artifactId>
+ <groupId>org.apache.iotdb</groupId>
+ <version>0.14.0-SNAPSHOT</version>
+ </parent>
+ <modelVersion>4.0.0</modelVersion>
+ <artifactId>schema-engine-rocksdb</artifactId>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.iotdb</groupId>
+ <artifactId>iotdb-server</artifactId>
+ <version>${project.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.rocksdb</groupId>
+ <artifactId>rocksdbjni</artifactId>
+ <version>6.27.3</version>
+ </dependency>
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ <version>[${guava.version},)</version>
+ </dependency>
+ </dependencies>
+ <build>
+ <finalName>schema-engine-rocksdb</finalName>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-assembly-plugin</artifactId>
+ <version>${maven.assembly.version}</version>
+ <executions>
+ <!-- Package binaries-->
+ <execution>
+ <id>schema-engine-rocksdb-assembly</id>
+ <phase>package</phase>
+ <goals>
+ <goal>single</goal>
+ </goals>
+ <configuration>
+ <descriptors>
+
<descriptor>src/assembly/schema-engine-rocksdb.xml</descriptor>
+ </descriptors>
+ <appendAssemblyId>false</appendAssemblyId>
+ <archive>
+ <manifest>
+
<addDefaultImplementationEntries>true</addDefaultImplementationEntries>
+
<addDefaultSpecificationEntries>true</addDefaultSpecificationEntries>
+ </manifest>
+ </archive>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+</project>
diff --git a/server/src/assembly/resources/conf/schema-rocksdb.properties
b/schema-engine-rocksdb/src/assembly/resources/conf/schema-rocksdb.properties
similarity index 100%
rename from server/src/assembly/resources/conf/schema-rocksdb.properties
rename to
schema-engine-rocksdb/src/assembly/resources/conf/schema-rocksdb.properties
diff --git a/schema-engine-rocksdb/src/assembly/schema-engine-rocksdb.xml
b/schema-engine-rocksdb/src/assembly/schema-engine-rocksdb.xml
new file mode 100644
index 0000000000..4c99986a09
--- /dev/null
+++ b/schema-engine-rocksdb/src/assembly/schema-engine-rocksdb.xml
@@ -0,0 +1,44 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+<assembly>
+ <id>schema-engine-rocksdb</id>
+ <formats>
+ <format>dir</format>
+ <format>zip</format>
+ </formats>
+ <includeBaseDirectory>false</includeBaseDirectory>
+ <dependencySets>
+ <dependencySet>
+ <outputDirectory>lib/rschema-region</outputDirectory>
+ <includes>
+ <include>org.rocksdb:rocksdbjni</include>
+ <include>org.apache.iotdb:schema-engine-rocksdb</include>
+ </includes>
+ </dependencySet>
+ </dependencySets>
+ <fileSets>
+ <fileSet>
+ <directory>src/assembly/resources</directory>
+ <outputDirectory>${file.separator}</outputDirectory>
+ </fileSet>
+ </fileSets>
+</assembly>
diff --git
a/server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/rocksdb/CheckKeyResult.java
b/schema-engine-rocksdb/src/main/java/org/apache/iotdb/db/metadata/schemaregion/rocksdb/CheckKeyResult.java
similarity index 100%
rename from
server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/rocksdb/CheckKeyResult.java
rename to
schema-engine-rocksdb/src/main/java/org/apache/iotdb/db/metadata/schemaregion/rocksdb/CheckKeyResult.java
diff --git
a/server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/rocksdb/RSchemaConfLoader.java
b/schema-engine-rocksdb/src/main/java/org/apache/iotdb/db/metadata/schemaregion/rocksdb/RSchemaConfLoader.java
similarity index 100%
rename from
server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/rocksdb/RSchemaConfLoader.java
rename to
schema-engine-rocksdb/src/main/java/org/apache/iotdb/db/metadata/schemaregion/rocksdb/RSchemaConfLoader.java
diff --git
a/server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/rocksdb/RSchemaConstants.java
b/schema-engine-rocksdb/src/main/java/org/apache/iotdb/db/metadata/schemaregion/rocksdb/RSchemaConstants.java
similarity index 100%
rename from
server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/rocksdb/RSchemaConstants.java
rename to
schema-engine-rocksdb/src/main/java/org/apache/iotdb/db/metadata/schemaregion/rocksdb/RSchemaConstants.java
diff --git
a/server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/rocksdb/RSchemaLogger.java
b/schema-engine-rocksdb/src/main/java/org/apache/iotdb/db/metadata/schemaregion/rocksdb/RSchemaLogger.java
similarity index 100%
rename from
server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/rocksdb/RSchemaLogger.java
rename to
schema-engine-rocksdb/src/main/java/org/apache/iotdb/db/metadata/schemaregion/rocksdb/RSchemaLogger.java
diff --git
a/server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/rocksdb/RSchemaReadWriteHandler.java
b/schema-engine-rocksdb/src/main/java/org/apache/iotdb/db/metadata/schemaregion/rocksdb/RSchemaReadWriteHandler.java
similarity index 100%
rename from
server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/rocksdb/RSchemaReadWriteHandler.java
rename to
schema-engine-rocksdb/src/main/java/org/apache/iotdb/db/metadata/schemaregion/rocksdb/RSchemaReadWriteHandler.java
diff --git
a/server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/rocksdb/RSchemaRegion.java
b/schema-engine-rocksdb/src/main/java/org/apache/iotdb/db/metadata/schemaregion/rocksdb/RSchemaRegion.java
similarity index 98%
rename from
server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/rocksdb/RSchemaRegion.java
rename to
schema-engine-rocksdb/src/main/java/org/apache/iotdb/db/metadata/schemaregion/rocksdb/RSchemaRegion.java
index 4a604ee5de..5cdcb64a9e 100644
---
a/server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/rocksdb/RSchemaRegion.java
+++
b/schema-engine-rocksdb/src/main/java/org/apache/iotdb/db/metadata/schemaregion/rocksdb/RSchemaRegion.java
@@ -68,6 +68,7 @@ import org.apache.iotdb.db.query.context.QueryContext;
import org.apache.iotdb.db.query.dataset.ShowDevicesResult;
import org.apache.iotdb.db.query.dataset.ShowTimeSeriesResult;
import org.apache.iotdb.db.service.IoTDB;
+import org.apache.iotdb.db.utils.EncodingInferenceUtils;
import org.apache.iotdb.db.utils.SchemaUtils;
import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
@@ -113,9 +114,6 @@ import java.util.concurrent.locks.ReentrantReadWriteLock;
import java.util.function.BiFunction;
import java.util.function.Function;
-import static
org.apache.iotdb.commons.conf.IoTDBConstant.MULTI_LEVEL_PATH_WILDCARD;
-import static
org.apache.iotdb.commons.conf.IoTDBConstant.ONE_LEVEL_PATH_WILDCARD;
-import static org.apache.iotdb.commons.conf.IoTDBConstant.PATH_ROOT;
import static
org.apache.iotdb.db.metadata.schemaregion.rocksdb.RSchemaConstants.ALL_NODE_TYPE_ARRAY;
import static
org.apache.iotdb.db.metadata.schemaregion.rocksdb.RSchemaConstants.DEFAULT_ALIGNED_ENTITY_VALUE;
import static
org.apache.iotdb.db.metadata.schemaregion.rocksdb.RSchemaConstants.DEFAULT_NODE_VALUE;
@@ -125,7 +123,6 @@ import static
org.apache.iotdb.db.metadata.schemaregion.rocksdb.RSchemaConstants
import static
org.apache.iotdb.db.metadata.schemaregion.rocksdb.RSchemaConstants.NODE_TYPE_MEASUREMENT;
import static
org.apache.iotdb.db.metadata.schemaregion.rocksdb.RSchemaConstants.ROOT_STRING;
import static
org.apache.iotdb.db.metadata.schemaregion.rocksdb.RSchemaConstants.TABLE_NAME_TAGS;
-import static
org.apache.iotdb.db.utils.EncodingInferenceUtils.getDefaultEncoding;
public class RSchemaRegion implements ISchemaRegion {
@@ -763,8 +760,8 @@ public class RSchemaRegion implements ISchemaRegion {
private int indexOfFirstWildcard(String[] nodes, int start) {
int index = start;
for (; index < nodes.length; index++) {
- if (ONE_LEVEL_PATH_WILDCARD.equals(nodes[index])
- || MULTI_LEVEL_PATH_WILDCARD.equals(nodes[index])) {
+ if (IoTDBConstant.ONE_LEVEL_PATH_WILDCARD.equals(nodes[index])
+ || IoTDBConstant.MULTI_LEVEL_PATH_WILDCARD.equals(nodes[index])) {
break;
}
}
@@ -774,8 +771,8 @@ public class RSchemaRegion implements ISchemaRegion {
private int indexOfFirstNonWildcard(String[] nodes, int start) {
int index = start;
for (; index < nodes.length; index++) {
- if (!ONE_LEVEL_PATH_WILDCARD.equals(nodes[index])
- && !MULTI_LEVEL_PATH_WILDCARD.equals(nodes[index])) {
+ if (!IoTDBConstant.ONE_LEVEL_PATH_WILDCARD.equals(nodes[index])
+ && !IoTDBConstant.MULTI_LEVEL_PATH_WILDCARD.equals(nodes[index])) {
break;
}
}
@@ -818,7 +815,7 @@ public class RSchemaRegion implements ISchemaRegion {
@Override
public boolean isPathExist(PartialPath path) throws MetadataException {
- if (PATH_ROOT.equals(path.getFullPath())) {
+ if (IoTDBConstant.PATH_ROOT.equals(path.getFullPath())) {
return true;
}
@@ -867,7 +864,7 @@ public class RSchemaRegion implements ISchemaRegion {
public int getNodesCountInGivenLevel(PartialPath pathPattern, int level,
boolean isPrefixMatch)
throws MetadataException {
// todo support wildcard
- if (pathPattern.getFullPath().contains(ONE_LEVEL_PATH_WILDCARD)) {
+ if
(pathPattern.getFullPath().contains(IoTDBConstant.ONE_LEVEL_PATH_WILDCARD)) {
throw new UnsupportedOperationException(
"Wildcards are not currently supported for this operation"
+ " [COUNT NODES pathPattern].");
@@ -946,7 +943,7 @@ public class RSchemaRegion implements ISchemaRegion {
public List<PartialPath> getNodesListInGivenLevel(
PartialPath pathPattern, int nodeLevel, boolean isPrefixMatch,
StorageGroupFilter filter)
throws MetadataException {
- if (pathPattern.getFullPath().contains(ONE_LEVEL_PATH_WILDCARD)) {
+ if
(pathPattern.getFullPath().contains(IoTDBConstant.ONE_LEVEL_PATH_WILDCARD)) {
throw new UnsupportedOperationException(
formatNotSupportInfo(Thread.currentThread().getStackTrace()[1].getMethodName()));
}
@@ -986,7 +983,7 @@ public class RSchemaRegion implements ISchemaRegion {
@Override
public Set<String> getChildNodePathInNextLevel(PartialPath pathPattern)
throws MetadataException {
// todo support wildcard
- if (pathPattern.getFullPath().contains(ONE_LEVEL_PATH_WILDCARD)) {
+ if
(pathPattern.getFullPath().contains(IoTDBConstant.ONE_LEVEL_PATH_WILDCARD)) {
throw new MetadataException(
"Wildcards are not currently supported for this operation"
+ " [SHOW CHILD PATHS pathPattern].");
@@ -1766,7 +1763,7 @@ public class RSchemaRegion implements ISchemaRegion {
measurements.add(measurementList[index]);
TSDataType type = plan.getDataTypes()[index];
dataTypes.add(type);
- encodings.add(getDefaultEncoding(type));
+ encodings.add(EncodingInferenceUtils.getDefaultEncoding(type));
}
createAlignedTimeSeries(devicePath, measurements, dataTypes,
encodings);
} else {
diff --git
a/server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/rocksdb/RSchemaUtils.java
b/schema-engine-rocksdb/src/main/java/org/apache/iotdb/db/metadata/schemaregion/rocksdb/RSchemaUtils.java
similarity index 98%
rename from
server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/rocksdb/RSchemaUtils.java
rename to
schema-engine-rocksdb/src/main/java/org/apache/iotdb/db/metadata/schemaregion/rocksdb/RSchemaUtils.java
index 1ad26f3aa8..ef45309b65 100644
---
a/server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/rocksdb/RSchemaUtils.java
+++
b/schema-engine-rocksdb/src/main/java/org/apache/iotdb/db/metadata/schemaregion/rocksdb/RSchemaUtils.java
@@ -19,6 +19,7 @@
package org.apache.iotdb.db.metadata.schemaregion.rocksdb;
+import org.apache.iotdb.commons.conf.IoTDBConstant;
import org.apache.iotdb.db.exception.metadata.IllegalPathException;
import org.apache.iotdb.db.metadata.mnode.IMNode;
import org.apache.iotdb.db.metadata.path.PartialPath;
@@ -45,9 +46,6 @@ import java.util.Map;
import java.util.Objects;
import java.util.stream.Collectors;
-import static
org.apache.iotdb.commons.conf.IoTDBConstant.MULTI_LEVEL_PATH_WILDCARD;
-import static
org.apache.iotdb.commons.conf.IoTDBConstant.ONE_LEVEL_PATH_WILDCARD;
-import static org.apache.iotdb.commons.conf.IoTDBConstant.PATH_ROOT;
import static
org.apache.iotdb.db.metadata.schemaregion.rocksdb.RSchemaConstants.DATA_BLOCK_TYPE_ALIAS;
import static
org.apache.iotdb.db.metadata.schemaregion.rocksdb.RSchemaConstants.DATA_BLOCK_TYPE_ATTRIBUTES;
import static
org.apache.iotdb.db.metadata.schemaregion.rocksdb.RSchemaConstants.DATA_BLOCK_TYPE_ORIGIN_KEY;
@@ -166,7 +164,7 @@ public class RSchemaUtils {
public static PartialPath getPartialPathFromInnerPath(String path, char
level) {
String pathWithoutLevel = path.replace(PATH_SEPARATOR + level,
PATH_SEPARATOR);
String[] nodes = pathWithoutLevel.split(ESCAPE_PATH_SEPARATOR);
- nodes[0] = PATH_ROOT;
+ nodes[0] = IoTDBConstant.PATH_ROOT;
return new PartialPath(nodes);
}
@@ -508,7 +506,9 @@ public class RSchemaUtils {
public static String replaceWildcard(int num) {
StringBuilder stringBuilder = new StringBuilder();
for (int i = 0; i < num; i++) {
-
stringBuilder.append(RSchemaConstants.PATH_SEPARATOR).append(ONE_LEVEL_PATH_WILDCARD);
+ stringBuilder
+ .append(RSchemaConstants.PATH_SEPARATOR)
+ .append(IoTDBConstant.ONE_LEVEL_PATH_WILDCARD);
}
return stringBuilder.substring(1);
}
@@ -540,7 +540,7 @@ public class RSchemaUtils {
List<String[]> allNodesArray = new ArrayList<>();
List<Integer> multiWildcardPosition = new ArrayList<>();
for (int i = 0; i < nodes.length; i++) {
- if (MULTI_LEVEL_PATH_WILDCARD.equals(nodes[i])) {
+ if (IoTDBConstant.MULTI_LEVEL_PATH_WILDCARD.equals(nodes[i])) {
multiWildcardPosition.add(i);
}
}
diff --git
a/server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/rocksdb/mnode/REntityMNode.java
b/schema-engine-rocksdb/src/main/java/org/apache/iotdb/db/metadata/schemaregion/rocksdb/mnode/REntityMNode.java
similarity index 100%
rename from
server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/rocksdb/mnode/REntityMNode.java
rename to
schema-engine-rocksdb/src/main/java/org/apache/iotdb/db/metadata/schemaregion/rocksdb/mnode/REntityMNode.java
diff --git
a/server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/rocksdb/mnode/RInternalMNode.java
b/schema-engine-rocksdb/src/main/java/org/apache/iotdb/db/metadata/schemaregion/rocksdb/mnode/RInternalMNode.java
similarity index 100%
rename from
server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/rocksdb/mnode/RInternalMNode.java
rename to
schema-engine-rocksdb/src/main/java/org/apache/iotdb/db/metadata/schemaregion/rocksdb/mnode/RInternalMNode.java
diff --git
a/server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/rocksdb/mnode/RMNode.java
b/schema-engine-rocksdb/src/main/java/org/apache/iotdb/db/metadata/schemaregion/rocksdb/mnode/RMNode.java
similarity index 100%
rename from
server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/rocksdb/mnode/RMNode.java
rename to
schema-engine-rocksdb/src/main/java/org/apache/iotdb/db/metadata/schemaregion/rocksdb/mnode/RMNode.java
diff --git
a/server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/rocksdb/mnode/RMNodeType.java
b/schema-engine-rocksdb/src/main/java/org/apache/iotdb/db/metadata/schemaregion/rocksdb/mnode/RMNodeType.java
similarity index 100%
rename from
server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/rocksdb/mnode/RMNodeType.java
rename to
schema-engine-rocksdb/src/main/java/org/apache/iotdb/db/metadata/schemaregion/rocksdb/mnode/RMNodeType.java
diff --git
a/server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/rocksdb/mnode/RMNodeValueType.java
b/schema-engine-rocksdb/src/main/java/org/apache/iotdb/db/metadata/schemaregion/rocksdb/mnode/RMNodeValueType.java
similarity index 100%
rename from
server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/rocksdb/mnode/RMNodeValueType.java
rename to
schema-engine-rocksdb/src/main/java/org/apache/iotdb/db/metadata/schemaregion/rocksdb/mnode/RMNodeValueType.java
diff --git
a/server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/rocksdb/mnode/RMeasurementMNode.java
b/schema-engine-rocksdb/src/main/java/org/apache/iotdb/db/metadata/schemaregion/rocksdb/mnode/RMeasurementMNode.java
similarity index 100%
rename from
server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/rocksdb/mnode/RMeasurementMNode.java
rename to
schema-engine-rocksdb/src/main/java/org/apache/iotdb/db/metadata/schemaregion/rocksdb/mnode/RMeasurementMNode.java
diff --git
a/server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/rocksdb/mnode/RStorageGroupMNode.java
b/schema-engine-rocksdb/src/main/java/org/apache/iotdb/db/metadata/schemaregion/rocksdb/mnode/RStorageGroupMNode.java
similarity index 100%
rename from
server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/rocksdb/mnode/RStorageGroupMNode.java
rename to
schema-engine-rocksdb/src/main/java/org/apache/iotdb/db/metadata/schemaregion/rocksdb/mnode/RStorageGroupMNode.java
diff --git
a/server/src/test/java/org/apache/iotdb/db/metadata/schemaregion/rocksdb/MRocksDBBenchmark.java
b/schema-engine-rocksdb/src/test/org.apache.iotdb.db/metadata/rocksdb/MRocksDBBenchmark.java
similarity index 100%
rename from
server/src/test/java/org/apache/iotdb/db/metadata/schemaregion/rocksdb/MRocksDBBenchmark.java
rename to
schema-engine-rocksdb/src/test/org.apache.iotdb.db/metadata/rocksdb/MRocksDBBenchmark.java
diff --git
a/server/src/test/java/org/apache/iotdb/db/metadata/schemaregion/rocksdb/MRocksDBUnitTest.java
b/schema-engine-rocksdb/src/test/org.apache.iotdb.db/metadata/rocksdb/MRocksDBUnitTest.java
similarity index 100%
rename from
server/src/test/java/org/apache/iotdb/db/metadata/schemaregion/rocksdb/MRocksDBUnitTest.java
rename to
schema-engine-rocksdb/src/test/org.apache.iotdb.db/metadata/rocksdb/MRocksDBUnitTest.java
diff --git
a/server/src/test/java/org/apache/iotdb/db/metadata/schemaregion/rocksdb/RSchemaReadWriteHandlerTest.java
b/schema-engine-rocksdb/src/test/org.apache.iotdb.db/metadata/rocksdb/RSchemaReadWriteHandlerTest.java
similarity index 100%
rename from
server/src/test/java/org/apache/iotdb/db/metadata/schemaregion/rocksdb/RSchemaReadWriteHandlerTest.java
rename to
schema-engine-rocksdb/src/test/org.apache.iotdb.db/metadata/rocksdb/RSchemaReadWriteHandlerTest.java
diff --git
a/server/src/test/java/org/apache/iotdb/db/metadata/schemaregion/rocksdb/RSchemaRegionAdvancedTest.java
b/schema-engine-rocksdb/src/test/org.apache.iotdb.db/metadata/rocksdb/RSchemaRegionAdvancedTest.java
similarity index 100%
rename from
server/src/test/java/org/apache/iotdb/db/metadata/schemaregion/rocksdb/RSchemaRegionAdvancedTest.java
rename to
schema-engine-rocksdb/src/test/org.apache.iotdb.db/metadata/rocksdb/RSchemaRegionAdvancedTest.java
diff --git
a/server/src/test/java/org/apache/iotdb/db/metadata/schemaregion/rocksdb/RocksDBBenchmarkEngine.java
b/schema-engine-rocksdb/src/test/org.apache.iotdb.db/metadata/rocksdb/RocksDBBenchmarkEngine.java
similarity index 100%
rename from
server/src/test/java/org/apache/iotdb/db/metadata/schemaregion/rocksdb/RocksDBBenchmarkEngine.java
rename to
schema-engine-rocksdb/src/test/org.apache.iotdb.db/metadata/rocksdb/RocksDBBenchmarkEngine.java
diff --git
a/server/src/test/java/org/apache/iotdb/db/metadata/schemaregion/rocksdb/RocksDBBenchmarkTask.java
b/schema-engine-rocksdb/src/test/org.apache.iotdb.db/metadata/rocksdb/RocksDBBenchmarkTask.java
similarity index 100%
rename from
server/src/test/java/org/apache/iotdb/db/metadata/schemaregion/rocksdb/RocksDBBenchmarkTask.java
rename to
schema-engine-rocksdb/src/test/org.apache.iotdb.db/metadata/rocksdb/RocksDBBenchmarkTask.java
diff --git
a/server/src/test/java/org/apache/iotdb/db/metadata/schemaregion/rocksdb/RocksDBTestUtils.java
b/schema-engine-rocksdb/src/test/org.apache.iotdb.db/metadata/rocksdb/RocksDBTestUtils.java
similarity index 100%
rename from
server/src/test/java/org/apache/iotdb/db/metadata/schemaregion/rocksdb/RocksDBTestUtils.java
rename to
schema-engine-rocksdb/src/test/org.apache.iotdb.db/metadata/rocksdb/RocksDBTestUtils.java
diff --git a/server/pom.xml b/server/pom.xml
index 791dd03a5a..53615eff58 100644
--- a/server/pom.xml
+++ b/server/pom.xml
@@ -249,11 +249,6 @@
<version>2.6</version>
<scope>compile</scope>
</dependency>
- <dependency>
- <groupId>org.rocksdb</groupId>
- <artifactId>rocksdbjni</artifactId>
- <version>6.27.3</version>
- </dependency>
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
index c1651f66eb..487c9ef441 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
@@ -226,11 +226,6 @@ public class IoTDBDescriptor {
properties.getProperty(
"meta_data_cache_enable",
Boolean.toString(conf.isMetaDataCacheEnable()))));
- conf.setEnableLastCache(
- Boolean.parseBoolean(
- properties.getProperty(
- "enable_last_cache",
Boolean.toString(conf.isLastCacheEnabled()))));
-
initMemoryAllocate(properties);
loadWALProps(properties);
@@ -760,6 +755,15 @@ public class IoTDBDescriptor {
conf.setSchemaEngineMode(
properties.getProperty("schema_engine_mode",
String.valueOf(conf.getSchemaEngineMode())));
+ conf.setEnableLastCache(
+ Boolean.parseBoolean(
+ properties.getProperty(
+ "enable_last_cache",
Boolean.toString(conf.isLastCacheEnabled()))));
+
+ if (conf.getSchemaEngineMode().equals("Rocksdb_based")) {
+ conf.setEnableLastCache(false);
+ }
+
conf.setCachedMNodeSizeInSchemaFileMode(
Integer.parseInt(
properties.getProperty(
diff --git
a/server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/RSchemaRegionLoader.java
b/server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/RSchemaRegionLoader.java
new file mode 100644
index 0000000000..5cf6ae8be9
--- /dev/null
+++
b/server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/RSchemaRegionLoader.java
@@ -0,0 +1,111 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iotdb.db.metadata.schemaregion;
+
+import org.apache.iotdb.commons.consensus.SchemaRegionId;
+import org.apache.iotdb.db.metadata.mnode.IStorageGroupMNode;
+import org.apache.iotdb.db.metadata.path.PartialPath;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.lang.reflect.Constructor;
+import java.lang.reflect.InvocationTargetException;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.net.URLClassLoader;
+import java.util.LinkedList;
+import java.util.List;
+
+public class RSchemaRegionLoader {
+ private static final Logger LOGGER =
LoggerFactory.getLogger(RSchemaRegionLoader.class);
+ private static URLClassLoader urlClassLoader = null;
+ private static final String RSCHEMA_REGION_CLASS_NAME =
+ "org.apache.iotdb.db.metadata.schemaregion.rocksdb.RSchemaRegion";
+ private static final String RSCHEMA_CONF_LOADER_CLASS_NAME =
+ "org.apache.iotdb.db.metadata.schemaregion.rocksdb.RSchemaConfLoader";
+ private static final String LIB_PATH =
+ ".." + File.separator + "lib" + File.separator + "rschema-region" +
File.separator;
+
+ public RSchemaRegionLoader() {}
+
+ /**
+ * Load the jar files for RSchemaRegion and create an instance of it. The
jar files should be
+ * located in "../lib/rschema-region". If jar files cannot be found, the
function will return
+ * null.
+ *
+ * @param storageGroup
+ * @param schemaRegionId
+ * @param node
+ * @return
+ */
+ public ISchemaRegion loadRSchemaRegion(
+ PartialPath storageGroup, SchemaRegionId schemaRegionId,
IStorageGroupMNode node) {
+ ISchemaRegion region = null;
+ LOGGER.info("Creating instance for schema-engine-rocksdb");
+ try {
+ loadRSchemaRegionJar();
+ Class<?> classForRSchemaRegion =
urlClassLoader.loadClass(RSCHEMA_REGION_CLASS_NAME);
+ Class<?> classForRSchemaConfLoader =
urlClassLoader.loadClass(RSCHEMA_CONF_LOADER_CLASS_NAME);
+ Constructor<?> constructor =
+ classForRSchemaRegion.getConstructor(
+ PartialPath.class,
+ SchemaRegionId.class,
+ IStorageGroupMNode.class,
+ classForRSchemaConfLoader);
+ Object rSchemaLoader =
classForRSchemaConfLoader.getConstructor().newInstance();
+ region =
+ (ISchemaRegion)
+ constructor.newInstance(storageGroup, schemaRegionId, node,
rSchemaLoader);
+ } catch (ClassNotFoundException
+ | NoSuchMethodException
+ | InvocationTargetException
+ | InstantiationException
+ | IllegalAccessException
+ | MalformedURLException
+ | RuntimeException e) {
+ LOGGER.error("Cannot initialize RSchemaRegion", e);
+ return null;
+ }
+ return region;
+ }
+
+ /**
+ * Load the jar files for rocksdb and RSchemaRegion. The jar files should be
located in directory
+ * "../lib/rschema-region". If the jar files have been loaded, it will do
nothing.
+ */
+ private void loadRSchemaRegionJar() throws MalformedURLException {
+ LOGGER.info("Loading jar for schema-engine-rocksdb");
+ if (urlClassLoader == null) {
+ File[] jars = new File(LIB_PATH).listFiles();
+ if (jars == null) {
+ throw new RuntimeException(
+ String.format("Cannot get jars from %s", new
File(LIB_PATH).getAbsolutePath()));
+ }
+ List<URL> dependentJars = new LinkedList<>();
+ for (File jar : jars) {
+ if (jar.getName().endsWith(".jar")) {
+ dependentJars.add(new URL("file:" + jar.getAbsolutePath()));
+ }
+ }
+ urlClassLoader = new URLClassLoader(dependentJars.toArray(new URL[] {}));
+ }
+ }
+}
diff --git
a/server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/SchemaEngine.java
b/server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/SchemaEngine.java
index 8398f0ab22..fce9c38aeb 100644
---
a/server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/SchemaEngine.java
+++
b/server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/SchemaEngine.java
@@ -26,8 +26,6 @@ import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.exception.metadata.MetadataException;
import org.apache.iotdb.db.metadata.mnode.IStorageGroupMNode;
import org.apache.iotdb.db.metadata.path.PartialPath;
-import org.apache.iotdb.db.metadata.schemaregion.rocksdb.RSchemaConfLoader;
-import org.apache.iotdb.db.metadata.schemaregion.rocksdb.RSchemaRegion;
import org.apache.iotdb.db.metadata.storagegroup.IStorageGroupSchemaManager;
import org.apache.iotdb.db.metadata.storagegroup.StorageGroupSchemaManager;
@@ -56,7 +54,6 @@ public class SchemaEngine {
private Map<SchemaRegionId, ISchemaRegion> schemaRegionMap;
private SchemaEngineMode schemaRegionStoredMode;
- private RSchemaConfLoader rSchemaConfLoader;
private static final Logger logger =
LoggerFactory.getLogger(SchemaEngine.class);
private static class SchemaEngineManagerHolder {
@@ -217,8 +214,8 @@ public class SchemaEngine {
break;
case Rocksdb_based:
schemaRegion =
- new RSchemaRegion(
- storageGroup, schemaRegionId, storageGroupMNode,
loadRocksdbConfFile());
+ new RSchemaRegionLoader()
+ .loadRSchemaRegion(storageGroup, schemaRegionId,
storageGroupMNode);
break;
default:
throw new UnsupportedOperationException(
@@ -233,11 +230,4 @@ public class SchemaEngine {
schemaRegionMap.get(schemaRegionId).deleteSchemaRegion();
schemaRegionMap.remove(schemaRegionId);
}
-
- private RSchemaConfLoader loadRocksdbConfFile() {
- if (rSchemaConfLoader == null) {
- rSchemaConfLoader = new RSchemaConfLoader();
- }
- return rSchemaConfLoader;
- }
}
