This is an automated email from the ASF dual-hosted git repository.
heiming pushed a commit to branch tiered_storage
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/tiered_storage by this push:
new 9b4ae42f903 add class path in the tsfile config
9b4ae42f903 is described below
commit 9b4ae42f903fa58a640f0cdbb16790e19441cbc6
Author: HeimingZ <[email protected]>
AuthorDate: Mon May 29 15:57:17 2023 +0800
add class path in the tsfile config
---
.../iotdb/tsfile/common/conf/TSFileConfig.java | 60 ++++++++++++++++++++++
.../fileInputFactory/HDFSInputFactory.java | 8 +--
.../fileInputFactory/OSFileInputFactory.java | 9 ++--
.../fileOutputFactory/HDFSOutputFactory.java | 8 +--
.../fileOutputFactory/OSFileOutputFactory.java | 10 ++--
.../tsfile/fileSystem/fsFactory/HDFSFactory.java | 33 ++++++------
.../tsfile/fileSystem/fsFactory/OSFSFactory.java | 35 +++++++------
.../org/apache/iotdb/tsfile/utils/FSUtils.java | 9 +++-
8 files changed, 122 insertions(+), 50 deletions(-)
diff --git
a/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileConfig.java
b/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileConfig.java
index 8712ba0aba5..1442a1d418f 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileConfig.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileConfig.java
@@ -160,6 +160,18 @@ public class TSFileConfig implements Serializable {
private int patternMatchingThreshold = 1000000;
+ private String hdfsFile = "org.apache.iotdb.hadoop.fileSystem.HDFSFile";
+
+ private String hdfsTsFileInput =
"org.apache.iotdb.hadoop.fileSystem.HDFSInput";
+
+ private String hdfsTsFileOutput =
"org.apache.iotdb.hadoop.fileSystem.HDFSOutput";
+
+ private String objectStorageFile = "org.apache.iotdb.os.fileSystem.OSFile";
+
+ private String objectStorageTsFileInput =
"org.apache.iotdb.os.fileSystem.OSTsFileInput";
+
+ private String objectStorageTsFileOutput =
"org.apache.iotdb.os.fileSystem.OSTsFileOutput";
+
/** customizedProperties, this should be empty by default. */
private Properties customizedProperties = new Properties();
@@ -481,4 +493,52 @@ public class TSFileConfig implements Serializable {
public String getSprintzPredictScheme() {
return "fire";
}
+
+ public String getHdfsFile() {
+ return hdfsFile;
+ }
+
+ public void setHdfsFile(String hdfsFile) {
+ this.hdfsFile = hdfsFile;
+ }
+
+ public String getHdfsTsFileInput() {
+ return hdfsTsFileInput;
+ }
+
+ public void setHdfsTsFileInput(String hdfsTsFileInput) {
+ this.hdfsTsFileInput = hdfsTsFileInput;
+ }
+
+ public String getHdfsTsFileOutput() {
+ return hdfsTsFileOutput;
+ }
+
+ public void setHdfsTsFileOutput(String hdfsTsFileOutput) {
+ this.hdfsTsFileOutput = hdfsTsFileOutput;
+ }
+
+ public String getObjectStorageFile() {
+ return objectStorageFile;
+ }
+
+ public void setObjectStorageFile(String objectStorageFile) {
+ this.objectStorageFile = objectStorageFile;
+ }
+
+ public String getObjectStorageTsFileInput() {
+ return objectStorageTsFileInput;
+ }
+
+ public void setObjectStorageTsFileInput(String objectStorageTsFileInput) {
+ this.objectStorageTsFileInput = objectStorageTsFileInput;
+ }
+
+ public String getObjectStorageTsFileOutput() {
+ return objectStorageTsFileOutput;
+ }
+
+ public void setObjectStorageTsFileOutput(String objectStorageTsFileOutput) {
+ this.objectStorageTsFileOutput = objectStorageTsFileOutput;
+ }
}
diff --git
a/tsfile/src/main/java/org/apache/iotdb/tsfile/fileSystem/fileInputFactory/HDFSInputFactory.java
b/tsfile/src/main/java/org/apache/iotdb/tsfile/fileSystem/fileInputFactory/HDFSInputFactory.java
index 6cd1c54050b..6735df82416 100644
---
a/tsfile/src/main/java/org/apache/iotdb/tsfile/fileSystem/fileInputFactory/HDFSInputFactory.java
+++
b/tsfile/src/main/java/org/apache/iotdb/tsfile/fileSystem/fileInputFactory/HDFSInputFactory.java
@@ -19,6 +19,7 @@
package org.apache.iotdb.tsfile.fileSystem.fileInputFactory;
+import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor;
import org.apache.iotdb.tsfile.read.reader.TsFileInput;
import org.slf4j.Logger;
@@ -31,11 +32,12 @@ import java.lang.reflect.InvocationTargetException;
public class HDFSInputFactory implements FileInputFactory {
private static final Logger logger =
LoggerFactory.getLogger(HDFSInputFactory.class);
- private static Constructor constructor;
+ private Constructor constructor;
- static {
+ public HDFSInputFactory() {
try {
- Class<?> clazz =
Class.forName("org.apache.iotdb.hadoop.fileSystem.HDFSInput");
+ Class<?> clazz =
+
Class.forName(TSFileDescriptor.getInstance().getConfig().getHdfsTsFileInput());
constructor = clazz.getConstructor(String.class);
} catch (ClassNotFoundException | NoSuchMethodException e) {
logger.error(
diff --git
a/tsfile/src/main/java/org/apache/iotdb/tsfile/fileSystem/fileInputFactory/OSFileInputFactory.java
b/tsfile/src/main/java/org/apache/iotdb/tsfile/fileSystem/fileInputFactory/OSFileInputFactory.java
index eac57b971d1..ce397f5edc3 100644
---
a/tsfile/src/main/java/org/apache/iotdb/tsfile/fileSystem/fileInputFactory/OSFileInputFactory.java
+++
b/tsfile/src/main/java/org/apache/iotdb/tsfile/fileSystem/fileInputFactory/OSFileInputFactory.java
@@ -18,6 +18,7 @@
*/
package org.apache.iotdb.tsfile.fileSystem.fileInputFactory;
+import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor;
import org.apache.iotdb.tsfile.read.reader.TsFileInput;
import org.slf4j.Logger;
@@ -29,12 +30,12 @@ import java.lang.reflect.InvocationTargetException;
public class OSFileInputFactory implements FileInputFactory {
private static final Logger logger =
LoggerFactory.getLogger(OSFileInputFactory.class);
- private static final String OS_INPUT_CLASS_NAME =
"org.apache.iotdb.os.fileSystem.OSTsFileInput";
- private static Constructor constructor;
+ private Constructor constructor;
- static {
+ public OSFileInputFactory() {
try {
- Class<?> clazz = Class.forName(OS_INPUT_CLASS_NAME);
+ Class<?> clazz =
+
Class.forName(TSFileDescriptor.getInstance().getConfig().getObjectStorageTsFileInput());
constructor = clazz.getConstructor(String.class);
} catch (ClassNotFoundException | NoSuchMethodException e) {
logger.error(
diff --git
a/tsfile/src/main/java/org/apache/iotdb/tsfile/fileSystem/fileOutputFactory/HDFSOutputFactory.java
b/tsfile/src/main/java/org/apache/iotdb/tsfile/fileSystem/fileOutputFactory/HDFSOutputFactory.java
index b1d3675047a..914801d38c9 100644
---
a/tsfile/src/main/java/org/apache/iotdb/tsfile/fileSystem/fileOutputFactory/HDFSOutputFactory.java
+++
b/tsfile/src/main/java/org/apache/iotdb/tsfile/fileSystem/fileOutputFactory/HDFSOutputFactory.java
@@ -19,6 +19,7 @@
package org.apache.iotdb.tsfile.fileSystem.fileOutputFactory;
+import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor;
import org.apache.iotdb.tsfile.write.writer.TsFileOutput;
import org.slf4j.Logger;
@@ -30,11 +31,12 @@ import java.lang.reflect.InvocationTargetException;
public class HDFSOutputFactory implements FileOutputFactory {
private static final Logger logger =
LoggerFactory.getLogger(HDFSOutputFactory.class);
- private static Constructor constructor;
+ private Constructor constructor;
- static {
+ public HDFSOutputFactory() {
try {
- Class<?> clazz =
Class.forName("org.apache.iotdb.hadoop.fileSystem.HDFSOutput");
+ Class<?> clazz =
+
Class.forName(TSFileDescriptor.getInstance().getConfig().getHdfsTsFileOutput());
constructor = clazz.getConstructor(String.class, boolean.class);
} catch (ClassNotFoundException | NoSuchMethodException e) {
logger.error(
diff --git
a/tsfile/src/main/java/org/apache/iotdb/tsfile/fileSystem/fileOutputFactory/OSFileOutputFactory.java
b/tsfile/src/main/java/org/apache/iotdb/tsfile/fileSystem/fileOutputFactory/OSFileOutputFactory.java
index e96b86f1af8..9bb723c0b4f 100644
---
a/tsfile/src/main/java/org/apache/iotdb/tsfile/fileSystem/fileOutputFactory/OSFileOutputFactory.java
+++
b/tsfile/src/main/java/org/apache/iotdb/tsfile/fileSystem/fileOutputFactory/OSFileOutputFactory.java
@@ -18,6 +18,7 @@
*/
package org.apache.iotdb.tsfile.fileSystem.fileOutputFactory;
+import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor;
import org.apache.iotdb.tsfile.write.writer.TsFileOutput;
import org.slf4j.Logger;
@@ -28,13 +29,12 @@ import java.lang.reflect.InvocationTargetException;
public class OSFileOutputFactory implements FileOutputFactory {
private static final Logger logger =
LoggerFactory.getLogger(OSFileOutputFactory.class);
- private static final String OS_OUTPUT_CLASS_NAME =
- "org.apache.iotdb.os.fileSystem.OSTsFileOutput";
- private static Constructor constructor;
+ private Constructor constructor;
- static {
+ public OSFileOutputFactory() {
try {
- Class<?> clazz = Class.forName(OS_OUTPUT_CLASS_NAME);
+ Class<?> clazz =
+
Class.forName(TSFileDescriptor.getInstance().getConfig().getObjectStorageTsFileOutput());
constructor = clazz.getConstructor(String.class, boolean.class);
} catch (ClassNotFoundException | NoSuchMethodException e) {
logger.error(
diff --git
a/tsfile/src/main/java/org/apache/iotdb/tsfile/fileSystem/fsFactory/HDFSFactory.java
b/tsfile/src/main/java/org/apache/iotdb/tsfile/fileSystem/fsFactory/HDFSFactory.java
index 057ccf2cf29..fac32d0a756 100644
---
a/tsfile/src/main/java/org/apache/iotdb/tsfile/fileSystem/fsFactory/HDFSFactory.java
+++
b/tsfile/src/main/java/org/apache/iotdb/tsfile/fileSystem/fsFactory/HDFSFactory.java
@@ -19,6 +19,7 @@
package org.apache.iotdb.tsfile.fileSystem.fsFactory;
+import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor;
import org.apache.iotdb.tsfile.fileSystem.FSType;
import org.apache.iotdb.tsfile.utils.FSUtils;
@@ -39,24 +40,24 @@ import java.net.URI;
public class HDFSFactory implements FSFactory {
private static final Logger logger =
LoggerFactory.getLogger(HDFSFactory.class);
- private static Constructor constructorWithPathname;
- private static Constructor constructorWithParentStringAndChild;
- private static Constructor constructorWithParentFileAndChild;
- private static Constructor constructorWithUri;
- private static Method getBufferedReader;
- private static Method getBufferedWriter;
- private static Method getBufferedInputStream;
- private static Method getBufferedOutputStream;
- private static Method listFilesBySuffix;
- private static Method listFilesByPrefix;
- private static Method renameTo;
- private static Method copyToLocal;
- private static Method copyFromLocal;
- private static Method copyTo;
+ private Constructor constructorWithPathname;
+ private Constructor constructorWithParentStringAndChild;
+ private Constructor constructorWithParentFileAndChild;
+ private Constructor constructorWithUri;
+ private Method getBufferedReader;
+ private Method getBufferedWriter;
+ private Method getBufferedInputStream;
+ private Method getBufferedOutputStream;
+ private Method listFilesBySuffix;
+ private Method listFilesByPrefix;
+ private Method renameTo;
+ private Method copyToLocal;
+ private Method copyFromLocal;
+ private Method copyTo;
- static {
+ public HDFSFactory() {
try {
- Class<?> clazz =
Class.forName("org.apache.iotdb.hadoop.fileSystem.HDFSFile");
+ Class<?> clazz =
Class.forName(TSFileDescriptor.getInstance().getConfig().getHdfsFile());
constructorWithPathname = clazz.getConstructor(String.class);
constructorWithParentStringAndChild = clazz.getConstructor(String.class,
String.class);
constructorWithParentFileAndChild = clazz.getConstructor(File.class,
String.class);
diff --git
a/tsfile/src/main/java/org/apache/iotdb/tsfile/fileSystem/fsFactory/OSFSFactory.java
b/tsfile/src/main/java/org/apache/iotdb/tsfile/fileSystem/fsFactory/OSFSFactory.java
index 6f9fe4e49ab..bed220c2b63 100644
---
a/tsfile/src/main/java/org/apache/iotdb/tsfile/fileSystem/fsFactory/OSFSFactory.java
+++
b/tsfile/src/main/java/org/apache/iotdb/tsfile/fileSystem/fsFactory/OSFSFactory.java
@@ -18,6 +18,7 @@
*/
package org.apache.iotdb.tsfile.fileSystem.fsFactory;
+import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor;
import org.apache.iotdb.tsfile.fileSystem.FSType;
import org.apache.iotdb.tsfile.utils.FSUtils;
@@ -37,26 +38,26 @@ import java.net.URI;
public class OSFSFactory implements FSFactory {
private static final Logger logger =
LoggerFactory.getLogger(OSFSFactory.class);
- private static final String OS_FILE_CLASS_NAME =
"org.apache.iotdb.os.fileSystem.OSFile";
- private static Constructor constructorWithPathname;
- private static Constructor constructorWithParentStringAndChild;
- private static Constructor constructorWithParentFileAndChild;
- private static Constructor constructorWithUri;
- private static Method getBufferedReader;
- private static Method getBufferedWriter;
- private static Method getBufferedInputStream;
- private static Method getBufferedOutputStream;
- private static Method listFilesBySuffix;
- private static Method listFilesByPrefix;
- private static Method renameTo;
- private static Method putFile;
- private static Method copyTo;
- private static Method deleteObjectsByPrefix;
+ private Constructor constructorWithPathname;
+ private Constructor constructorWithParentStringAndChild;
+ private Constructor constructorWithParentFileAndChild;
+ private Constructor constructorWithUri;
+ private Method getBufferedReader;
+ private Method getBufferedWriter;
+ private Method getBufferedInputStream;
+ private Method getBufferedOutputStream;
+ private Method listFilesBySuffix;
+ private Method listFilesByPrefix;
+ private Method renameTo;
+ private Method putFile;
+ private Method copyTo;
+ private Method deleteObjectsByPrefix;
- static {
+ public OSFSFactory() {
try {
- Class<?> clazz = Class.forName(OS_FILE_CLASS_NAME);
+ Class<?> clazz =
+
Class.forName(TSFileDescriptor.getInstance().getConfig().getObjectStorageFile());
constructorWithPathname = clazz.getConstructor(String.class);
constructorWithParentStringAndChild = clazz.getConstructor(String.class,
String.class);
constructorWithParentFileAndChild = clazz.getConstructor(File.class,
String.class);
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/utils/FSUtils.java
b/tsfile/src/main/java/org/apache/iotdb/tsfile/utils/FSUtils.java
index c5f80343d70..bd74dccc1d9 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/utils/FSUtils.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/utils/FSUtils.java
@@ -18,6 +18,7 @@
*/
package org.apache.iotdb.tsfile.utils;
+import org.apache.iotdb.tsfile.common.conf.TSFileConfig;
import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor;
import org.apache.iotdb.tsfile.fileSystem.FSPath;
import org.apache.iotdb.tsfile.fileSystem.FSType;
@@ -31,14 +32,16 @@ import java.util.Arrays;
public class FSUtils {
private static final Logger logger = LoggerFactory.getLogger(FSUtils.class);
+ private static final TSFileConfig config =
TSFileDescriptor.getInstance().getConfig();
private static final FSType[] fsTypes = {FSType.OBJECT_STORAGE, FSType.HDFS};
public static final int PATH_FROM_SEQUENCE_LEVEL = 5;
public static final int PATH_FROM_DATABASE_LEVEL = 4;
public static final String[] fsPrefix = {"os://", "hdfs://"};
public static final String OS_FILE_SEPARATOR = "/";
private static final String[] fsFileClassName = {
- "org.apache.iotdb.os.fileSystem.OSFile",
"org.apache.iotdb.hadoop.fileSystem.HDFSFile"
+ config.getObjectStorageFile(), config.getHdfsFile()
};
+
private static final boolean[] isSupported = new boolean[fsTypes.length];
private static final Class<?>[] fsFileClass = new Class[fsTypes.length];
@@ -49,7 +52,9 @@ public class FSUtils {
}
public static synchronized void reload() {
- for (FSType fsType :
TSFileDescriptor.getInstance().getConfig().getTSFileStorageFs()) {
+ fsFileClassName[0] = config.getObjectStorageFile();
+ fsFileClassName[1] = config.getHdfsFile();
+ for (FSType fsType : config.getTSFileStorageFs()) {
if (fsType == FSType.OBJECT_STORAGE) {
isSupported[0] = true;
} else if (fsType == FSType.HDFS) {
