This is an automated email from the ASF dual-hosted git repository.
heiming pushed a commit to branch tiered_storage
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/tiered_storage by this push:
new 9891ab1d757 fix config load error
9891ab1d757 is described below
commit 9891ab1d7573b82fdab378503dd882b3a743099d
Author: HeimingZ <[email protected]>
AuthorDate: Thu May 25 16:37:59 2023 +0800
fix config load error
---
.../apache/iotdb/os/fileSystem/OSTsFileOutput.java | 2 +
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 10 +++-
.../iotdb/db/conf/directories/TierManager.java | 62 ++++++++++++++--------
.../iotdb/db/engine/storagegroup/DataRegion.java | 49 ++++++++++-------
.../iotdb/tsfile/common/conf/TSFileConfig.java | 3 +-
.../fileInputFactory/HybridFileInputFactory.java | 28 +++++++---
.../fileOutputFactory/HybridFileOutputFactory.java | 24 +++++++--
.../fileSystem/fsFactory/HybridFSFactory.java | 52 +++++++++++-------
8 files changed, 155 insertions(+), 75 deletions(-)
diff --git
a/object-storage/src/main/java/org/apache/iotdb/os/fileSystem/OSTsFileOutput.java
b/object-storage/src/main/java/org/apache/iotdb/os/fileSystem/OSTsFileOutput.java
index d580355c451..6932623d271 100644
---
a/object-storage/src/main/java/org/apache/iotdb/os/fileSystem/OSTsFileOutput.java
+++
b/object-storage/src/main/java/org/apache/iotdb/os/fileSystem/OSTsFileOutput.java
@@ -30,6 +30,8 @@ import java.nio.ByteBuffer;
public class OSTsFileOutput implements TsFileOutput {
private static final Logger logger =
LoggerFactory.getLogger(OSTsFileOutput.class);
+ public OSTsFileOutput(String filePath, boolean overwrite) throws IOException
{}
+
@Override
public void write(byte[] b) throws IOException {
throw new UnsupportedOperationException();
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
index 78830e0e417..e52f0b1af04 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
@@ -65,6 +65,7 @@ import java.util.concurrent.TimeUnit;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
+import static org.apache.iotdb.commons.conf.IoTDBConstant.OBJECT_STORAGE_DIR;
import static
org.apache.iotdb.tsfile.common.constant.TsFileConstant.PATH_SEPARATOR;
public class IoTDBConfig {
@@ -1252,7 +1253,7 @@ public class IoTDBConfig {
this.timeIndexLevel = TimeIndexLevel.valueOf(timeIndexLevel);
}
- void updatePath() {
+ public void updatePath() {
formulateFolders();
confirmMultiDirStrategy();
}
@@ -1283,13 +1284,20 @@ public class IoTDBConfig {
private void formulateDataDirs(String[][] tierDataDirs) {
for (int i = 0; i < tierDataDirs.length; i++) {
for (int j = 0; j < tierDataDirs[i].length; j++) {
+ if (tierDataDirs[i][j].equals(OBJECT_STORAGE_DIR)) {
+ // Notice: dataNodeId hasn't been initialized
+ tierDataDirs[i][j] =
FSUtils.getOSDefaultPath(getObjectStorageBucket(), dataNodeId);
+ }
switch (FSUtils.getFSType(tierDataDirs[i][j])) {
case HDFS:
tierDataDirs[i][j] = getHdfsDir() + File.separatorChar +
tierDataDirs[i][j];
break;
case LOCAL:
tierDataDirs[i][j] = addDataHomeDir(tierDataDirs[i][j]);
+ break;
case OBJECT_STORAGE:
+ tierDataDirs[i][j] =
FSUtils.getOSDefaultPath(getObjectStorageBucket(), dataNodeId);
+ break;
default:
break;
}
diff --git
a/server/src/main/java/org/apache/iotdb/db/conf/directories/TierManager.java
b/server/src/main/java/org/apache/iotdb/db/conf/directories/TierManager.java
index 81281a46ca0..910ab46bfd4 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/directories/TierManager.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/directories/TierManager.java
@@ -27,6 +27,7 @@ import
org.apache.iotdb.db.conf.directories.strategy.MinFolderOccupiedSpaceFirst
import
org.apache.iotdb.db.conf.directories.strategy.RandomOnDiskUsableSpaceStrategy;
import org.apache.iotdb.db.exception.DiskSpaceInsufficientException;
import org.apache.iotdb.tsfile.fileSystem.FSFactoryProducer;
+import org.apache.iotdb.tsfile.fileSystem.FSType;
import org.apache.iotdb.tsfile.utils.FSUtils;
import org.slf4j.Logger;
@@ -44,11 +45,10 @@ import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
+import java.util.Objects;
import java.util.Set;
import java.util.stream.Collectors;
-import static org.apache.iotdb.commons.conf.IoTDBConstant.OBJECT_STORAGE_DIR;
-
/** The main class of multiple directories. Used to allocate folders to data
files. */
public class TierManager {
private static final Logger logger =
LoggerFactory.getLogger(TierManager.class);
@@ -70,7 +70,13 @@ public class TierManager {
/** total space of each tier, Long.MAX_VALUE when one tier contains remote
storage */
private long[] tierDiskTotalSpace;
- private TierManager() {
+ private TierManager() {}
+
+ public void resetFolders() {
+ if (config.getDataNodeId() == -1) {
+ return;
+ }
+
try {
String strategyName =
Class.forName(config.getMultiDirStrategyClassName()).getSimpleName();
if
(strategyName.equals(MaxDiskUsableSpaceFirstStrategy.class.getSimpleName())) {
@@ -84,34 +90,41 @@ public class TierManager {
logger.error(
"Can't find strategy {} for mult-directories.",
config.getMultiDirStrategyClassName(), e);
}
- resetFolders();
- }
-
- public void resetFolders() {
- if (config.getDataNodeId() == -1) {
- return;
- }
seqTiers.clear();
unSeqTiers.clear();
seqDir2TierLevel.clear();
unSeqDir2TierLevel.clear();
+ config.updatePath();
String[][] tierDirs = config.getTierDataDirs();
for (int i = 0; i < tierDirs.length; ++i) {
for (int j = 0; j < tierDirs[i].length; ++j) {
- if (tierDirs[i][j].equals(OBJECT_STORAGE_DIR)) {
- if (i != tierDirs.length - 1) {
- logger.error("Object Storage can only exist on the last tier.");
- }
- tierDirs[i][j] =
- FSUtils.getOSDefaultPath(config.getObjectStorageBucket(),
config.getDataNodeId());
- } else if (FSUtils.isLocal(tierDirs[i][j])) {
- try {
- tierDirs[i][j] = new File(tierDirs[i][j]).getCanonicalPath();
- } catch (IOException e) {
- logger.error("Fail to get canonical path of data dir {}",
tierDirs[i][j], e);
- }
+ switch (FSUtils.getFSType(tierDirs[i][j])) {
+ case LOCAL:
+ try {
+ tierDirs[i][j] = new File(tierDirs[i][j]).getCanonicalPath();
+ } catch (IOException e) {
+ logger.error("Fail to get canonical path of data dir {}",
tierDirs[i][j], e);
+ }
+ break;
+ case OBJECT_STORAGE:
+ if (!config.isEnableObjectStorage()) {
+ logger.error(
+ "Cannot configure object storage directory when
enable_object_storage is false, use default data dir instead.");
+ tierDirs[i][j] =
+ IoTDBConstant.DEFAULT_BASE_DIR + File.separator +
IoTDBConstant.DATA_FOLDER_NAME;
+ }
+ if (i != tierDirs.length - 1) {
+ logger.error("Object Storage can only exist on the last tier.");
+ }
+ // reset datanode id
+ tierDirs[i][j] =
+ FSUtils.getOSDefaultPath(config.getObjectStorageBucket(),
config.getDataNodeId());
+ break;
+ case HDFS:
+ default:
+ break;
}
}
}
@@ -119,6 +132,7 @@ public class TierManager {
for (int tierLevel = 0; tierLevel < tierDirs.length; ++tierLevel) {
List<String> seqDirs =
Arrays.stream(tierDirs[tierLevel])
+ .filter(Objects::nonNull)
.map(
v ->
FSFactoryProducer.getFSFactory()
@@ -137,6 +151,7 @@ public class TierManager {
List<String> unSeqDirs =
Arrays.stream(tierDirs[tierLevel])
+ .filter(Objects::nonNull)
.map(
v ->
FSFactoryProducer.getFSFactory()
@@ -160,6 +175,9 @@ public class TierManager {
private void mkDataDirs(List<String> folders) {
for (String folder : folders) {
File file = FSFactoryProducer.getFSFactory().getFile(folder);
+ if (FSUtils.getFSType(file) == FSType.OBJECT_STORAGE) {
+ continue;
+ }
if (file.mkdirs()) {
logger.info("folder {} doesn't exist, create it", file.getPath());
} else {
diff --git
a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/DataRegion.java
b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/DataRegion.java
index 54f97a29609..21d0b1b3fc5 100644
---
a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/DataRegion.java
+++
b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/DataRegion.java
@@ -714,7 +714,8 @@ public class DataRegion implements IDataRegionForQuery {
if (tsFilePartitionPath2File.containsKey(tsFilePartitionPath)) {
// check migration: two same name tsfile exists, only keep one
of them
File actualFile =
- deleteDuplicateTsFiles(f,
tsFilePartitionPath2File.get(tsFilePartitionPath));
+ deleteDuplicateMigrationTsFile(
+ f, tsFilePartitionPath2File.get(tsFilePartitionPath));
tsFilePartitionPath2File.put(tsFilePartitionPath, actualFile);
} else {
tsFilePartitionPath2File.put(tsFilePartitionPath, f);
@@ -784,25 +785,35 @@ public class DataRegion implements IDataRegionForQuery {
}
/** Remove the duplicate TsFile and return the actual TsFile (has .tsfile
and .tsfile.resource) */
- private File deleteDuplicateTsFiles(File f1, File f2) {
- File f1Resource = fsFactory.getFile(f1 + RESOURCE_SUFFIX);
- File f2Resource = fsFactory.getFile(f2 + RESOURCE_SUFFIX);
- if (f1.exists() && f1Resource.exists()) {
- if (f2.exists()) {
- f2.delete();
- }
- if (f2Resource.exists()) {
- f2Resource.delete();
- }
- return f1;
+ private File deleteDuplicateMigrationTsFile(File f1, File f2) {
+ int f1Tier = TierManager.getInstance().getFileTierLevel(f1);
+ int f2Tier = TierManager.getInstance().getFileTierLevel(f2);
+ File lowerTierFile = f1Tier < f2Tier ? f1 : f2;
+ File higherTierFile = f1Tier < f2Tier ? f2 : f1;
+ File lowerTierFileResource = fsFactory.getFile(lowerTierFile +
RESOURCE_SUFFIX);
+ File higherTierFileResource = fsFactory.getFile(higherTierFile +
RESOURCE_SUFFIX);
+ if (lowerTierFile.exists() && lowerTierFileResource.exists()) {
+ deleteIfExist(higherTierFile);
+ deleteIfExist(higherTierFileResource);
+ return lowerTierFile;
+ } else if (higherTierFile.exists() && higherTierFileResource.exists()) {
+ deleteIfExist(lowerTierFile);
+ deleteIfExist(lowerTierFileResource);
+ return higherTierFile;
} else {
- if (f1.exists()) {
- f1.delete();
- }
- if (f1Resource.exists()) {
- f1Resource.delete();
- }
- return f2;
+ logger.error(
+ "TsFile status is abnormal, please check {}, {}, {}, {}.",
+ lowerTierFile,
+ lowerTierFileResource,
+ higherTierFile,
+ higherTierFileResource);
+ return lowerTierFile;
+ }
+ }
+
+ private void deleteIfExist(File file) {
+ if (file.exists()) {
+ file.delete();
}
}
diff --git
a/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileConfig.java
b/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileConfig.java
index 9d7813784a9..8712ba0aba5 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileConfig.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileConfig.java
@@ -120,8 +120,7 @@ public class TSFileConfig implements Serializable {
/** Default endian value is BIG_ENDIAN. */
private String endian = "BIG_ENDIAN";
/** Default storage is in local file system */
- // TODO: (haiming) fix the bug that the config is not loaded
- private FSType[] TSFileStorageFs = new FSType[] {FSType.LOCAL,
FSType.OBJECT_STORAGE};
+ private FSType[] TSFileStorageFs = new FSType[] {FSType.LOCAL};
/** Default core-site.xml file path is /etc/hadoop/conf/core-site.xml */
private String coreSitePath = "/etc/hadoop/conf/core-site.xml";
/** Default hdfs-site.xml file path is /etc/hadoop/conf/hdfs-site.xml */
diff --git
a/tsfile/src/main/java/org/apache/iotdb/tsfile/fileSystem/fileInputFactory/HybridFileInputFactory.java
b/tsfile/src/main/java/org/apache/iotdb/tsfile/fileSystem/fileInputFactory/HybridFileInputFactory.java
index f9140af71e9..783f9dc0145 100644
---
a/tsfile/src/main/java/org/apache/iotdb/tsfile/fileSystem/fileInputFactory/HybridFileInputFactory.java
+++
b/tsfile/src/main/java/org/apache/iotdb/tsfile/fileSystem/fileInputFactory/HybridFileInputFactory.java
@@ -27,22 +27,36 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
-import java.util.HashMap;
import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
public class HybridFileInputFactory implements FileInputFactory {
private static final Logger logger =
LoggerFactory.getLogger(HybridFileInputFactory.class);
- private static final Map<FSType, FileInputFactory> inputFactories = new
HashMap<>();
+ private static final Map<FSType, FileInputFactory> inputFactories = new
ConcurrentHashMap<>();
- static {
- inputFactories.put(FSType.LOCAL, new LocalFSInputFactory());
- inputFactories.put(FSType.HDFS, new HDFSInputFactory());
- inputFactories.put(FSType.OBJECT_STORAGE, new OSFileInputFactory());
+ private FileInputFactory getFileInputFactory(FSType fsType) {
+ return inputFactories.compute(
+ fsType,
+ (k, v) -> {
+ if (v != null) {
+ return v;
+ }
+ switch (fsType) {
+ case LOCAL:
+ return new LocalFSInputFactory();
+ case OBJECT_STORAGE:
+ return new OSFileInputFactory();
+ case HDFS:
+ return new HDFSInputFactory();
+ default:
+ return null;
+ }
+ });
}
@Override
public TsFileInput getTsFileInput(String filePath) throws IOException {
FSPath path = FSUtils.parse(filePath);
- return inputFactories.get(path.getFsType()).getTsFileInput(path.getPath());
+ return
getFileInputFactory(path.getFsType()).getTsFileInput(path.getPath());
}
}
diff --git
a/tsfile/src/main/java/org/apache/iotdb/tsfile/fileSystem/fileOutputFactory/HybridFileOutputFactory.java
b/tsfile/src/main/java/org/apache/iotdb/tsfile/fileSystem/fileOutputFactory/HybridFileOutputFactory.java
index 80e628bd393..de971cf306d 100644
---
a/tsfile/src/main/java/org/apache/iotdb/tsfile/fileSystem/fileOutputFactory/HybridFileOutputFactory.java
+++
b/tsfile/src/main/java/org/apache/iotdb/tsfile/fileSystem/fileOutputFactory/HybridFileOutputFactory.java
@@ -33,15 +33,29 @@ public class HybridFileOutputFactory implements
FileOutputFactory {
private static final Logger logger =
LoggerFactory.getLogger(HybridFileOutputFactory.class);
private static final Map<FSType, FileOutputFactory> outputFactories = new
ConcurrentHashMap<>();
- static {
- outputFactories.put(FSType.LOCAL, new LocalFSOutputFactory());
- outputFactories.put(FSType.HDFS, new HDFSOutputFactory());
- outputFactories.put(FSType.OBJECT_STORAGE, new OSFileOutputFactory());
+ private FileOutputFactory getFileOutputFactory(FSType fsType) {
+ return outputFactories.compute(
+ fsType,
+ (k, v) -> {
+ if (v != null) {
+ return v;
+ }
+ switch (fsType) {
+ case LOCAL:
+ return new LocalFSOutputFactory();
+ case OBJECT_STORAGE:
+ return new OSFileOutputFactory();
+ case HDFS:
+ return new HDFSOutputFactory();
+ default:
+ return null;
+ }
+ });
}
@Override
public TsFileOutput getTsFileOutput(String filePath, boolean append) {
FSPath path = FSUtils.parse(filePath);
- return
outputFactories.get(path.getFsType()).getTsFileOutput(path.getPath(), append);
+ return
getFileOutputFactory(path.getFsType()).getTsFileOutput(path.getPath(), append);
}
}
diff --git
a/tsfile/src/main/java/org/apache/iotdb/tsfile/fileSystem/fsFactory/HybridFSFactory.java
b/tsfile/src/main/java/org/apache/iotdb/tsfile/fileSystem/fsFactory/HybridFSFactory.java
index 5e9f76914fa..1fe30a3c2b6 100644
---
a/tsfile/src/main/java/org/apache/iotdb/tsfile/fileSystem/fsFactory/HybridFSFactory.java
+++
b/tsfile/src/main/java/org/apache/iotdb/tsfile/fileSystem/fsFactory/HybridFSFactory.java
@@ -39,34 +39,48 @@ public class HybridFSFactory implements FSFactory {
private static final Logger logger =
LoggerFactory.getLogger(HybridFSFactory.class);
private static final Map<FSType, FSFactory> fsFactories = new
ConcurrentHashMap<>();
- static {
- fsFactories.put(FSType.LOCAL, new LocalFSFactory());
- fsFactories.put(FSType.HDFS, new HDFSFactory());
- fsFactories.put(FSType.OBJECT_STORAGE, new OSFSFactory());
+ private FSFactory getFSFactory(FSType fsType) {
+ return fsFactories.compute(
+ fsType,
+ (k, v) -> {
+ if (v != null) {
+ return v;
+ }
+ switch (fsType) {
+ case LOCAL:
+ return new LocalFSFactory();
+ case OBJECT_STORAGE:
+ return new OSFSFactory();
+ case HDFS:
+ return new HDFSFactory();
+ default:
+ return null;
+ }
+ });
}
@Override
public File getFileWithParent(String pathname) {
FSPath path = FSUtils.parse(pathname);
- return fsFactories.get(path.getFsType()).getFileWithParent(path.getPath());
+ return getFSFactory(path.getFsType()).getFileWithParent(path.getPath());
}
@Override
public File getFile(String pathname) {
FSPath path = FSUtils.parse(pathname);
- return fsFactories.get(path.getFsType()).getFile(path.getPath());
+ return getFSFactory(path.getFsType()).getFile(path.getPath());
}
@Override
public File getFile(String parent, String child) {
FSPath parentPath = FSUtils.parse(parent);
- return
fsFactories.get(parentPath.getFsType()).getFile(parentPath.getPath(), child);
+ return getFSFactory(parentPath.getFsType()).getFile(parentPath.getPath(),
child);
}
@Override
public File getFile(File parent, String child) {
FSType type = FSUtils.getFSType(parent);
- return fsFactories.get(type).getFile(parent, child);
+ return getFSFactory(type).getFile(parent, child);
}
@Override
@@ -77,25 +91,25 @@ public class HybridFSFactory implements FSFactory {
@Override
public BufferedReader getBufferedReader(String filePath) {
FSPath path = FSUtils.parse(filePath);
- return fsFactories.get(path.getFsType()).getBufferedReader(path.getPath());
+ return getFSFactory(path.getFsType()).getBufferedReader(path.getPath());
}
@Override
public BufferedWriter getBufferedWriter(String filePath, boolean append) {
FSPath path = FSUtils.parse(filePath);
- return fsFactories.get(path.getFsType()).getBufferedWriter(path.getPath(),
append);
+ return getFSFactory(path.getFsType()).getBufferedWriter(path.getPath(),
append);
}
@Override
public BufferedInputStream getBufferedInputStream(String filePath) {
FSPath path = FSUtils.parse(filePath);
- return
fsFactories.get(path.getFsType()).getBufferedInputStream(path.getPath());
+ return
getFSFactory(path.getFsType()).getBufferedInputStream(path.getPath());
}
@Override
public BufferedOutputStream getBufferedOutputStream(String filePath) {
FSPath path = FSUtils.parse(filePath);
- return
fsFactories.get(path.getFsType()).getBufferedOutputStream(path.getPath());
+ return
getFSFactory(path.getFsType()).getBufferedOutputStream(path.getPath());
}
@Override
@@ -103,7 +117,7 @@ public class HybridFSFactory implements FSFactory {
FSType srcType = FSUtils.getFSType(srcFile);
FSType destType = FSUtils.getFSType(destFile);
if (srcType == destType) {
- fsFactories.get(destType).moveFile(srcFile, destFile);
+ getFSFactory(destType).moveFile(srcFile, destFile);
} else {
throw new IOException(
String.format("Doesn't support move file from %s to %s.", srcType,
destType));
@@ -115,10 +129,10 @@ public class HybridFSFactory implements FSFactory {
FSType srcType = FSUtils.getFSType(srcFile);
FSType destType = FSUtils.getFSType(destFile);
if (srcType == destType || (srcType == FSType.LOCAL && destType ==
FSType.OBJECT_STORAGE)) {
- fsFactories.get(destType).copyFile(srcFile, destFile);
+ getFSFactory(destType).copyFile(srcFile, destFile);
} else if ((srcType == FSType.LOCAL || srcType == FSType.HDFS)
&& (destType == FSType.LOCAL || destType == FSType.HDFS)) {
- fsFactories.get(FSType.HDFS).copyFile(srcFile, destFile);
+ getFSFactory(FSType.HDFS).copyFile(srcFile, destFile);
} else {
throw new IOException(
String.format("Doesn't support move file from %s to %s.", srcType,
destType));
@@ -128,24 +142,24 @@ public class HybridFSFactory implements FSFactory {
@Override
public File[] listFilesBySuffix(String fileFolder, String suffix) {
FSPath folder = FSUtils.parse(fileFolder);
- return
fsFactories.get(folder.getFsType()).listFilesBySuffix(folder.getPath(), suffix);
+ return
getFSFactory(folder.getFsType()).listFilesBySuffix(folder.getPath(), suffix);
}
@Override
public File[] listFilesByPrefix(String fileFolder, String prefix) {
FSPath folder = FSUtils.parse(fileFolder);
- return
fsFactories.get(folder.getFsType()).listFilesByPrefix(folder.getPath(), prefix);
+ return
getFSFactory(folder.getFsType()).listFilesByPrefix(folder.getPath(), prefix);
}
@Override
public boolean deleteIfExists(File file) throws IOException {
FSType type = FSUtils.getFSType(file);
- return fsFactories.get(type).deleteIfExists(file);
+ return getFSFactory(type).deleteIfExists(file);
}
@Override
public void deleteDirectory(String dir) throws IOException {
FSType type = FSUtils.getFSType(dir);
- fsFactories.get(type).deleteDirectory(dir);
+ getFSFactory(type).deleteDirectory(dir);
}
}
