SteveYurongSu commented on code in PR #13176:
URL: https://github.com/apache/iotdb/pull/13176#discussion_r1724286990
##########
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/thrift/IoTDBDataNodeReceiver.java:
##########
@@ -337,11 +342,35 @@ protected TSStatus loadFileV2(
throws IOException, IllegalPathException {
return req instanceof PipeTransferTsFileSealWithModReq
// TsFile's absolute path will be the second element
- ? loadTsFile(fileAbsolutePaths.get(1))
+ ? (IS_USING_ASYNC_LOAD_TS_FILE_STRATEGY.get()
+ ? loadTsFileAsync(fileAbsolutePaths)
+ : loadTsFileSync(fileAbsolutePaths.get(1)))
: loadSchemaSnapShot(req.getParameters(), fileAbsolutePaths);
}
- private TSStatus loadTsFile(final String fileAbsolutePath) throws
FileNotFoundException {
+ // load tsfile async
+ protected TSStatus loadTsFileAsync(final List<String> absolutePaths) {
+ final String moveToLoadTsFileDirBySequence =
IOTDB_CONFIG.getLoadActiveListeningPipeDir();
+
+ for (String absolutePath : absolutePaths) {
+ final File sourceFile = new File(absolutePath);
+
+ // whether the move folder equal the listening folder
+ if
(moveToLoadTsFileDirBySequence.equals(sourceFile.getParentFile().getAbsolutePath()))
{
+ continue;
+ }
+
+ try {
+ FileUtils.moveFileWithMD5Check(sourceFile, new
File(moveToLoadTsFileDirBySequence));
+ } catch (IOException e) {
+ LOGGER.warn("failed to move {} to {}", absolutePath,
moveToLoadTsFileDirBySequence, e);
+ }
+ }
+
Review Comment:
Check if avtive load is enabled. If active load is disable, we enable it and
log
##########
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/load/active/ActiveLoadTsFileLoader.java:
##########
@@ -252,59 +246,9 @@ private void removeToFailDir(final String filePath) {
final File targetDir = new File(failDir.get());
try {
- moveFileWithMD5Check(sourceFile, targetDir);
+
org.apache.iotdb.commons.utils.FileUtils.moveFileWithMD5Check(sourceFile,
targetDir);
Review Comment:
Please fix import style
##########
iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/connector/protocol/IoTDBConnector.java:
##########
@@ -116,6 +120,8 @@ public abstract class IoTDBConnector implements
PipeConnector {
protected String loadBalanceStrategy;
+ protected String loadTsFileStrategy;
Review Comment:
Add an IT for it
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
