This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new 80dca5c7bb [IOTDB-4688] use streaming md5 computing to replace
blocking md5
80dca5c7bb is described below
commit 80dca5c7bb7df7c9bdcc0259320d00f4296eba37
Author: William Song <[email protected]>
AuthorDate: Wed Oct 19 20:13:10 2022 +0800
[IOTDB-4688] use streaming md5 computing to replace blocking md5
---
.../ratis/FileInfoWithDelayedMd5Computing.java | 75 ----------------------
.../iotdb/consensus/ratis/SnapshotStorage.java | 2 +-
2 files changed, 1 insertion(+), 76 deletions(-)
diff --git
a/consensus/src/main/java/org/apache/iotdb/consensus/ratis/FileInfoWithDelayedMd5Computing.java
b/consensus/src/main/java/org/apache/iotdb/consensus/ratis/FileInfoWithDelayedMd5Computing.java
deleted file mode 100644
index ef584f416c..0000000000
---
a/consensus/src/main/java/org/apache/iotdb/consensus/ratis/FileInfoWithDelayedMd5Computing.java
+++ /dev/null
@@ -1,75 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.iotdb.consensus.ratis;
-
-import org.apache.ratis.io.MD5Hash;
-import org.apache.ratis.server.storage.FileInfo;
-import org.apache.ratis.util.MD5FileUtil;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.nio.file.Path;
-
-/**
- * When Leader sends snapshots to a follower, it will send the file itself
together with md5 digest.
- * When snapshot file is large, this computation incurs significant overhead.
This class implements
- * a strategy that delay the computing when the md5 info is first fetched.
- *
- * <p>TODO A better strategy is to calculate the md5 along when sending the
InstallSnapshot request
- * with file chunks.
- */
-public class FileInfoWithDelayedMd5Computing extends FileInfo {
-
- private static final Logger logger =
- LoggerFactory.getLogger(FileInfoWithDelayedMd5Computing.class);
- private volatile MD5Hash digest;
-
- public FileInfoWithDelayedMd5Computing(Path path, MD5Hash fileDigest) {
- super(path, fileDigest);
- digest = null;
- }
-
- public FileInfoWithDelayedMd5Computing(Path path) {
- this(path, null);
- }
-
- // return null iff sync md5 computing failed
- @Override
- public MD5Hash getFileDigest() {
- if (digest == null) {
- synchronized (this) {
- if (digest == null) {
- try {
- if (MD5FileUtil.getDigestFileForFile(getPath().toFile()).exists())
{
- digest = MD5FileUtil.readStoredMd5ForFile(getPath().toFile());
- }
- digest = MD5FileUtil.computeMd5ForFile(getPath().toFile());
- MD5FileUtil.saveMD5File(getPath().toFile(), digest);
- } catch (IOException ioException) {
- logger.error("compute file digest for {} failed due to {}",
getPath(), ioException);
- return null;
- }
- }
- }
- }
- return digest;
- }
-}
diff --git
a/consensus/src/main/java/org/apache/iotdb/consensus/ratis/SnapshotStorage.java
b/consensus/src/main/java/org/apache/iotdb/consensus/ratis/SnapshotStorage.java
index 5b679fe336..2917ee7e19 100644
---
a/consensus/src/main/java/org/apache/iotdb/consensus/ratis/SnapshotStorage.java
+++
b/consensus/src/main/java/org/apache/iotdb/consensus/ratis/SnapshotStorage.java
@@ -111,7 +111,7 @@ public class SnapshotStorage implements StateMachineStorage
{
if (file.endsWith(".md5")) {
continue;
}
- FileInfo fileInfo = new FileInfoWithDelayedMd5Computing(file);
+ FileInfo fileInfo = new FileInfo(file, null);
fileInfos.add(fileInfo);
}
