anoopsjohn commented on a change in pull request #633: HBASE-22890 Verify the
file integrity in persistent IOEngine
URL: https://github.com/apache/hbase/pull/633#discussion_r325500913
##########
File path:
hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/FileIOEngine.java
##########
@@ -20,45 +20,107 @@
import java.io.File;
import java.io.IOException;
+import java.io.ObjectInputStream;
import java.io.RandomAccessFile;
import java.nio.ByteBuffer;
import java.nio.channels.ClosedByInterruptException;
import java.nio.channels.ClosedChannelException;
import java.nio.channels.FileChannel;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
import java.util.Arrays;
import java.util.concurrent.locks.ReentrantLock;
import com.google.common.annotations.VisibleForTesting;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.util.Shell;
import org.apache.hadoop.util.StringUtils;
/**
* IO engine that stores data to a file on the local file system.
*/
@InterfaceAudience.Private
-public class FileIOEngine implements IOEngine {
+public class FileIOEngine implements PersistentIOEngine {
private static final Log LOG = LogFactory.getLog(FileIOEngine.class);
public static final String FILE_DELIMITER = ",";
+ private static final DuFileCommand DU = new DuFileCommand(new String[]
{"du", ""});
+
private final String[] filePaths;
private final FileChannel[] fileChannels;
private final RandomAccessFile[] rafs;
private final ReentrantLock[] channelLocks;
private final long sizePerFile;
private final long capacity;
+ private final String algorithmName;
+ /**
+ * Whether the persistent file support verify file integrity, old version
file
+ * does not support verification,
+ */
+ private boolean oldVersion;
private FileReadAccessor readAccessor = new FileReadAccessor();
private FileWriteAccessor writeAccessor = new FileWriteAccessor();
- public FileIOEngine(long capacity, String... filePaths) throws IOException {
+ public FileIOEngine(String algorithmName, String persistentPath,
+ long capacity, String... filePaths) throws IOException {
this.sizePerFile = capacity / filePaths.length;
this.capacity = this.sizePerFile * filePaths.length;
this.filePaths = filePaths;
this.fileChannels = new FileChannel[filePaths.length];
this.rafs = new RandomAccessFile[filePaths.length];
this.channelLocks = new ReentrantLock[filePaths.length];
+ this.algorithmName = algorithmName;
+ // not configure persistent path
+ if (persistentPath == null) {
+ deleteCacheDataFile();
+ }
+ init();
+ }
+
+ /**
+ * Verify cache files's integrity
+ * @param persistentPath the backingMap persistent path
+ * @param ois the ObjectInputStream to read persistent file
+ * @return true if verify successfully
+ */
+ @Override
+ public boolean verifyFileIntegrity(String persistentPath, ObjectInputStream
ois)
+ throws IOException {
+ try {
+ byte[] PBMagic = new byte[ProtobufUtil.PB_MAGIC.length];
+ ois.read(PBMagic);
+ if (Bytes.equals(ProtobufUtil.PB_MAGIC, PBMagic)) {
+ int length = ois.readInt();
+ byte[] persistentChecksum = new byte[length];
+ ois.read(persistentChecksum);
+ byte[] calculateChecksum = calculateChecksum();
+ if (!Bytes.equals(persistentChecksum, calculateChecksum)) {
+ LOG.warn("The persistent checksum is " +
Bytes.toString(persistentChecksum) +
+ ", but the calculate checksum is " +
Bytes.toString(calculateChecksum));
+ throw new IOException();
+ }
+ return true;
+ } else {
+ // if the persistent file is not start with PB_MAGIC, it's an old
version file
+ oldVersion = true;
Review comment:
Do we still need this oldVersion state here in FileIOE?
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
