ZhaoBQ commented on a change in pull request #528: HBASE-22890 Verify the files when RegionServer is starting and BucketCache is in file mode URL: https://github.com/apache/hbase/pull/528#discussion_r321696734
########## File path: hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/PersistentIOEngineUtils.java ########## @@ -0,0 +1,146 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.io.hfile.bucket; + +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.ObjectInputStream; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.protobuf.ProtobufUtil; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.util.Shell; + +/** + * Utilities for FileIOEngine's file operation + */ +@InterfaceAudience.Private +public final class PersistentIOEngineUtils { + private static final Log LOG = LogFactory.getLog(PersistentIOEngineUtils.class); + private static final DuFileCommand du = new DuFileCommand(new String[] {"du", ""}); + + private PersistentIOEngineUtils() { + } + + /** + * Read the persistence checksum from persistence path + * @param persistencePath the backingMap persistence path + * @return the persistence checksum + */ + public static byte[] readPersistenceChecksum(String persistencePath) { + if (persistencePath == null) { + return null; + } + try (ObjectInputStream ois = new ObjectInputStream(new FileInputStream(persistencePath))) { + byte[] PBMagic = new byte[ProtobufUtil.PB_MAGIC.length]; + ois.read(PBMagic); + if (Bytes.equals(ProtobufUtil.PB_MAGIC, PBMagic)) { + int length = ois.readInt(); + byte[] persistenceChecksum = new byte[length]; + ois.read(persistenceChecksum); + return persistenceChecksum; + } + } catch (IOException ioex) { + LOG.warn("Failed read persistent checksum, because of " + ioex); + return null; + } + return null; + } + + /** + * Delete bucketcache files + * @param filePaths the cache data files + */ + public static void deleteCacheDataFile(String[] filePaths) { + if (filePaths == null) { + return; + } + for (String file : filePaths) { + new File(file).delete(); + } + } + + /** + * Using an encryption algorithm to get a checksum, the default encryption algorithm is MD5 + * @param ioEngineName the IOEngine name + * @param algorithmName the encryption algorithm + * @return the checksum which is convert to HexString + * @throws IOException something happened like file not exists + * @throws NoSuchAlgorithmException no such algorithm + */ + public static byte[] calculateChecksum(String ioEngineName, String algorithmName) + throws IOException, NoSuchAlgorithmException { + String[] filePaths = + ioEngineName.substring(ioEngineName.indexOf(":") + 1).split(FileIOEngine.FILE_DELIMITER); + return calculateChecksum(filePaths, algorithmName); + } + + public static byte[] calculateChecksum(String[] filePaths, String algorithmName) + throws IOException, NoSuchAlgorithmException { + if (filePaths == null) { + return null; + } + StringBuilder sb = new StringBuilder(); + for (String filePath : filePaths){ + File file = new File(filePath); + if (file.exists()){ + sb.append(getFileSize(filePath)); Review comment: It's a good idea. If the configuration of cache file paths is changed, the persistent file and cache files are inconsistent. ---------------------------------------------------------------- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
