jhungund commented on code in PR #6183:
URL: https://github.com/apache/hbase/pull/6183#discussion_r1736414805
##########
hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketProtoUtils.java:
##########
@@ -45,28 +46,45 @@ private BucketProtoUtils() {
}
- static BucketCacheProtos.BucketCacheEntry toPB(BucketCache cache) {
+ static BucketCacheProtos.BucketCacheEntry toPB(BucketCache cache,
BucketCacheProtos.BackingMap backingMap) {
return
BucketCacheProtos.BucketCacheEntry.newBuilder().setCacheCapacity(cache.getMaxSize())
.setIoClass(cache.ioEngine.getClass().getName())
.setMapClass(cache.backingMap.getClass().getName())
.putAllDeserializers(CacheableDeserializerIdManager.save())
.putAllCachedFiles(toCachedPB(cache.fullyCachedFiles))
- .setBackingMap(BucketProtoUtils.toPB(cache.backingMap))
+ .setBackingMap(backingMap)
.setChecksum(ByteString
.copyFrom(((PersistentIOEngine)
cache.ioEngine).calculateChecksum(cache.getAlgorithm())))
.build();
}
- private static BucketCacheProtos.BackingMap toPB(Map<BlockCacheKey,
BucketEntry> backingMap) {
+ static void toPB(BucketCache cache, FileOutputStream fos, long chunkSize)
throws IOException{
Review Comment:
ack
##########
hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java:
##########
@@ -1465,52 +1555,54 @@ private void parsePB(BucketCacheProtos.BucketCacheEntry
proto) throws IOExceptio
blocksByHFile = pair.getSecond();
fullyCachedFiles.clear();
fullyCachedFiles.putAll(BucketProtoUtils.fromPB(proto.getCachedFilesMap()));
- if (proto.hasChecksum()) {
- try {
- ((PersistentIOEngine)
ioEngine).verifyFileIntegrity(proto.getChecksum().toByteArray(),
- algorithm);
- backingMapValidated.set(true);
- } catch (IOException e) {
- LOG.warn("Checksum for cache file failed. "
- + "We need to validate each cache key in the backing map. "
- + "This may take some time, so we'll do it in a background thread,");
- Runnable cacheValidator = () -> {
- while (bucketAllocator == null) {
- try {
- Thread.sleep(50);
- } catch (InterruptedException ex) {
- throw new RuntimeException(ex);
- }
- }
- long startTime = EnvironmentEdgeManager.currentTime();
- int totalKeysOriginally = backingMap.size();
- for (Map.Entry<BlockCacheKey, BucketEntry> keyEntry :
backingMap.entrySet()) {
- try {
- ((FileIOEngine) ioEngine).checkCacheTime(keyEntry.getValue());
- } catch (IOException e1) {
- LOG.debug("Check for key {} failed. Evicting.",
keyEntry.getKey());
- evictBlock(keyEntry.getKey());
- fullyCachedFiles.remove(keyEntry.getKey().getHfileName());
- }
- }
- backingMapValidated.set(true);
- LOG.info("Finished validating {} keys in the backing map. Recovered:
{}. This took {}ms.",
- totalKeysOriginally, backingMap.size(),
- (EnvironmentEdgeManager.currentTime() - startTime));
- };
- Thread t = new Thread(cacheValidator);
- t.setDaemon(true);
- t.start();
- }
- } else {
- // if has not checksum, it means the persistence file is old format
- LOG.info("Persistent file is old format, it does not support verifying
file integrity!");
- backingMapValidated.set(true);
- }
+ verifyFileIntegrity(proto);
updateRegionSizeMapWhileRetrievingFromFile();
verifyCapacityAndClasses(proto.getCacheCapacity(), proto.getIoClass(),
proto.getMapClass());
}
+ private void persistChunkedBackingMap(FileOutputStream fos) throws
IOException {
+ fos.write(PB_MAGIC_V2);
+ long numChunks = backingMap.size() / persistenceChunkSize;
+ if (backingMap.size() % persistenceChunkSize != 0) {
+ numChunks += 1;
+ }
+
+ LOG.debug("persistToFile: before persisting backing map size: {}, "
+ + "fullycachedFiles size: {}, chunkSize: {}, numberofChunks: {}",
+ backingMap.size(), fullyCachedFiles.size(), persistenceChunkSize,
numChunks);
+
+ fos.write(Bytes.toBytes(persistenceChunkSize));
+ fos.write(Bytes.toBytes(numChunks));
Review Comment:
ack
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
