[
https://issues.apache.org/jira/browse/HDDS-1200?focusedWorklogId=297436&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-297436
]
ASF GitHub Bot logged work on HDDS-1200:
----------------------------------------
Author: ASF GitHub Bot
Created on: 19/Aug/19 21:44
Start Date: 19/Aug/19 21:44
Worklog Time Spent: 10m
Work Description: hgadre commented on pull request #1154: [HDDS-1200] Add
support for checksum verification in data scrubber
URL: https://github.com/apache/hadoop/pull/1154#discussion_r315425380
##########
File path:
hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueContainerCheck.java
##########
@@ -120,10 +133,70 @@ public TestKeyValueContainerCheck(String metadataImpl) {
container.close();
// next run checks on a Closed Container
- valid = kvCheck.fullCheck();
+ valid = kvCheck.fullCheck(new DataTransferThrottler(
+ c.getBandwidthPerVolume()), null);
assertTrue(valid);
}
+ /**
+ * Sanity test, when there are corruptions induced.
+ * @throws Exception
+ */
+ @Test
+ public void testKeyValueContainerCheckCorruption() throws Exception {
+ long containerID = 102;
+ int deletedBlocks = 1;
+ int normalBlocks = 3;
+ int chunksPerBlock = 4;
+ boolean valid = false;
+ ContainerScrubberConfiguration sc = conf.getObject(
+ ContainerScrubberConfiguration.class);
+
+ // test Closed Container
+ createContainerWithBlocks(containerID, normalBlocks, deletedBlocks, 65536,
+ chunksPerBlock);
+ File chunksPath = new File(containerData.getChunksPath());
+ assertTrue(chunksPath.listFiles().length
+ == (deletedBlocks + normalBlocks) * chunksPerBlock);
+
+ container.close();
+
+ KeyValueContainerCheck kvCheck =
+ new KeyValueContainerCheck(containerData.getMetadataPath(), conf,
+ containerID);
+
+ File metaDir = new File(containerData.getMetadataPath());
+ File dbFile = KeyValueContainerLocationUtil
+ .getContainerDBFile(metaDir, containerID);
+ containerData.setDbFile(dbFile);
+ try(ReferenceCountedDB db =
+ BlockUtils.getDB(containerData, conf);
+ KeyValueBlockIterator kvIter = new KeyValueBlockIterator(containerID,
+ new File(containerData.getContainerPath()))) {
+ BlockData block = kvIter.nextBlock();
+ assertTrue(!block.getChunks().isEmpty());
+ ContainerProtos.ChunkInfo c = block.getChunks().get(0);
+ File chunkFile = ChunkUtils.getChunkFile(containerData,
+ ChunkInfo.getFromProtoBuf(c));
+ long length = chunkFile.length();
+ assertTrue(length > 0);
+ // forcefully truncate the file to induce failure.
+ try (RandomAccessFile file = new RandomAccessFile(chunkFile, "rws")) {
+ file.setLength(length / 2);
+ }
+ assertEquals(length/2, chunkFile.length());
+ }
+
+ // metadata check should pass.
+ valid = kvCheck.fastCheck();
+ assertTrue(valid);
+
+ // checksum validation should fail.
+ valid = kvCheck.fullCheck(new DataTransferThrottler(
+ sc.getBandwidthPerVolume()), null);
+ assertFalse(valid);
+ }
+
/**
* Creates a container with normal and deleted blocks.
* First it will insert normal blocks, and then it will insert
Review comment:
Not sure I am following you. Can you elaborate which part do you find
misleading? This function was present before this patch ...
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
Issue Time Tracking
-------------------
Worklog Id: (was: 297436)
Time Spent: 4.5h (was: 4h 20m)
> Ozone Data Scrubbing : Checksum verification for chunks
> -------------------------------------------------------
>
> Key: HDDS-1200
> URL: https://issues.apache.org/jira/browse/HDDS-1200
> Project: Hadoop Distributed Data Store
> Issue Type: Sub-task
> Reporter: Supratim Deka
> Assignee: Hrishikesh Gadre
> Priority: Critical
> Labels: pull-request-available
> Time Spent: 4.5h
> Remaining Estimate: 0h
>
> Background scrubber should read each chunk and verify the checksum.
--
This message was sent by Atlassian Jira
(v8.3.2#803003)
---------------------------------------------------------------------
To unsubscribe, e-mail: hdfs-issues-unsubscr...@hadoop.apache.org
For additional commands, e-mail: hdfs-issues-h...@hadoop.apache.org
