[jira] [Commented] (HDFS-12570) [SPS]: Refactor Co-ordinator datanode logic to track the block storage movements
[ https://issues.apache.org/jira/browse/HDFS-12570?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16577557#comment-16577557 ] Hudson commented on HDFS-12570: --- SUCCESS: Integrated in Jenkins build Hadoop-trunk-Commit #14752 (See [https://builds.apache.org/job/Hadoop-trunk-Commit/14752/]) HDFS-12570: [SPS]: Refactor Co-ordinator datanode logic to track the (umamahesh: rev 00eceed233d6e80d5c7137bf5b5286746ec4d5fb) * (edit) hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/StoragePolicySatisfyWorker.java * (edit) hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/DatanodeProtocol.java * (edit) hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestFsDatasetCache.java * (edit) hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestStorageReport.java * (edit) hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml * (edit) hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BlockStorageMovementAttemptedItems.java * (edit) hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockRecovery.java * (edit) hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java * (add) hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/BlocksStorageMoveAttemptFinished.java * (edit) hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/ArchivalStorage.md * (edit) hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStoragePolicySatisfierWithStripedFile.java * (edit) hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDatanodeProtocolRetryPolicy.java * (edit) hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java * (edit) hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStoragePolicySatisfier.java * (edit) hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java * (edit) hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestNameNodePrunesMissingStorages.java * (edit) hadoop-hdfs-project/hadoop-hdfs/src/main/proto/DatanodeProtocol.proto * (edit) hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestBlockStorageMovementAttemptedItems.java * (delete) hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BlockStorageMovementInfosBatch.java * (edit) hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java * (edit) hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/DatanodeProtocolClientSideTranslatorPB.java * (edit) hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java * (edit) hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/DatanodeProtocolServerSideTranslatorPB.java * (edit) hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java * (edit) hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/BlockStorageMovementCommand.java * (delete) hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/BlocksStorageMovementResult.java * (edit) hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/InternalDataNodeTestUtils.java * (edit) hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NameNodeAdapter.java * (edit) hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockStorageMovementTracker.java * (edit) hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java * (edit) hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDeadDatanode.java * (edit) hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBPOfferService.java * (edit) hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java * (edit) hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java * (edit) hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeLifeline.java * (edit) hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestStoragePolicySatisfyWorker.java * (edit) hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/StoragePolicySatisfier.java > [SPS]: Refactor Co-ordinator datanode logic to track the
[jira] [Commented] (HDFS-12570) [SPS]: Refactor Co-ordinator datanode logic to track the block storage movements
[ https://issues.apache.org/jira/browse/HDFS-12570?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16202847#comment-16202847 ] Uma Maheswara Rao G commented on HDFS-12570: +1 on the latest patch. Thanks Rakesh for updating the patch. Thanks [~surendrasingh] for the reviews. > [SPS]: Refactor Co-ordinator datanode logic to track the block storage > movements > > > Key: HDFS-12570 > URL: https://issues.apache.org/jira/browse/HDFS-12570 > Project: Hadoop HDFS > Issue Type: Sub-task > Components: datanode, namenode >Reporter: Rakesh R >Assignee: Rakesh R > Attachments: HDFS-12570-HDFS-10285-00.patch, > HDFS-12570-HDFS-10285-01.patch, HDFS-12570-HDFS-10285-02.patch, > HDFS-12570-HDFS-10285-03.patch, HDFS-12570-HDFS-10285-04.patch > > > This task is to refactor the C-DN block storage movements. Basically, the > idea is to move the scheduling and tracking logic to Namenode rather than at > the special C-DN. Please refer the discussion with [~andrew.wang] to > understand the [background and the necessity of > refactoring|https://issues.apache.org/jira/browse/HDFS-10285?focusedCommentId=16141060=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel#comment-16141060]. -- This message was sent by Atlassian JIRA (v6.4.14#64029) - To unsubscribe, e-mail: hdfs-issues-unsubscr...@hadoop.apache.org For additional commands, e-mail: hdfs-issues-h...@hadoop.apache.org
[jira] [Commented] (HDFS-12570) [SPS]: Refactor Co-ordinator datanode logic to track the block storage movements
[
https://issues.apache.org/jira/browse/HDFS-12570?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16202006#comment-16202006
]
Hadoop QA commented on HDFS-12570:
--
| (x) *{color:red}-1 overall{color}* |
\\
\\
|| Vote || Subsystem || Runtime || Comment ||
| {color:blue}0{color} | {color:blue} reexec {color} | {color:blue} 0m
26s{color} | {color:blue} Docker mode activated. {color} |
|| || || || {color:brown} Prechecks {color} ||
| {color:green}+1{color} | {color:green} @author {color} | {color:green} 0m
0s{color} | {color:green} The patch does not contain any @author tags. {color} |
| {color:green}+1{color} | {color:green} test4tests {color} | {color:green} 0m
0s{color} | {color:green} The patch appears to include 15 new or modified test
files. {color} |
|| || || || {color:brown} HDFS-10285 Compile Tests {color} ||
| {color:green}+1{color} | {color:green} mvninstall {color} | {color:green} 15m
19s{color} | {color:green} HDFS-10285 passed {color} |
| {color:green}+1{color} | {color:green} compile {color} | {color:green} 1m
0s{color} | {color:green} HDFS-10285 passed {color} |
| {color:green}+1{color} | {color:green} checkstyle {color} | {color:green} 0m
59s{color} | {color:green} HDFS-10285 passed {color} |
| {color:green}+1{color} | {color:green} mvnsite {color} | {color:green} 1m
7s{color} | {color:green} HDFS-10285 passed {color} |
| {color:red}-1{color} | {color:red} shadedclient {color} | {color:red} 12m
4s{color} | {color:red} branch has errors when building and testing our client
artifacts. {color} |
| {color:green}+1{color} | {color:green} findbugs {color} | {color:green} 1m
46s{color} | {color:green} HDFS-10285 passed {color} |
| {color:green}+1{color} | {color:green} javadoc {color} | {color:green} 0m
43s{color} | {color:green} HDFS-10285 passed {color} |
|| || || || {color:brown} Patch Compile Tests {color} ||
| {color:green}+1{color} | {color:green} mvninstall {color} | {color:green} 0m
53s{color} | {color:green} the patch passed {color} |
| {color:green}+1{color} | {color:green} compile {color} | {color:green} 0m
47s{color} | {color:green} the patch passed {color} |
| {color:green}+1{color} | {color:green} cc {color} | {color:green} 0m
47s{color} | {color:green} the patch passed {color} |
| {color:green}+1{color} | {color:green} javac {color} | {color:green} 0m
47s{color} | {color:green} the patch passed {color} |
| {color:orange}-0{color} | {color:orange} checkstyle {color} | {color:orange}
0m 51s{color} | {color:orange} hadoop-hdfs-project/hadoop-hdfs: The patch
generated 10 new + 1142 unchanged - 2 fixed = 1152 total (was 1144) {color} |
| {color:green}+1{color} | {color:green} mvnsite {color} | {color:green} 0m
55s{color} | {color:green} the patch passed {color} |
| {color:red}-1{color} | {color:red} whitespace {color} | {color:red} 0m
0s{color} | {color:red} The patch has 1 line(s) that end in whitespace. Use git
apply --whitespace=fix <>. Refer https://git-scm.com/docs/git-apply
{color} |
| {color:green}+1{color} | {color:green} xml {color} | {color:green} 0m
1s{color} | {color:green} The patch has no ill-formed XML file. {color} |
| {color:red}-1{color} | {color:red} shadedclient {color} | {color:red} 10m
47s{color} | {color:red} patch has errors when building and testing our client
artifacts. {color} |
| {color:green}+1{color} | {color:green} findbugs {color} | {color:green} 1m
49s{color} | {color:green} the patch passed {color} |
| {color:green}+1{color} | {color:green} javadoc {color} | {color:green} 0m
40s{color} | {color:green} the patch passed {color} |
|| || || || {color:brown} Other Tests {color} ||
| {color:red}-1{color} | {color:red} unit {color} | {color:red}119m 19s{color}
| {color:red} hadoop-hdfs in the patch failed. {color} |
| {color:green}+1{color} | {color:green} asflicense {color} | {color:green} 0m
22s{color} | {color:green} The patch does not generate ASF License warnings.
{color} |
| {color:black}{color} | {color:black} {color} | {color:black}169m 3s{color} |
{color:black} {color} |
\\
\\
|| Reason || Tests ||
| Failed junit tests |
hadoop.hdfs.server.namenode.TestPersistentStoragePolicySatisfier |
| | hadoop.hdfs.server.namenode.TestNamenodeCapacityReport |
| | hadoop.hdfs.TestClientProtocolForPipelineRecovery |
| | hadoop.hdfs.TestLeaseRecoveryStriped |
| | hadoop.hdfs.TestFileAppend |
| | hadoop.hdfs.TestReadStripedFileWithMissingBlocks |
| | hadoop.hdfs.server.blockmanagement.TestBlockStatsMXBean |
| | hadoop.hdfs.server.namenode.TestStoragePolicySatisfierWithStripedFile |
| | hadoop.hdfs.TestDFSStripedOutputStreamWithFailure150 |
| Timed out junit tests | org.apache.hadoop.hdfs.TestWriteReadStripedFile |
| | org.apache.hadoop.hdfs.TestReadStripedFileWithDecoding |
\\
\\
|| Subsystem || Report/Notes ||
| Docker | Image:yetus/hadoop:14b5c93 |
| JIRA Issue | HDFS-12570 |
| JIRA Patch URL |
[jira] [Commented] (HDFS-12570) [SPS]: Refactor Co-ordinator datanode logic to track the block storage movements
[
https://issues.apache.org/jira/browse/HDFS-12570?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16201817#comment-16201817
]
Rakesh R commented on HDFS-12570:
-
Attached another patch fixing {{TestStoragePolicySatisfierWithStripedFile}} and
{{TestHdfsConfigFields}} test failures. I could see
{{TestPersistentStoragePolicySatisfier}} failures will be fixed as part of
HDFS-12556.
> [SPS]: Refactor Co-ordinator datanode logic to track the block storage
> movements
>
>
> Key: HDFS-12570
> URL: https://issues.apache.org/jira/browse/HDFS-12570
> Project: Hadoop HDFS
> Issue Type: Sub-task
> Components: datanode, namenode
>Reporter: Rakesh R
>Assignee: Rakesh R
> Attachments: HDFS-12570-HDFS-10285-00.patch,
> HDFS-12570-HDFS-10285-01.patch, HDFS-12570-HDFS-10285-02.patch,
> HDFS-12570-HDFS-10285-03.patch, HDFS-12570-HDFS-10285-04.patch
>
>
> This task is to refactor the C-DN block storage movements. Basically, the
> idea is to move the scheduling and tracking logic to Namenode rather than at
> the special C-DN. Please refer the discussion with [~andrew.wang] to
> understand the [background and the necessity of
> refactoring|https://issues.apache.org/jira/browse/HDFS-10285?focusedCommentId=16141060=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel#comment-16141060].
--
This message was sent by Atlassian JIRA
(v6.4.14#64029)
-
To unsubscribe, e-mail: hdfs-issues-unsubscr...@hadoop.apache.org
For additional commands, e-mail: hdfs-issues-h...@hadoop.apache.org
[jira] [Commented] (HDFS-12570) [SPS]: Refactor Co-ordinator datanode logic to track the block storage movements
[
https://issues.apache.org/jira/browse/HDFS-12570?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16201676#comment-16201676
]
Hadoop QA commented on HDFS-12570:
--
| (x) *{color:red}-1 overall{color}* |
\\
\\
|| Vote || Subsystem || Runtime || Comment ||
| {color:blue}0{color} | {color:blue} reexec {color} | {color:blue} 21m
46s{color} | {color:blue} Docker mode activated. {color} |
|| || || || {color:brown} Prechecks {color} ||
| {color:green}+1{color} | {color:green} @author {color} | {color:green} 0m
0s{color} | {color:green} The patch does not contain any @author tags. {color} |
| {color:green}+1{color} | {color:green} test4tests {color} | {color:green} 0m
0s{color} | {color:green} The patch appears to include 15 new or modified test
files. {color} |
|| || || || {color:brown} HDFS-10285 Compile Tests {color} ||
| {color:green}+1{color} | {color:green} mvninstall {color} | {color:green} 18m
52s{color} | {color:green} HDFS-10285 passed {color} |
| {color:green}+1{color} | {color:green} compile {color} | {color:green} 1m
5s{color} | {color:green} HDFS-10285 passed {color} |
| {color:green}+1{color} | {color:green} checkstyle {color} | {color:green} 1m
1s{color} | {color:green} HDFS-10285 passed {color} |
| {color:green}+1{color} | {color:green} mvnsite {color} | {color:green} 1m
3s{color} | {color:green} HDFS-10285 passed {color} |
| {color:red}-1{color} | {color:red} shadedclient {color} | {color:red} 12m
11s{color} | {color:red} branch has errors when building and testing our client
artifacts. {color} |
| {color:green}+1{color} | {color:green} findbugs {color} | {color:green} 1m
53s{color} | {color:green} HDFS-10285 passed {color} |
| {color:green}+1{color} | {color:green} javadoc {color} | {color:green} 0m
45s{color} | {color:green} HDFS-10285 passed {color} |
|| || || || {color:brown} Patch Compile Tests {color} ||
| {color:green}+1{color} | {color:green} mvninstall {color} | {color:green} 1m
0s{color} | {color:green} the patch passed {color} |
| {color:green}+1{color} | {color:green} compile {color} | {color:green} 0m
50s{color} | {color:green} the patch passed {color} |
| {color:green}+1{color} | {color:green} cc {color} | {color:green} 0m
50s{color} | {color:green} the patch passed {color} |
| {color:green}+1{color} | {color:green} javac {color} | {color:green} 0m
50s{color} | {color:green} the patch passed {color} |
| {color:orange}-0{color} | {color:orange} checkstyle {color} | {color:orange}
0m 53s{color} | {color:orange} hadoop-hdfs-project/hadoop-hdfs: The patch
generated 11 new + 1142 unchanged - 2 fixed = 1153 total (was 1144) {color} |
| {color:green}+1{color} | {color:green} mvnsite {color} | {color:green} 0m
58s{color} | {color:green} the patch passed {color} |
| {color:green}+1{color} | {color:green} whitespace {color} | {color:green} 0m
0s{color} | {color:green} The patch has no whitespace issues. {color} |
| {color:red}-1{color} | {color:red} shadedclient {color} | {color:red} 11m
21s{color} | {color:red} patch has errors when building and testing our client
artifacts. {color} |
| {color:green}+1{color} | {color:green} findbugs {color} | {color:green} 2m
7s{color} | {color:green} the patch passed {color} |
| {color:green}+1{color} | {color:green} javadoc {color} | {color:green} 0m
44s{color} | {color:green} the patch passed {color} |
|| || || || {color:brown} Other Tests {color} ||
| {color:red}-1{color} | {color:red} unit {color} | {color:red}117m 28s{color}
| {color:red} hadoop-hdfs in the patch failed. {color} |
| {color:green}+1{color} | {color:green} asflicense {color} | {color:green} 0m
31s{color} | {color:green} The patch does not generate ASF License warnings.
{color} |
| {color:black}{color} | {color:black} {color} | {color:black}193m 54s{color} |
{color:black} {color} |
\\
\\
|| Reason || Tests ||
| Failed junit tests |
hadoop.hdfs.server.namenode.TestPersistentStoragePolicySatisfier |
| | hadoop.hdfs.TestParallelShortCircuitReadNoChecksum |
| | hadoop.tools.TestHdfsConfigFields |
| | hadoop.hdfs.TestDFSStripedOutputStreamWithFailure070 |
| | hadoop.hdfs.TestDFSStripedOutputStreamWithFailure010 |
| | hadoop.hdfs.TestDFSStripedOutputStreamWithFailure210 |
| | hadoop.hdfs.TestLeaseRecoveryStriped |
| | hadoop.hdfs.TestDFSStripedOutputStreamWithFailure160 |
| | hadoop.hdfs.TestDFSStripedOutputStreamWithFailure080 |
| | hadoop.hdfs.TestMultiThreadedHflush |
| | hadoop.hdfs.TestFileAppendRestart |
| | hadoop.hdfs.server.namenode.TestDecommissioningStatus |
| | hadoop.hdfs.server.namenode.TestStoragePolicySatisfierWithStripedFile |
| | hadoop.hdfs.TestDFSInotifyEventInputStream |
| | hadoop.hdfs.server.namenode.TestNameNodeMetadataConsistency |
| | hadoop.hdfs.server.datanode.TestDataNodeVolumeFailure |
| | hadoop.hdfs.server.namenode.TestReencryption |
| Timed out junit tests | org.apache.hadoop.hdfs.TestWriteReadStripedFile |
|
[jira] [Commented] (HDFS-12570) [SPS]: Refactor Co-ordinator datanode logic to track the block storage movements
[
https://issues.apache.org/jira/browse/HDFS-12570?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16201504#comment-16201504
]
Rakesh R commented on HDFS-12570:
-
Thanks a lot [~umamaheswararao] for the detailed review comments and
suggestions.
Addressed below comments and taken care in the latest patch:
Comment-1) modified the config key name.
Comment-2) Fixed typo.
Comment-3) Since not differentiating NO, FEW, ALL paired I've clubbed these
together. In the new patch, I added
{{BlocksMovingAnalysis.Status.NO_BLOCKS_TARGETS_PAIRED}} for much better
readability and logic more clear covering all the corner cases.
Comment-4) Cleaned up the unwanted comment.
Comment-5) Moved the {{blockCount++;}} inside for-loop. Hope this is fine.
Comment-6) Updated BlockStorageMovementCommand class level doc
Comment-7) Updated the doc.
> [SPS]: Refactor Co-ordinator datanode logic to track the block storage
> movements
>
>
> Key: HDFS-12570
> URL: https://issues.apache.org/jira/browse/HDFS-12570
> Project: Hadoop HDFS
> Issue Type: Sub-task
> Components: datanode, namenode
>Reporter: Rakesh R
>Assignee: Rakesh R
> Attachments: HDFS-12570-HDFS-10285-00.patch,
> HDFS-12570-HDFS-10285-01.patch, HDFS-12570-HDFS-10285-02.patch,
> HDFS-12570-HDFS-10285-03.patch
>
>
> This task is to refactor the C-DN block storage movements. Basically, the
> idea is to move the scheduling and tracking logic to Namenode rather than at
> the special C-DN. Please refer the discussion with [~andrew.wang] to
> understand the [background and the necessity of
> refactoring|https://issues.apache.org/jira/browse/HDFS-10285?focusedCommentId=16141060=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel#comment-16141060].
--
This message was sent by Atlassian JIRA
(v6.4.14#64029)
-
To unsubscribe, e-mail: hdfs-issues-unsubscr...@hadoop.apache.org
For additional commands, e-mail: hdfs-issues-h...@hadoop.apache.org
[jira] [Commented] (HDFS-12570) [SPS]: Refactor Co-ordinator datanode logic to track the block storage movements
[
https://issues.apache.org/jira/browse/HDFS-12570?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16198546#comment-16198546
]
Uma Maheswara Rao G commented on HDFS-12570:
[~rakeshr] Thank you for updating the patch. Patch mostly looks good to me,
except the below comments
* . {code}
public static final String
DFS_STORAGE_POLICY_SATISFIER_SHARE_EQUAL_REPLICA_MAX_STREAMS_KEY =
+ "dfs.storage.policy.satisfier.share.equal.replication.max-streams";
+ public static final boolean
DFS_STORAGE_POLICY_SATISFIER_SHARE_EQUAL_REPLICA_MAX_STREAMS_DEFAULT =
+ false;
{code}
I am thinking this config can be more meaningful. How about something like,
"dfs.storage.policy.satisfier.low.max-streams.preference”. This can be true by
default. Any other better names are most welcomed.
* finshedBlksIter — finihedBlksIter
*We are removing the state: status =
BlocksMovingAnalysisStatus.FEW_BLOCKS_TARGETS_PAIRED;
how are we covering the case when we can’t find any targets at all even though
there are movement needed blocks?
What is the status for this?
* . {code}
// 'BlockCollectionId' is used as the tracking ID. All the blocks under
+// this
+// blockCollectionID will be added to this datanode.
+// TODO: assign to each target node
{code}
Please remove this comment or update.
* . {code}
+((DatanodeDescriptor) blkMovingInfo.getTarget())
+.addBlocksToMoveStorage(blkMovingInfo);
{code}
Could we also increment scheduled block count for this node?
* BlockStorageMovementCommand: We removed tracked, but class java doc still
representing old behavior.
> [SPS]: Refactor Co-ordinator datanode logic to track the block storage
> movements
>
>
> Key: HDFS-12570
> URL: https://issues.apache.org/jira/browse/HDFS-12570
> Project: Hadoop HDFS
> Issue Type: Sub-task
> Components: datanode, namenode
>Reporter: Rakesh R
>Assignee: Rakesh R
> Attachments: HDFS-12570-HDFS-10285-00.patch,
> HDFS-12570-HDFS-10285-01.patch, HDFS-12570-HDFS-10285-02.patch
>
>
> This task is to refactor the C-DN block storage movements. Basically, the
> idea is to move the scheduling and tracking logic to Namenode rather than at
> the special C-DN. Please refer the discussion with [~andrew.wang] to
> understand the [background and the necessity of
> refactoring|https://issues.apache.org/jira/browse/HDFS-10285?focusedCommentId=16141060=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel#comment-16141060].
--
This message was sent by Atlassian JIRA
(v6.4.14#64029)
-
To unsubscribe, e-mail: hdfs-issues-unsubscr...@hadoop.apache.org
For additional commands, e-mail: hdfs-issues-h...@hadoop.apache.org
[jira] [Commented] (HDFS-12570) [SPS]: Refactor Co-ordinator datanode logic to track the block storage movements
[
https://issues.apache.org/jira/browse/HDFS-12570?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16195883#comment-16195883
]
Hadoop QA commented on HDFS-12570:
--
| (x) *{color:red}-1 overall{color}* |
\\
\\
|| Vote || Subsystem || Runtime || Comment ||
| {color:blue}0{color} | {color:blue} reexec {color} | {color:blue} 0m
24s{color} | {color:blue} Docker mode activated. {color} |
|| || || || {color:brown} Prechecks {color} ||
| {color:green}+1{color} | {color:green} @author {color} | {color:green} 0m
0s{color} | {color:green} The patch does not contain any @author tags. {color} |
| {color:green}+1{color} | {color:green} test4tests {color} | {color:green} 0m
0s{color} | {color:green} The patch appears to include 15 new or modified test
files. {color} |
|| || || || {color:brown} HDFS-10285 Compile Tests {color} ||
| {color:green}+1{color} | {color:green} mvninstall {color} | {color:green} 16m
42s{color} | {color:green} HDFS-10285 passed {color} |
| {color:green}+1{color} | {color:green} compile {color} | {color:green} 0m
56s{color} | {color:green} HDFS-10285 passed {color} |
| {color:green}+1{color} | {color:green} checkstyle {color} | {color:green} 0m
55s{color} | {color:green} HDFS-10285 passed {color} |
| {color:green}+1{color} | {color:green} mvnsite {color} | {color:green} 1m
27s{color} | {color:green} HDFS-10285 passed {color} |
| {color:red}-1{color} | {color:red} shadedclient {color} | {color:red} 12m
27s{color} | {color:red} branch has errors when building and testing our client
artifacts. {color} |
| {color:green}+1{color} | {color:green} findbugs {color} | {color:green} 2m
2s{color} | {color:green} HDFS-10285 passed {color} |
| {color:green}+1{color} | {color:green} javadoc {color} | {color:green} 0m
47s{color} | {color:green} HDFS-10285 passed {color} |
|| || || || {color:brown} Patch Compile Tests {color} ||
| {color:green}+1{color} | {color:green} mvninstall {color} | {color:green} 1m
0s{color} | {color:green} the patch passed {color} |
| {color:green}+1{color} | {color:green} compile {color} | {color:green} 0m
57s{color} | {color:green} the patch passed {color} |
| {color:green}+1{color} | {color:green} cc {color} | {color:green} 0m
57s{color} | {color:green} the patch passed {color} |
| {color:green}+1{color} | {color:green} javac {color} | {color:green} 0m
57s{color} | {color:green} the patch passed {color} |
| {color:orange}-0{color} | {color:orange} checkstyle {color} | {color:orange}
0m 51s{color} | {color:orange} hadoop-hdfs-project/hadoop-hdfs: The patch
generated 11 new + 1142 unchanged - 2 fixed = 1153 total (was 1144) {color} |
| {color:green}+1{color} | {color:green} mvnsite {color} | {color:green} 0m
59s{color} | {color:green} the patch passed {color} |
| {color:green}+1{color} | {color:green} whitespace {color} | {color:green} 0m
0s{color} | {color:green} The patch has no whitespace issues. {color} |
| {color:red}-1{color} | {color:red} shadedclient {color} | {color:red} 1m
56s{color} | {color:red} patch has errors when building and testing our client
artifacts. {color} |
| {color:green}+1{color} | {color:green} findbugs {color} | {color:green} 2m
7s{color} | {color:green} the patch passed {color} |
| {color:green}+1{color} | {color:green} javadoc {color} | {color:green} 0m
42s{color} | {color:green} the patch passed {color} |
|| || || || {color:brown} Other Tests {color} ||
| {color:red}-1{color} | {color:red} unit {color} | {color:red}107m 55s{color}
| {color:red} hadoop-hdfs in the patch failed. {color} |
| {color:green}+1{color} | {color:green} asflicense {color} | {color:green} 0m
23s{color} | {color:green} The patch does not generate ASF License warnings.
{color} |
| {color:black}{color} | {color:black} {color} | {color:black}151m 41s{color} |
{color:black} {color} |
\\
\\
|| Reason || Tests ||
| Failed junit tests | hadoop.hdfs.TestReadStripedFileWithDecoding |
| | hadoop.hdfs.server.namenode.TestPersistentStoragePolicySatisfier |
| | hadoop.hdfs.TestDFSStripedOutputStreamWithFailure140 |
| | hadoop.hdfs.TestDFSStripedOutputStreamWithFailure000 |
| | hadoop.hdfs.TestDFSStripedOutputStreamWithFailure090 |
| | hadoop.hdfs.TestDFSStripedInputStreamWithRandomECPolicy |
| | hadoop.hdfs.TestFileAppendRestart |
| | hadoop.hdfs.TestReadStripedFileWithMissingBlocks |
| | hadoop.hdfs.TestDFSStripedOutputStreamWithFailure010 |
| | hadoop.hdfs.TestReconstructStripedFile |
| | hadoop.hdfs.TestDFSStripedOutputStreamWithFailureWithRandomECPolicy |
| | hadoop.tools.TestHdfsConfigFields |
| | hadoop.hdfs.server.namenode.TestStoragePolicySatisfierWithStripedFile |
| | hadoop.hdfs.TestDFSStripedOutputStreamWithFailure040 |
| | hadoop.hdfs.TestDFSStripedOutputStreamWithFailure080 |
| | hadoop.hdfs.TestClientProtocolForPipelineRecovery |
| | hadoop.hdfs.TestDFSStripedOutputStreamWithFailure |
| | hadoop.hdfs.TestLeaseRecoveryStriped |
| |
[jira] [Commented] (HDFS-12570) [SPS]: Refactor Co-ordinator datanode logic to track the block storage movements
[
https://issues.apache.org/jira/browse/HDFS-12570?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16194213#comment-16194213
]
Hadoop QA commented on HDFS-12570:
--
| (x) *{color:red}-1 overall{color}* |
\\
\\
|| Vote || Subsystem || Runtime || Comment ||
| {color:blue}0{color} | {color:blue} reexec {color} | {color:blue} 14m
52s{color} | {color:blue} Docker mode activated. {color} |
|| || || || {color:brown} Prechecks {color} ||
| {color:green}+1{color} | {color:green} @author {color} | {color:green} 0m
0s{color} | {color:green} The patch does not contain any @author tags. {color} |
| {color:green}+1{color} | {color:green} test4tests {color} | {color:green} 0m
0s{color} | {color:green} The patch appears to include 15 new or modified test
files. {color} |
|| || || || {color:brown} HDFS-10285 Compile Tests {color} ||
| {color:green}+1{color} | {color:green} mvninstall {color} | {color:green} 14m
19s{color} | {color:green} HDFS-10285 passed {color} |
| {color:green}+1{color} | {color:green} compile {color} | {color:green} 0m
49s{color} | {color:green} HDFS-10285 passed {color} |
| {color:green}+1{color} | {color:green} checkstyle {color} | {color:green} 0m
47s{color} | {color:green} HDFS-10285 passed {color} |
| {color:green}+1{color} | {color:green} mvnsite {color} | {color:green} 0m
57s{color} | {color:green} HDFS-10285 passed {color} |
| {color:red}-1{color} | {color:red} shadedclient {color} | {color:red} 11m
9s{color} | {color:red} branch has errors when building and testing our client
artifacts. {color} |
| {color:green}+1{color} | {color:green} findbugs {color} | {color:green} 1m
44s{color} | {color:green} HDFS-10285 passed {color} |
| {color:green}+1{color} | {color:green} javadoc {color} | {color:green} 0m
42s{color} | {color:green} HDFS-10285 passed {color} |
|| || || || {color:brown} Patch Compile Tests {color} ||
| {color:green}+1{color} | {color:green} mvninstall {color} | {color:green} 0m
50s{color} | {color:green} the patch passed {color} |
| {color:green}+1{color} | {color:green} compile {color} | {color:green} 0m
47s{color} | {color:green} the patch passed {color} |
| {color:green}+1{color} | {color:green} cc {color} | {color:green} 0m
47s{color} | {color:green} the patch passed {color} |
| {color:green}+1{color} | {color:green} javac {color} | {color:green} 0m
47s{color} | {color:green} the patch passed {color} |
| {color:orange}-0{color} | {color:orange} checkstyle {color} | {color:orange}
0m 44s{color} | {color:orange} hadoop-hdfs-project/hadoop-hdfs: The patch
generated 8 new + 725 unchanged - 2 fixed = 733 total (was 727) {color} |
| {color:green}+1{color} | {color:green} mvnsite {color} | {color:green} 0m
53s{color} | {color:green} the patch passed {color} |
| {color:green}+1{color} | {color:green} whitespace {color} | {color:green} 0m
0s{color} | {color:green} The patch has no whitespace issues. {color} |
| {color:red}-1{color} | {color:red} shadedclient {color} | {color:red} 10m
26s{color} | {color:red} patch has errors when building and testing our client
artifacts. {color} |
| {color:green}+1{color} | {color:green} findbugs {color} | {color:green} 1m
49s{color} | {color:green} the patch passed {color} |
| {color:red}-1{color} | {color:red} javadoc {color} | {color:red} 0m
39s{color} | {color:red} hadoop-hdfs-project_hadoop-hdfs generated 1 new + 9
unchanged - 0 fixed = 10 total (was 9) {color} |
|| || || || {color:brown} Other Tests {color} ||
| {color:red}-1{color} | {color:red} unit {color} | {color:red} 93m 19s{color}
| {color:red} hadoop-hdfs in the patch failed. {color} |
| {color:green}+1{color} | {color:green} asflicense {color} | {color:green} 0m
21s{color} | {color:green} The patch does not generate ASF License warnings.
{color} |
| {color:black}{color} | {color:black} {color} | {color:black}154m 42s{color} |
{color:black} {color} |
\\
\\
|| Reason || Tests ||
| Failed junit tests | hadoop.hdfs.TestDFSStripedInputStreamWithRandomECPolicy |
| | hadoop.hdfs.TestFileAppendRestart |
| | hadoop.hdfs.server.namenode.TestPersistentStoragePolicySatisfier |
| | hadoop.hdfs.TestReadStripedFileWithDecoding |
| | hadoop.hdfs.TestDFSStripedOutputStreamWithFailure |
| | hadoop.hdfs.server.namenode.ha.TestPipelinesFailover |
| | hadoop.hdfs.TestLeaseRecoveryStriped |
| | hadoop.hdfs.TestAclsEndToEnd |
| | hadoop.hdfs.TestClientProtocolForPipelineRecovery |
| | hadoop.hdfs.TestDFSStripedOutputStreamWithFailure170 |
| | hadoop.hdfs.TestDFSStripedOutputStreamWithFailure070 |
| Timed out junit tests | org.apache.hadoop.hdfs.TestWriteReadStripedFile |
\\
\\
|| Subsystem || Report/Notes ||
| Docker | Image:yetus/hadoop:14b5c93 |
| JIRA Issue | HDFS-12570 |
| JIRA Patch URL |
https://issues.apache.org/jira/secure/attachment/12890644/HDFS-12570-HDFS-10285-01.patch
|
| Optional Tests | asflicense compile javac javadoc mvninstall mvnsite
unit
[jira] [Commented] (HDFS-12570) [SPS]: Refactor Co-ordinator datanode logic to track the block storage movements
[
https://issues.apache.org/jira/browse/HDFS-12570?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16194124#comment-16194124
]
Rakesh R commented on HDFS-12570:
-
Thanks [~surendrasingh] for the reviews.
Attached latest patch fixing checkstyle, findbug, test case failure and
addressed (1) and (2) comments of Surendra.
comment-1) => Actually {{attemptedItemInfo.allBlockLocsAttemptedToSatisfy }}
attribute is not required as SPS will re-check the trackId/fileId once it
receives movementreport from DN. So, I've removed this flag.
comment-2) => Done
comment-3) => Interesting thought. In my current patch, kept blk movement
status reporting simple via heartbeat. As we know, IBR is reporting block
received and deleted items only. We need to change the semantics of
#blockReceivedAndDeleted call to incorporate the block move finished status.
Let me think for a while and get back to this point.
> [SPS]: Refactor Co-ordinator datanode logic to track the block storage
> movements
>
>
> Key: HDFS-12570
> URL: https://issues.apache.org/jira/browse/HDFS-12570
> Project: Hadoop HDFS
> Issue Type: Sub-task
> Components: datanode, namenode
>Reporter: Rakesh R
>Assignee: Rakesh R
> Attachments: HDFS-12570-HDFS-10285-00.patch,
> HDFS-12570-HDFS-10285-01.patch
>
>
> This task is to refactor the C-DN block storage movements. Basically, the
> idea is to move the scheduling and tracking logic to Namenode rather than at
> the special C-DN. Please refer the discussion with [~andrew.wang] to
> understand the [background and the necessity of
> refactoring|https://issues.apache.org/jira/browse/HDFS-10285?focusedCommentId=16141060=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel#comment-16141060].
--
This message was sent by Atlassian JIRA
(v6.4.14#64029)
-
To unsubscribe, e-mail: hdfs-issues-unsubscr...@hadoop.apache.org
For additional commands, e-mail: hdfs-issues-h...@hadoop.apache.org
[jira] [Commented] (HDFS-12570) [SPS]: Refactor Co-ordinator datanode logic to track the block storage movements
[
https://issues.apache.org/jira/browse/HDFS-12570?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16192915#comment-16192915
]
Surendra Singh Lilhore commented on HDFS-12570:
---
Thanks [~rakeshr] for patch...
I looked at the patch. Following are my comment in my first look.
# In
{{BlockStorageMovementAttemptedItems#blockStorageMovementReportedItemsCheck()}},
if {{attemptedItemInfo.allBlockLocsAttemptedToSatisfy}} is false then we
should add {{ItemInfo}} back to the {{blockStorageMovementNeeded}} queue.
# FSNamesystem#handleHeartbeat() : Correct the log message, now are not using
"co-ordinator datanode".
{code}
if (LOG.isDebugEnabled()) {
LOG.debug(
"Storage policy satisfier is not running. So, ignoring block "
+ "storage movement results sent by co-ordinator datanode");
}
{code}
# Can we try to use IBR instead of adding new parameter
{{BlocksStorageMoveAttemptFinished}} in {{NameNodeRpcServer#sendHeartbeat()}} ?.
> [SPS]: Refactor Co-ordinator datanode logic to track the block storage
> movements
>
>
> Key: HDFS-12570
> URL: https://issues.apache.org/jira/browse/HDFS-12570
> Project: Hadoop HDFS
> Issue Type: Sub-task
> Components: datanode, namenode
>Reporter: Rakesh R
>Assignee: Rakesh R
> Attachments: HDFS-12570-HDFS-10285-00.patch
>
>
> This task is to refactor the C-DN block storage movements. Basically, the
> idea is to move the scheduling and tracking logic to Namenode rather than at
> the special C-DN. Please refer the discussion with [~andrew.wang] to
> understand the [background and the necessity of
> refactoring|https://issues.apache.org/jira/browse/HDFS-10285?focusedCommentId=16141060=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel#comment-16141060].
--
This message was sent by Atlassian JIRA
(v6.4.14#64029)
-
To unsubscribe, e-mail: hdfs-issues-unsubscr...@hadoop.apache.org
For additional commands, e-mail: hdfs-issues-h...@hadoop.apache.org
[jira] [Commented] (HDFS-12570) [SPS]: Refactor Co-ordinator datanode logic to track the block storage movements
[
https://issues.apache.org/jira/browse/HDFS-12570?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16189714#comment-16189714
]
Hadoop QA commented on HDFS-12570:
--
| (x) *{color:red}-1 overall{color}* |
\\
\\
|| Vote || Subsystem || Runtime || Comment ||
| {color:blue}0{color} | {color:blue} reexec {color} | {color:blue} 0m
19s{color} | {color:blue} Docker mode activated. {color} |
|| || || || {color:brown} Prechecks {color} ||
| {color:green}+1{color} | {color:green} @author {color} | {color:green} 0m
0s{color} | {color:green} The patch does not contain any @author tags. {color} |
| {color:green}+1{color} | {color:green} test4tests {color} | {color:green} 0m
0s{color} | {color:green} The patch appears to include 14 new or modified test
files. {color} |
|| || || || {color:brown} HDFS-10285 Compile Tests {color} ||
| {color:green}+1{color} | {color:green} mvninstall {color} | {color:green} 14m
11s{color} | {color:green} HDFS-10285 passed {color} |
| {color:green}+1{color} | {color:green} compile {color} | {color:green} 0m
49s{color} | {color:green} HDFS-10285 passed {color} |
| {color:green}+1{color} | {color:green} checkstyle {color} | {color:green} 0m
46s{color} | {color:green} HDFS-10285 passed {color} |
| {color:green}+1{color} | {color:green} mvnsite {color} | {color:green} 0m
55s{color} | {color:green} HDFS-10285 passed {color} |
| {color:red}-1{color} | {color:red} shadedclient {color} | {color:red} 11m
7s{color} | {color:red} branch has errors when building and testing our client
artifacts. {color} |
| {color:green}+1{color} | {color:green} findbugs {color} | {color:green} 1m
43s{color} | {color:green} HDFS-10285 passed {color} |
| {color:green}+1{color} | {color:green} javadoc {color} | {color:green} 0m
41s{color} | {color:green} HDFS-10285 passed {color} |
|| || || || {color:brown} Patch Compile Tests {color} ||
| {color:green}+1{color} | {color:green} mvninstall {color} | {color:green} 0m
49s{color} | {color:green} the patch passed {color} |
| {color:green}+1{color} | {color:green} compile {color} | {color:green} 0m
46s{color} | {color:green} the patch passed {color} |
| {color:green}+1{color} | {color:green} cc {color} | {color:green} 0m
46s{color} | {color:green} the patch passed {color} |
| {color:green}+1{color} | {color:green} javac {color} | {color:green} 0m
46s{color} | {color:green} the patch passed {color} |
| {color:orange}-0{color} | {color:orange} checkstyle {color} | {color:orange}
0m 44s{color} | {color:orange} hadoop-hdfs-project/hadoop-hdfs: The patch
generated 10 new + 725 unchanged - 2 fixed = 735 total (was 727) {color} |
| {color:green}+1{color} | {color:green} mvnsite {color} | {color:green} 0m
51s{color} | {color:green} the patch passed {color} |
| {color:green}+1{color} | {color:green} whitespace {color} | {color:green} 0m
0s{color} | {color:green} The patch has no whitespace issues. {color} |
| {color:red}-1{color} | {color:red} shadedclient {color} | {color:red} 10m
29s{color} | {color:red} patch has errors when building and testing our client
artifacts. {color} |
| {color:red}-1{color} | {color:red} findbugs {color} | {color:red} 1m
52s{color} | {color:red} hadoop-hdfs-project/hadoop-hdfs generated 1 new + 0
unchanged - 0 fixed = 1 total (was 0) {color} |
| {color:red}-1{color} | {color:red} javadoc {color} | {color:red} 0m
40s{color} | {color:red} hadoop-hdfs-project_hadoop-hdfs generated 1 new + 9
unchanged - 0 fixed = 10 total (was 9) {color} |
|| || || || {color:brown} Other Tests {color} ||
| {color:red}-1{color} | {color:red} unit {color} | {color:red} 99m 2s{color}
| {color:red} hadoop-hdfs in the patch failed. {color} |
| {color:green}+1{color} | {color:green} asflicense {color} | {color:green} 0m
24s{color} | {color:green} The patch does not generate ASF License warnings.
{color} |
| {color:black}{color} | {color:black} {color} | {color:black}145m 48s{color} |
{color:black} {color} |
\\
\\
|| Reason || Tests ||
| FindBugs | module:hadoop-hdfs-project/hadoop-hdfs |
| | Invocation of toString on
BlocksStorageMoveAttemptFinished.movementFinishedBlocks in
org.apache.hadoop.hdfs.server.protocol.BlocksStorageMoveAttemptFinished.toString()
At BlocksStorageMoveAttemptFinished.java:in
org.apache.hadoop.hdfs.server.protocol.BlocksStorageMoveAttemptFinished.toString()
At BlocksStorageMoveAttemptFinished.java:[line 43] |
| Failed junit tests | hadoop.hdfs.TestReadStripedFileWithDecoding |
| | hadoop.hdfs.server.namenode.TestPersistentStoragePolicySatisfier |
| | hadoop.hdfs.server.namenode.TestStoragePolicySatisfier |
| | hadoop.hdfs.TestDFSStripedOutputStreamWithFailure150 |
| | hadoop.hdfs.TestDFSStripedInputStreamWithRandomECPolicy |
| | hadoop.hdfs.TestDFSStripedOutputStreamWithFailureWithRandomECPolicy |
| | hadoop.hdfs.TestDFSStripedOutputStreamWithFailure050 |
| | hadoop.hdfs.TestDFSStripedOutputStreamWithFailure080 |
| |
