[
https://issues.apache.org/jira/browse/HDFS-16757?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=17906593#comment-17906593
]
ASF GitHub Bot commented on HDFS-16757:
---------------------------------------
tomscut commented on code in PR #6926:
URL: https://github.com/apache/hadoop/pull/6926#discussion_r1889576893
##########
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java:
##########
@@ -4388,4 +4409,94 @@ boolean isSlownode() {
public BlockPoolManager getBlockPoolManager() {
return blockPoolManager;
}
+
+ public void copyBlockCrossNamespace(ExtendedBlock sourceBlk, ExtendedBlock
targetBlk,
+ DatanodeInfo targetDn) throws IOException {
+ if (!data.isValidBlock(sourceBlk)) {
+ // block does not exist or is under-construction
+ String errStr =
+ "copyBlock:(" + this.getInfoPort() + ") Can't send invalid block " +
sourceBlk + " "
+ + data.getReplicaString(sourceBlk.getBlockPoolId(),
sourceBlk.getBlockId());
+ LOG.info(errStr);
+ throw new IOException(errStr);
+ }
+ long onDiskLength = data.getLength(sourceBlk);
+ if (sourceBlk.getNumBytes() > onDiskLength) {
+ // Shorter on-disk len indicates corruption so report NN the corrupt
block
+ String msg = "copyBlock: Can't replicate block " + sourceBlk + " because
on-disk length "
+ + onDiskLength + " is shorter than provided length " +
sourceBlk.getNumBytes();
+ LOG.info(msg);
+ throw new IOException(msg);
+ }
+ LOG.info(getDatanodeInfo() + " copyBlock: Starting thread to transfer: " +
"block:"
+ + sourceBlk + " from " + this.getDatanodeUuid() + " to " +
targetDn.getDatanodeUuid()
+ + "(" + targetDn + ")");
+ Future<?> result;
+ if (this.getDatanodeUuid().equals(targetDn.getDatanodeUuid())) {
+ result = copyBlockCrossNamespaceExecutor.submit(new
LocalBlockCopy(sourceBlk, targetBlk));
+ } else {
+ result = copyBlockCrossNamespaceExecutor.submit(
+ new DataCopy(targetDn, sourceBlk, targetBlk).getDataTransfer());
+ }
+ try {
+ result.get(getDnConf().getCopyBlockCrossNamespaceSocketTimeout(),
TimeUnit.MILLISECONDS);
+ } catch (Exception e) {
+ LOG.error(e.getMessage());
+ throw new IOException(e);
+ }
+ }
+
+ private class DataCopy {
+ private final DataTransfer dataTransfer;
+
+ DataCopy(DatanodeInfo targetDn, ExtendedBlock sourceBlk, ExtendedBlock
targetBlk) {
+ FsVolumeImpl volume = (FsVolumeImpl) data.getVolume(sourceBlk);
+ StorageType storageType = volume.getStorageType();
+ String storageId = volume.getStorageID();
+
+ DatanodeInfo[] targets = new DatanodeInfo[] {targetDn};
+ StorageType[] targetStorageTypes = new StorageType[] {storageType};
+ String[] targetStorageIds = new String[] {storageId};
+ dataTransfer =
+ new DataTransfer(targets, targetStorageTypes, targetStorageIds,
sourceBlk, targetBlk,
+ PIPELINE_SETUP_CREATE, "");
+ }
+
+ public DataTransfer getDataTransfer() {
+ return dataTransfer;
+ }
+ }
+
+ class LocalBlockCopy implements Callable<Boolean> {
+ private ExtendedBlock sourceBlk = null;
+ private ExtendedBlock targetBlk = null;
+
+ LocalBlockCopy(ExtendedBlock sourceBlk, ExtendedBlock targetBlk) {
+ this.sourceBlk = sourceBlk;
+ this.targetBlk = targetBlk;
+ }
+
+ public Boolean call() throws IOException {
+ try {
+ targetBlk.setNumBytes(sourceBlk.getNumBytes());
+ data.hardLinkOneBlock(sourceBlk, targetBlk);
+ FsVolumeSpi v = (FsVolumeSpi) (getFSDataset().getVolume(targetBlk));
+ closeBlock(targetBlk, null, v.getStorageID(), v.isTransientStorage());
+
+ BlockLocalPathInfo srcBlpi = data.getBlockLocalPathInfo(sourceBlk);
+ BlockLocalPathInfo dstBlpi = data.getBlockLocalPathInfo(targetBlk);
+ LOG.info(
+ getClass().getSimpleName() + ": Hardlinked " + sourceBlk + "( " +
srcBlpi.getBlockPath()
+ + " " + srcBlpi.getMetaPath() + " ) " + "to " + targetBlk + "(
"
+ + dstBlpi.getBlockPath() + " " + dstBlpi.getMetaPath() + " )
");
+
Review Comment:
@LiuGuH If this is fixed, please "Resolve conversation", thanks.
> Add a new method copyBlockCrossNamespace to DataNode
> ----------------------------------------------------
>
> Key: HDFS-16757
> URL: https://issues.apache.org/jira/browse/HDFS-16757
> Project: Hadoop HDFS
> Issue Type: Sub-task
> Reporter: ZanderXu
> Assignee: liuguanghua
> Priority: Minor
> Labels: pull-request-available
>
> Add a new method copyBlockCrossNamespace in DataTransferProtocol at the
> DataNode Side.
> This method will copy a source block from one namespace to a target block
> from a different namespace. If the target DN is the same with the current DN,
> this method will copy the block via HardLink. If the target DN is different
> with the current DN, this method will copy the block via TransferBlock.
> This method will contains some parameters:
> * ExtendedBlock sourceBlock
> * Token<BlockTokenIdentifier> sourceBlockToken
> * ExtendedBlock targetBlock
> * Token<BlockTokenIdentifier> targetBlockToken
> * DatanodeInfo targetDN
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
---------------------------------------------------------------------
To unsubscribe, e-mail: hdfs-issues-unsubscr...@hadoop.apache.org
For additional commands, e-mail: hdfs-issues-h...@hadoop.apache.org
