[
https://issues.apache.org/jira/browse/HDFS-9670?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Yongjun Zhang updated HDFS-9670:
--------------------------------
Description:
Symptom:
{quote}
[root@vb0724 ~]# hadoop distcp hdfs://X:8020/ hdfs://Y:8020/
16/01/20 11:33:33 INFO tools.DistCp: Input Options:
DistCpOptions{atomicCommit=false, syncFolder=false, deleteMissing=false,
ignoreFailures=false, maxMaps=20, sslConfigurationFile='null',
copyStrategy='uniformsize', sourceFileListing=null,
sourcePaths=[hdfs://X:8020/], targetPath=hdfs://Y:8020/, targetPathExists=true,
preserveRawXattrs=false, filtersFile='null'}
16/01/20 11:33:33 INFO client.RMProxy: Connecting to ResourceManager at Z:8032
16/01/20 11:33:33 ERROR tools.DistCp: Exception encountered
java.lang.NullPointerException
at
org.apache.hadoop.tools.util.DistCpUtils.getRelativePath(DistCpUtils.java:144)
at
org.apache.hadoop.tools.SimpleCopyListing.writeToFileListing(SimpleCopyListing.java:598)
at
org.apache.hadoop.tools.SimpleCopyListing.writeToFileListingRoot(SimpleCopyListing.java:583)
at
org.apache.hadoop.tools.SimpleCopyListing.doBuildListing(SimpleCopyListing.java:313)
at
org.apache.hadoop.tools.SimpleCopyListing.doBuildListing(SimpleCopyListing.java:174)
at org.apache.hadoop.tools.CopyListing.buildListing(CopyListing.java:86)
at
org.apache.hadoop.tools.GlobbedCopyListing.doBuildListing(GlobbedCopyListing.java:90)
at org.apache.hadoop.tools.CopyListing.buildListing(CopyListing.java:86)
at org.apache.hadoop.tools.DistCp.createInputFileListing(DistCp.java:365)
at org.apache.hadoop.tools.DistCp.execute(DistCp.java:171)
at org.apache.hadoop.tools.DistCp.run(DistCp.java:122)
at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:70)
at org.apache.hadoop.tools.DistCp.main(DistCp.java:429)
{quote}
Relevant code:
{code}
private Path computeSourceRootPath(FileStatus sourceStatus,
DistCpOptions options) throws IOException {
Path target = options.getTargetPath();
FileSystem targetFS = target.getFileSystem(getConf());
final boolean targetPathExists = options.getTargetPathExists();
boolean solitaryFile = options.getSourcePaths().size() == 1
&& !sourceStatus.isDirectory();
if (solitaryFile) {
if (targetFS.isFile(target) || !targetPathExists) {
return sourceStatus.getPath();
} else {
return sourceStatus.getPath().getParent();
}
} else {
boolean specialHandling = (options.getSourcePaths().size() == 1 &&
!targetPathExists) ||
options.shouldSyncFolder() || options.shouldOverwrite();
return specialHandling && sourceStatus.isDirectory() ?
sourceStatus.getPath() :
sourceStatus.getPath().getParent();
}
}
{code}
We can see that it could return NULL at the end when doing
{{sourceStatus.getPath().getParent()}}
was:
Symptom:
[root@vb0724 ~]# hadoop distcp hdfs://X:8020/ hdfs://Y:8020/
16/01/20 11:33:33 INFO tools.DistCp: Input Options:
DistCpOptions{atomicCommit=false, syncFolder=false, deleteMissing=false,
ignoreFailures=false, maxMaps=20, sslConfigurationFile='null',
copyStrategy='uniformsize', sourceFileListing=null,
sourcePaths=[hdfs://X:8020/], targetPath=hdfs://Y:8020/, targetPathExists=true,
preserveRawXattrs=false, filtersFile='null'}
16/01/20 11:33:33 INFO client.RMProxy: Connecting to ResourceManager at Z:8032
16/01/20 11:33:33 ERROR tools.DistCp: Exception encountered
java.lang.NullPointerException
at
org.apache.hadoop.tools.util.DistCpUtils.getRelativePath(DistCpUtils.java:144)
at
org.apache.hadoop.tools.SimpleCopyListing.writeToFileListing(SimpleCopyListing.java:598)
at
org.apache.hadoop.tools.SimpleCopyListing.writeToFileListingRoot(SimpleCopyListing.java:583)
at
org.apache.hadoop.tools.SimpleCopyListing.doBuildListing(SimpleCopyListing.java:313)
at
org.apache.hadoop.tools.SimpleCopyListing.doBuildListing(SimpleCopyListing.java:174)
at org.apache.hadoop.tools.CopyListing.buildListing(CopyListing.java:86)
at
org.apache.hadoop.tools.GlobbedCopyListing.doBuildListing(GlobbedCopyListing.java:90)
at org.apache.hadoop.tools.CopyListing.buildListing(CopyListing.java:86)
at org.apache.hadoop.tools.DistCp.createInputFileListing(DistCp.java:365)
at org.apache.hadoop.tools.DistCp.execute(DistCp.java:171)
at org.apache.hadoop.tools.DistCp.run(DistCp.java:122)
at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:70)
at org.apache.hadoop.tools.DistCp.main(DistCp.java:429)
{code}
Relevant code:
{code}
private Path computeSourceRootPath(FileStatus sourceStatus,
DistCpOptions options) throws IOException {
Path target = options.getTargetPath();
FileSystem targetFS = target.getFileSystem(getConf());
final boolean targetPathExists = options.getTargetPathExists();
boolean solitaryFile = options.getSourcePaths().size() == 1
&& !sourceStatus.isDirectory();
if (solitaryFile) {
if (targetFS.isFile(target) || !targetPathExists) {
return sourceStatus.getPath();
} else {
return sourceStatus.getPath().getParent();
}
} else {
boolean specialHandling = (options.getSourcePaths().size() == 1 &&
!targetPathExists) ||
options.shouldSyncFolder() || options.shouldOverwrite();
return specialHandling && sourceStatus.isDirectory() ?
sourceStatus.getPath() :
sourceStatus.getPath().getParent();
}
}
{code}
We can see that it could return NULL at the end when doing
{{sourceStatus.getPath().getParent()}}
> DistCp throws NPE when source is root "/"
> -----------------------------------------
>
> Key: HDFS-9670
> URL: https://issues.apache.org/jira/browse/HDFS-9670
> Project: Hadoop HDFS
> Issue Type: Bug
> Reporter: Yongjun Zhang
> Assignee: Yongjun Zhang
>
> Symptom:
> {quote}
> [root@vb0724 ~]# hadoop distcp hdfs://X:8020/ hdfs://Y:8020/
> 16/01/20 11:33:33 INFO tools.DistCp: Input Options:
> DistCpOptions{atomicCommit=false, syncFolder=false, deleteMissing=false,
> ignoreFailures=false, maxMaps=20, sslConfigurationFile='null',
> copyStrategy='uniformsize', sourceFileListing=null,
> sourcePaths=[hdfs://X:8020/], targetPath=hdfs://Y:8020/,
> targetPathExists=true, preserveRawXattrs=false, filtersFile='null'}
> 16/01/20 11:33:33 INFO client.RMProxy: Connecting to ResourceManager at Z:8032
> 16/01/20 11:33:33 ERROR tools.DistCp: Exception encountered
> java.lang.NullPointerException
> at
> org.apache.hadoop.tools.util.DistCpUtils.getRelativePath(DistCpUtils.java:144)
> at
> org.apache.hadoop.tools.SimpleCopyListing.writeToFileListing(SimpleCopyListing.java:598)
> at
> org.apache.hadoop.tools.SimpleCopyListing.writeToFileListingRoot(SimpleCopyListing.java:583)
> at
> org.apache.hadoop.tools.SimpleCopyListing.doBuildListing(SimpleCopyListing.java:313)
> at
> org.apache.hadoop.tools.SimpleCopyListing.doBuildListing(SimpleCopyListing.java:174)
> at org.apache.hadoop.tools.CopyListing.buildListing(CopyListing.java:86)
> at
> org.apache.hadoop.tools.GlobbedCopyListing.doBuildListing(GlobbedCopyListing.java:90)
> at org.apache.hadoop.tools.CopyListing.buildListing(CopyListing.java:86)
> at org.apache.hadoop.tools.DistCp.createInputFileListing(DistCp.java:365)
> at org.apache.hadoop.tools.DistCp.execute(DistCp.java:171)
> at org.apache.hadoop.tools.DistCp.run(DistCp.java:122)
> at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:70)
> at org.apache.hadoop.tools.DistCp.main(DistCp.java:429)
> {quote}
> Relevant code:
> {code}
> private Path computeSourceRootPath(FileStatus sourceStatus,
> DistCpOptions options) throws
> IOException {
> Path target = options.getTargetPath();
> FileSystem targetFS = target.getFileSystem(getConf());
> final boolean targetPathExists = options.getTargetPathExists();
> boolean solitaryFile = options.getSourcePaths().size() == 1
> &&
> !sourceStatus.isDirectory();
> if (solitaryFile) {
> if (targetFS.isFile(target) || !targetPathExists) {
> return sourceStatus.getPath();
> } else {
> return sourceStatus.getPath().getParent();
> }
> } else {
> boolean specialHandling = (options.getSourcePaths().size() == 1 &&
> !targetPathExists) ||
> options.shouldSyncFolder() || options.shouldOverwrite();
> return specialHandling && sourceStatus.isDirectory() ?
> sourceStatus.getPath() :
> sourceStatus.getPath().getParent();
> }
> }
> {code}
> We can see that it could return NULL at the end when doing
> {{sourceStatus.getPath().getParent()}}
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
