[
https://issues.apache.org/jira/browse/HDFS-8424?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Sumana Sathish updated HDFS-8424:
---------------------------------
Description:
For TDE, distcp using HFTP protocol fails on raw namespace with the following
exception
{code}
sudo su - -c "/usr/hdp/current/hadoop-client/bin/hadoop distcp -i
-skipcrccheck -update
hftp://ip-172-31-33-229.ec2.internal:50070/.reserved/raw/user/hrt_qa/srcDistcpWithTDE/smallFiles/
hdfs://ip-172-31-33-229.ec2.internal:8020/.reserved/raw/user/hrt_qa/destDistcpWithTDE/distcpedFiles/"
hdfs
2015-05-15
23:55:35,579|beaver.machine|INFO|2581|140406820472576|MainThread|15/05/15
23:55:35 INFO tools.DistCp: Input Options: DistCpOptions{atomicCommit=false,
syncFolder=true, deleteMissing=false, ignoreFailures=true, maxMaps=20,
sslConfigurationFile='null', copyStrategy='uniformsize',
sourceFileListing=null,
sourcePaths=[hftp://ip-172-31-33-229.ec2.internal:50070/.reserved/raw/user/hrt_qa/srcDistcpWithTDE/smallFiles],
targetPath=hdfs://ip-172-31-33-229.ec2.internal:8020/.reserved/raw/user/hrt_qa/destDistcpWithTDE/distcpedFiles,
targetPathExists=false, preserveRawXattrs=false}
2015-05-15
23:55:36,829|beaver.machine|INFO|2581|140406820472576|MainThread|15/05/15
23:55:36 INFO impl.TimelineClientImpl: Timeline service address:
http://ip-172-31-33-229.ec2.internal:8188/ws/v1/timeline/
2015-05-15
23:55:37,073|beaver.machine|INFO|2581|140406820472576|MainThread|15/05/15
23:55:37 INFO client.RMProxy: Connecting to ResourceManager at
ip-172-31-33-229.ec2.internal/172.31.33.229:8050
2015-05-15
23:55:38,040|beaver.machine|INFO|2581|140406820472576|MainThread|15/05/15
23:55:38 ERROR tools.DistCp: Exception encountered
2015-05-15
23:55:38,040|beaver.machine|INFO|2581|140406820472576|MainThread|java.lang.UnsupportedOperationException:
HftpFileSystem doesn't support getXAttrs
2015-05-15 23:55:38,041|beaver.machine|INFO|2581|140406820472576|MainThread|at
org.apache.hadoop.fs.FileSystem.getXAttrs(FileSystem.java:2559)
2015-05-15 23:55:38,041|beaver.machine|INFO|2581|140406820472576|MainThread|at
org.apache.hadoop.tools.util.DistCpUtils.toCopyListingFileStatus(DistCpUtils.java:322)
2015-05-15 23:55:38,042|beaver.machine|INFO|2581|140406820472576|MainThread|at
org.apache.hadoop.tools.SimpleCopyListing.doBuildListing(SimpleCopyListing.java:177)
2015-05-15 23:55:38,042|beaver.machine|INFO|2581|140406820472576|MainThread|at
org.apache.hadoop.tools.SimpleCopyListing.doBuildListing(SimpleCopyListing.java:140)
2015-05-15 23:55:38,043|beaver.machine|INFO|2581|140406820472576|MainThread|at
org.apache.hadoop.tools.CopyListing.buildListing(CopyListing.java:84)
2015-05-15 23:55:38,043|beaver.machine|INFO|2581|140406820472576|MainThread|at
org.apache.hadoop.tools.GlobbedCopyListing.doBuildListing(GlobbedCopyListing.java:90)
2015-05-15 23:55:38,044|beaver.machine|INFO|2581|140406820472576|MainThread|at
org.apache.hadoop.tools.CopyListing.buildListing(CopyListing.java:84)
2015-05-15 23:55:38,044|beaver.machine|INFO|2581|140406820472576|MainThread|at
org.apache.hadoop.tools.DistCp.createInputFileListing(DistCp.java:382)
2015-05-15 23:55:38,045|beaver.machine|INFO|2581|140406820472576|MainThread|at
org.apache.hadoop.tools.DistCp.createAndSubmitJob(DistCp.java:181)
2015-05-15 23:55:38,045|beaver.machine|INFO|2581|140406820472576|MainThread|at
org.apache.hadoop.tools.DistCp.execute(DistCp.java:153)
2015-05-15 23:55:38,046|beaver.machine|INFO|2581|140406820472576|MainThread|at
org.apache.hadoop.tools.DistCp.run(DistCp.java:126)
2015-05-15 23:55:38,046|beaver.machine|INFO|2581|140406820472576|MainThread|at
org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:70)
2015-05-15 23:55:38,047|beaver.machine|INFO|2581|140406820472576|MainThread|at
org.apache.hadoop.tools.DistCp.main(DistCp.java:430)
{code}
was:
Distcp using HFTP protocol fails on raw namespace with the following exception
{code}
sudo su - -c "/usr/hdp/current/hadoop-client/bin/hadoop distcp -i
-skipcrccheck -update
hftp://ip-172-31-33-229.ec2.internal:50070/.reserved/raw/user/hrt_qa/srcDistcpWithTDE/smallFiles/
hdfs://ip-172-31-33-229.ec2.internal:8020/.reserved/raw/user/hrt_qa/destDistcpWithTDE/distcpedFiles/"
hdfs
2015-05-15
23:55:35,579|beaver.machine|INFO|2581|140406820472576|MainThread|15/05/15
23:55:35 INFO tools.DistCp: Input Options: DistCpOptions{atomicCommit=false,
syncFolder=true, deleteMissing=false, ignoreFailures=true, maxMaps=20,
sslConfigurationFile='null', copyStrategy='uniformsize',
sourceFileListing=null,
sourcePaths=[hftp://ip-172-31-33-229.ec2.internal:50070/.reserved/raw/user/hrt_qa/srcDistcpWithTDE/smallFiles],
targetPath=hdfs://ip-172-31-33-229.ec2.internal:8020/.reserved/raw/user/hrt_qa/destDistcpWithTDE/distcpedFiles,
targetPathExists=false, preserveRawXattrs=false}
2015-05-15
23:55:36,829|beaver.machine|INFO|2581|140406820472576|MainThread|15/05/15
23:55:36 INFO impl.TimelineClientImpl: Timeline service address:
http://ip-172-31-33-229.ec2.internal:8188/ws/v1/timeline/
2015-05-15
23:55:37,073|beaver.machine|INFO|2581|140406820472576|MainThread|15/05/15
23:55:37 INFO client.RMProxy: Connecting to ResourceManager at
ip-172-31-33-229.ec2.internal/172.31.33.229:8050
2015-05-15
23:55:38,040|beaver.machine|INFO|2581|140406820472576|MainThread|15/05/15
23:55:38 ERROR tools.DistCp: Exception encountered
2015-05-15
23:55:38,040|beaver.machine|INFO|2581|140406820472576|MainThread|java.lang.UnsupportedOperationException:
HftpFileSystem doesn't support getXAttrs
2015-05-15 23:55:38,041|beaver.machine|INFO|2581|140406820472576|MainThread|at
org.apache.hadoop.fs.FileSystem.getXAttrs(FileSystem.java:2559)
2015-05-15 23:55:38,041|beaver.machine|INFO|2581|140406820472576|MainThread|at
org.apache.hadoop.tools.util.DistCpUtils.toCopyListingFileStatus(DistCpUtils.java:322)
2015-05-15 23:55:38,042|beaver.machine|INFO|2581|140406820472576|MainThread|at
org.apache.hadoop.tools.SimpleCopyListing.doBuildListing(SimpleCopyListing.java:177)
2015-05-15 23:55:38,042|beaver.machine|INFO|2581|140406820472576|MainThread|at
org.apache.hadoop.tools.SimpleCopyListing.doBuildListing(SimpleCopyListing.java:140)
2015-05-15 23:55:38,043|beaver.machine|INFO|2581|140406820472576|MainThread|at
org.apache.hadoop.tools.CopyListing.buildListing(CopyListing.java:84)
2015-05-15 23:55:38,043|beaver.machine|INFO|2581|140406820472576|MainThread|at
org.apache.hadoop.tools.GlobbedCopyListing.doBuildListing(GlobbedCopyListing.java:90)
2015-05-15 23:55:38,044|beaver.machine|INFO|2581|140406820472576|MainThread|at
org.apache.hadoop.tools.CopyListing.buildListing(CopyListing.java:84)
2015-05-15 23:55:38,044|beaver.machine|INFO|2581|140406820472576|MainThread|at
org.apache.hadoop.tools.DistCp.createInputFileListing(DistCp.java:382)
2015-05-15 23:55:38,045|beaver.machine|INFO|2581|140406820472576|MainThread|at
org.apache.hadoop.tools.DistCp.createAndSubmitJob(DistCp.java:181)
2015-05-15 23:55:38,045|beaver.machine|INFO|2581|140406820472576|MainThread|at
org.apache.hadoop.tools.DistCp.execute(DistCp.java:153)
2015-05-15 23:55:38,046|beaver.machine|INFO|2581|140406820472576|MainThread|at
org.apache.hadoop.tools.DistCp.run(DistCp.java:126)
2015-05-15 23:55:38,046|beaver.machine|INFO|2581|140406820472576|MainThread|at
org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:70)
2015-05-15 23:55:38,047|beaver.machine|INFO|2581|140406820472576|MainThread|at
org.apache.hadoop.tools.DistCp.main(DistCp.java:430)
{code}
> distcp on raw namespace using HFTP protocol fails for TDE
> ---------------------------------------------------------
>
> Key: HDFS-8424
> URL: https://issues.apache.org/jira/browse/HDFS-8424
> Project: Hadoop HDFS
> Issue Type: Bug
> Components: security
> Affects Versions: 2.3.0
> Reporter: Sumana Sathish
> Priority: Critical
>
> For TDE, distcp using HFTP protocol fails on raw namespace with the following
> exception
> {code}
> sudo su - -c "/usr/hdp/current/hadoop-client/bin/hadoop distcp -i
> -skipcrccheck -update
> hftp://ip-172-31-33-229.ec2.internal:50070/.reserved/raw/user/hrt_qa/srcDistcpWithTDE/smallFiles/
>
> hdfs://ip-172-31-33-229.ec2.internal:8020/.reserved/raw/user/hrt_qa/destDistcpWithTDE/distcpedFiles/"
> hdfs
> 2015-05-15
> 23:55:35,579|beaver.machine|INFO|2581|140406820472576|MainThread|15/05/15
> 23:55:35 INFO tools.DistCp: Input Options: DistCpOptions{atomicCommit=false,
> syncFolder=true, deleteMissing=false, ignoreFailures=true, maxMaps=20,
> sslConfigurationFile='null', copyStrategy='uniformsize',
> sourceFileListing=null,
> sourcePaths=[hftp://ip-172-31-33-229.ec2.internal:50070/.reserved/raw/user/hrt_qa/srcDistcpWithTDE/smallFiles],
>
> targetPath=hdfs://ip-172-31-33-229.ec2.internal:8020/.reserved/raw/user/hrt_qa/destDistcpWithTDE/distcpedFiles,
> targetPathExists=false, preserveRawXattrs=false}
> 2015-05-15
> 23:55:36,829|beaver.machine|INFO|2581|140406820472576|MainThread|15/05/15
> 23:55:36 INFO impl.TimelineClientImpl: Timeline service address:
> http://ip-172-31-33-229.ec2.internal:8188/ws/v1/timeline/
> 2015-05-15
> 23:55:37,073|beaver.machine|INFO|2581|140406820472576|MainThread|15/05/15
> 23:55:37 INFO client.RMProxy: Connecting to ResourceManager at
> ip-172-31-33-229.ec2.internal/172.31.33.229:8050
> 2015-05-15
> 23:55:38,040|beaver.machine|INFO|2581|140406820472576|MainThread|15/05/15
> 23:55:38 ERROR tools.DistCp: Exception encountered
> 2015-05-15
> 23:55:38,040|beaver.machine|INFO|2581|140406820472576|MainThread|java.lang.UnsupportedOperationException:
> HftpFileSystem doesn't support getXAttrs
> 2015-05-15
> 23:55:38,041|beaver.machine|INFO|2581|140406820472576|MainThread|at
> org.apache.hadoop.fs.FileSystem.getXAttrs(FileSystem.java:2559)
> 2015-05-15
> 23:55:38,041|beaver.machine|INFO|2581|140406820472576|MainThread|at
> org.apache.hadoop.tools.util.DistCpUtils.toCopyListingFileStatus(DistCpUtils.java:322)
> 2015-05-15
> 23:55:38,042|beaver.machine|INFO|2581|140406820472576|MainThread|at
> org.apache.hadoop.tools.SimpleCopyListing.doBuildListing(SimpleCopyListing.java:177)
> 2015-05-15
> 23:55:38,042|beaver.machine|INFO|2581|140406820472576|MainThread|at
> org.apache.hadoop.tools.SimpleCopyListing.doBuildListing(SimpleCopyListing.java:140)
> 2015-05-15
> 23:55:38,043|beaver.machine|INFO|2581|140406820472576|MainThread|at
> org.apache.hadoop.tools.CopyListing.buildListing(CopyListing.java:84)
> 2015-05-15
> 23:55:38,043|beaver.machine|INFO|2581|140406820472576|MainThread|at
> org.apache.hadoop.tools.GlobbedCopyListing.doBuildListing(GlobbedCopyListing.java:90)
> 2015-05-15
> 23:55:38,044|beaver.machine|INFO|2581|140406820472576|MainThread|at
> org.apache.hadoop.tools.CopyListing.buildListing(CopyListing.java:84)
> 2015-05-15
> 23:55:38,044|beaver.machine|INFO|2581|140406820472576|MainThread|at
> org.apache.hadoop.tools.DistCp.createInputFileListing(DistCp.java:382)
> 2015-05-15
> 23:55:38,045|beaver.machine|INFO|2581|140406820472576|MainThread|at
> org.apache.hadoop.tools.DistCp.createAndSubmitJob(DistCp.java:181)
> 2015-05-15
> 23:55:38,045|beaver.machine|INFO|2581|140406820472576|MainThread|at
> org.apache.hadoop.tools.DistCp.execute(DistCp.java:153)
> 2015-05-15
> 23:55:38,046|beaver.machine|INFO|2581|140406820472576|MainThread|at
> org.apache.hadoop.tools.DistCp.run(DistCp.java:126)
> 2015-05-15
> 23:55:38,046|beaver.machine|INFO|2581|140406820472576|MainThread|at
> org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:70)
> 2015-05-15
> 23:55:38,047|beaver.machine|INFO|2581|140406820472576|MainThread|at
> org.apache.hadoop.tools.DistCp.main(DistCp.java:430)
> {code}
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
