HIVE-11820 : export tables with size of >32MB throws java.lang.IllegalArgumentException: Skip CRC is valid only with update options (Takahiko Saito via Ashutosh Chauhan)
Signed-off-by: Ashutosh Chauhan <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/93a66276 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/93a66276 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/93a66276 Branch: refs/heads/beeline-cli Commit: 93a6627606cb2dad0e04c3a885f71c1be405f51d Parents: cc78dd5 Author: Takahiko Saito <[email protected]> Authored: Wed Sep 16 15:34:00 2015 -0800 Committer: Ashutosh Chauhan <[email protected]> Committed: Sat Sep 19 23:50:49 2015 -0700 ---------------------------------------------------------------------- .../main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java | 5 ++++- .../main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java | 4 +++- 2 files changed, 7 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/93a66276/shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java ---------------------------------------------------------------------- diff --git a/shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java b/shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java index 6c6ccbc..93dcbd3 100644 --- a/shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java +++ b/shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java @@ -73,6 +73,8 @@ import org.apache.hadoop.security.token.Token; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.tools.distcp2.DistCp; import org.apache.hadoop.tools.distcp2.DistCpOptions; +import org.apache.hadoop.tools.distcp2.DistCpOptions.FileAttribute; + import org.apache.hadoop.util.Progressable; import org.apache.hadoop.util.Tool; import org.apache.hadoop.util.VersionInfo; @@ -672,8 +674,9 @@ public class Hadoop20SShims extends HadoopShimsSecure { public boolean runDistCp(Path src, Path dst, Configuration conf) throws IOException { DistCpOptions options = new DistCpOptions(Collections.singletonList(src), dst); - options.setSkipCRC(true); options.setSyncFolder(true); + options.setSkipCRC(true); + options.preserve(FileAttribute.BLOCKSIZE); try { DistCp distcp = new DistCp(conf, options); distcp.execute(); http://git-wip-us.apache.org/repos/asf/hive/blob/93a66276/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java ---------------------------------------------------------------------- diff --git a/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java b/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java index 3292cb3..83369ee 100644 --- a/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java +++ b/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java @@ -97,6 +97,7 @@ import org.apache.hadoop.security.Credentials; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.tools.DistCp; import org.apache.hadoop.tools.DistCpOptions; +import org.apache.hadoop.tools.DistCpOptions.FileAttribute; import org.apache.hadoop.util.Progressable; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.tez.test.MiniTezCluster; @@ -1213,8 +1214,9 @@ public class Hadoop23Shims extends HadoopShimsSecure { public boolean runDistCp(Path src, Path dst, Configuration conf) throws IOException { DistCpOptions options = new DistCpOptions(Collections.singletonList(src), dst); - options.setSkipCRC(true); options.setSyncFolder(true); + options.setSkipCRC(true); + options.preserve(FileAttribute.BLOCKSIZE); try { DistCp distcp = new DistCp(conf, options); distcp.execute();
