Repository: impala Updated Branches: refs/heads/master c1362afb9 -> 1ca077fd0
IMPALA-7063: Fix compilation for MiniProfile2 after Erasure Coding changes. This commit shims out HDFS's FileStatus.isErasureCoded() to manage working with multiple versions of Hadoop. I tested compilation with both profiles. Cherry-picks: not for 2.x. Change-Id: I423087078f84b0806545322519f224d58815123d Reviewed-on: http://gerrit.cloudera.org:8080/10487 Reviewed-by: Philip Zeyliger <[email protected]> Tested-by: Impala Public Jenkins <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/impala/repo Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/879f106d Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/879f106d Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/879f106d Branch: refs/heads/master Commit: 879f106d1b28380fbe71680d04cf6df568b3daa3 Parents: c1362af Author: Philip Zeyliger <[email protected]> Authored: Wed May 23 09:36:28 2018 -0700 Committer: Impala Public Jenkins <[email protected]> Committed: Wed May 23 21:45:09 2018 +0000 ---------------------------------------------------------------------- .../java/org/apache/impala/compat/HdfsShim.java | 31 ++++++++++++++++++++ .../java/org/apache/impala/compat/HdfsShim.java | 30 +++++++++++++++++++ .../org/apache/impala/catalog/HdfsTable.java | 5 ++-- 3 files changed, 64 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/impala/blob/879f106d/fe/src/compat-minicluster-profile-2/java/org/apache/impala/compat/HdfsShim.java ---------------------------------------------------------------------- diff --git a/fe/src/compat-minicluster-profile-2/java/org/apache/impala/compat/HdfsShim.java b/fe/src/compat-minicluster-profile-2/java/org/apache/impala/compat/HdfsShim.java new file mode 100644 index 0000000..ef3da61 --- /dev/null +++ b/fe/src/compat-minicluster-profile-2/java/org/apache/impala/compat/HdfsShim.java @@ -0,0 +1,31 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.impala.compat; + +import org.apache.hadoop.fs.FileStatus; + +/** + * Wrapper classes to abstract away differences between HDFS versions in + * the MiniCluster profiles. + */ +public class HdfsShim { + public static boolean isErasureCoded(FileStatus fileStatus) { + // Hadoop 2 didn't support Erasure Coding + return false; + } +} http://git-wip-us.apache.org/repos/asf/impala/blob/879f106d/fe/src/compat-minicluster-profile-3/java/org/apache/impala/compat/HdfsShim.java ---------------------------------------------------------------------- diff --git a/fe/src/compat-minicluster-profile-3/java/org/apache/impala/compat/HdfsShim.java b/fe/src/compat-minicluster-profile-3/java/org/apache/impala/compat/HdfsShim.java new file mode 100644 index 0000000..9453f80 --- /dev/null +++ b/fe/src/compat-minicluster-profile-3/java/org/apache/impala/compat/HdfsShim.java @@ -0,0 +1,30 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.impala.compat; + +import org.apache.hadoop.fs.FileStatus; + +/** + * Wrapper classes to abstract away differences between HDFS versions in + * the MiniCluster profiles. + */ +public class HdfsShim { + public static boolean isErasureCoded(FileStatus fileStatus) { + return fileStatus.isErasureCoded(); + } +} http://git-wip-us.apache.org/repos/asf/impala/blob/879f106d/fe/src/main/java/org/apache/impala/catalog/HdfsTable.java ---------------------------------------------------------------------- diff --git a/fe/src/main/java/org/apache/impala/catalog/HdfsTable.java b/fe/src/main/java/org/apache/impala/catalog/HdfsTable.java index d6b8fb8..8d744fc 100644 --- a/fe/src/main/java/org/apache/impala/catalog/HdfsTable.java +++ b/fe/src/main/java/org/apache/impala/catalog/HdfsTable.java @@ -64,6 +64,7 @@ import org.apache.impala.common.FileSystemUtil; import org.apache.impala.common.Pair; import org.apache.impala.common.PrintUtils; import org.apache.impala.common.Reference; +import org.apache.impala.compat.HdfsShim; import org.apache.impala.fb.FbFileBlock; import org.apache.impala.service.BackendConfig; import org.apache.impala.thrift.ImpalaInternalServiceConstants; @@ -441,7 +442,7 @@ public class HdfsTable extends Table { partitions.get(0).getFileFormat(), hostIndex_); } else { fd = FileDescriptor.create(fileStatus, fileStatus.getBlockLocations(), fs, - hostIndex_, fileStatus.isErasureCoded(), numUnknownDiskIds); + hostIndex_, HdfsShim.isErasureCoded(fileStatus), numUnknownDiskIds); } newFileDescs.add(fd); ++loadStats.loadedFiles; @@ -514,7 +515,7 @@ public class HdfsTable extends Table { BlockLocation[] locations = fs.getFileBlockLocations(fileStatus, 0, fileStatus.getLen()); fd = FileDescriptor.create(fileStatus, locations, fs, hostIndex_, - fileStatus.isErasureCoded(), numUnknownDiskIds); + HdfsShim.isErasureCoded(fileStatus), numUnknownDiskIds); } ++loadStats.loadedFiles; } else {
