Repository: oozie Updated Branches: refs/heads/master a5f9aa54a -> f730573b8
OOZIE-3054 Disable erasure coding for sharelib if Oozie runs on Hadoop 3 (pbacsko) Project: http://git-wip-us.apache.org/repos/asf/oozie/repo Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/f730573b Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/f730573b Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/f730573b Branch: refs/heads/master Commit: f730573b83ccddc35158b4c1b66de9ae5147c977 Parents: a5f9aa5 Author: Peter Bacsko <[email protected]> Authored: Fri Sep 22 12:58:37 2017 +0200 Committer: Peter Bacsko <[email protected]> Committed: Fri Sep 22 12:58:37 2017 +0200 ---------------------------------------------------------------------- release-log.txt | 1 + .../apache/oozie/tools/ECPolicyDisabler.java | 114 +++++++++++++++++++ .../apache/oozie/tools/OozieSharelibCLI.java | 10 +- 3 files changed, 120 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/f730573b/release-log.txt ---------------------------------------------------------------------- diff --git a/release-log.txt b/release-log.txt index 803da2f..250a51e 100644 --- a/release-log.txt +++ b/release-log.txt @@ -1,5 +1,6 @@ -- Oozie 5.0.0 release (trunk - unreleased) +OOZIE-3054 Disable erasure coding for sharelib if Oozie runs on Hadoop 3 (pbacsko) OOZIE-3048 Check El Functions for the coordinator action (satishsaley) OOZIE-3058 nocleanup option is missing in oozie-coordinator-0.5.xsd (satishsaley) OOZIE-2909 LauncherAM: rewrite UGI calls (gezapeti) http://git-wip-us.apache.org/repos/asf/oozie/blob/f730573b/tools/src/main/java/org/apache/oozie/tools/ECPolicyDisabler.java ---------------------------------------------------------------------- diff --git a/tools/src/main/java/org/apache/oozie/tools/ECPolicyDisabler.java b/tools/src/main/java/org/apache/oozie/tools/ECPolicyDisabler.java new file mode 100644 index 0000000..2d7f1f8 --- /dev/null +++ b/tools/src/main/java/org/apache/oozie/tools/ECPolicyDisabler.java @@ -0,0 +1,114 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.oozie.tools; + +import java.lang.reflect.Method; + +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hdfs.DistributedFileSystem; + +/** + * Utility class which can disable Erasure Coding for a given path. + * + * Note that Erasure Coding was introduced in Hadoop 3, so in order for this class + * to be compilable, reflection is used. Later, when we drop support for Hadoop 2.x, + * this can be rewritten. + */ +public final class ECPolicyDisabler { + private static final String GETREPLICATIONPOLICY_METHOD = "getReplicationPolicy"; + private static final String ERASURECODING_POLICIES_CLASS = "org.apache.hadoop.hdfs.protocol.SystemErasureCodingPolicies"; + private static final String GETNAME_METHOD = "getName"; + private static final String SETERASURECODINGPOLICY_METHOD = "setErasureCodingPolicy"; + private static final String GETERASURECODINGPOLICY_METHOD = "getErasureCodingPolicy"; + + public static void tryDisableECPolicyForPath(FileSystem fs, Path path) { + if (fs instanceof DistributedFileSystem && supportsErasureCoding()) { + System.out.println("Found Hadoop that supports Erasure Coding. Trying to disable Erasure Coding for path: "+ path); + DistributedFileSystem dfs = (DistributedFileSystem) fs; + final Object replicationPolicy = getReplicationPolicy(); + Method getErasureCodingPolicyMethod = getMethod(dfs, GETERASURECODINGPOLICY_METHOD); + final Object currentECPolicy = invokeMethod(getErasureCodingPolicyMethod, dfs, path); + + if (currentECPolicy != replicationPolicy) { + Method setECPolicyMethod = getMethod(dfs, SETERASURECODINGPOLICY_METHOD); + Method policyGetNameMethod = getMethod(replicationPolicy, GETNAME_METHOD); + + String name = (String) invokeMethod(policyGetNameMethod, replicationPolicy); + + invokeMethod(setECPolicyMethod, dfs, path, name); + System.out.println("Done"); + } else { + System.out.println("Current policy is already replication"); + } + } else { + System.out.println("Found Hadoop that does not support Erasure Coding. Not taking any action."); + } + } + + private static boolean supportsErasureCoding() { + try { + getECPoliciesClass(); + return true; + } catch (ClassNotFoundException e) { + return false; + } + } + + private static Object getReplicationPolicy() { + try { + Class<?> c = getECPoliciesClass(); + Method m = c.getMethod(GETREPLICATIONPOLICY_METHOD); + return m.invoke(null); + } catch (Exception e) { + System.err.println("Error accessing method with reflection"); + throw new RuntimeException(e); + } + } + + private static Class<?> getECPoliciesClass() throws ClassNotFoundException { + return Class.forName(ERASURECODING_POLICIES_CLASS); + } + + private static Method getMethod(Object object, String methodName) { + Method[] methods = object.getClass().getMethods(); + Method method = null; + for (Method m : methods) { + if (m.getName().equals(methodName)) { + method = m; + break; + } + } + + if (method == null) { + throw new RuntimeException("Method " + methodName + "() not found"); + } + + return method; + } + + private static Object invokeMethod(Method m, Object instance, Object... args) { + try { + return m.invoke(instance, args); + } catch (Exception e) { + System.err.println("Error invoking method with reflection"); + throw new RuntimeException(e); + } + } +} http://git-wip-us.apache.org/repos/asf/oozie/blob/f730573b/tools/src/main/java/org/apache/oozie/tools/OozieSharelibCLI.java ---------------------------------------------------------------------- diff --git a/tools/src/main/java/org/apache/oozie/tools/OozieSharelibCLI.java b/tools/src/main/java/org/apache/oozie/tools/OozieSharelibCLI.java index a844aa0..c565d9d 100644 --- a/tools/src/main/java/org/apache/oozie/tools/OozieSharelibCLI.java +++ b/tools/src/main/java/org/apache/oozie/tools/OozieSharelibCLI.java @@ -160,6 +160,11 @@ public class OozieSharelibCLI { WorkflowAppService lwas = services.get(WorkflowAppService.class); HadoopAccessorService has = services.get(HadoopAccessorService.class); Path dstPath = lwas.getSystemLibPath(); + URI uri = new Path(hdfsUri).toUri(); + Configuration fsConf = has.createConfiguration(uri.getAuthority()); + FileSystem fs = FileSystem.get(uri, fsConf); + + ECPolicyDisabler.tryDisableECPolicyForPath(fs, dstPath); if (sharelibAction.equals(CREATE_CMD) || sharelibAction.equals(UPGRADE_CMD)){ dstPath= new Path(dstPath.toString() + Path.SEPARATOR + SHARE_LIB_PREFIX + getTimestampDirectory() ); @@ -167,11 +172,6 @@ public class OozieSharelibCLI { System.out.println("the destination path for sharelib is: " + dstPath); - URI uri = new Path(hdfsUri).toUri(); - Configuration fsConf = has.createConfiguration(uri.getAuthority()); - FileSystem fs = FileSystem.get(uri, fsConf); - - if (!srcFile.exists()){ throw new IOException(srcPath + " cannot be found"); }
