amend OOZIE-2874 Make the Launcher Mapper map-only job's InputFormat class pluggable (andras.piros via gezapeti)
Project: http://git-wip-us.apache.org/repos/asf/oozie/repo Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/11038820 Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/11038820 Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/11038820 Branch: refs/heads/oya Commit: 110388208a232651fb82fa8e856541ef24aff81b Parents: 5cb3f33 Author: Gezapeti Cseh <[email protected]> Authored: Tue May 9 22:38:49 2017 -0700 Committer: Gezapeti Cseh <[email protected]> Committed: Tue May 9 22:40:17 2017 -0700 ---------------------------------------------------------------------- .../hadoop/LauncherInputFormatClassLocator.java | 84 ++++++++++++++++++++ .../TestLauncherInputFormatClassLocator.java | 57 +++++++++++++ 2 files changed, 141 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/11038820/core/src/main/java/org/apache/oozie/action/hadoop/LauncherInputFormatClassLocator.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/LauncherInputFormatClassLocator.java b/core/src/main/java/org/apache/oozie/action/hadoop/LauncherInputFormatClassLocator.java new file mode 100644 index 0000000..4dba776 --- /dev/null +++ b/core/src/main/java/org/apache/oozie/action/hadoop/LauncherInputFormatClassLocator.java @@ -0,0 +1,84 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.oozie.action.hadoop; + +import org.apache.hadoop.mapred.InputFormat; +import org.apache.oozie.service.ConfigurationService; +import org.apache.oozie.util.XLog; + +class LauncherInputFormatClassLocator { + private final XLog LOG = XLog.getLog(getClass()); + + static final String HADOOP_INPUT_FORMAT_CLASSNAME = "mapreduce.input.format.class"; + + private Class<? extends InputFormat> launcherInputFormatClass; + + /** + * Get the {@code LauncherMapper} map only MR job's {@link org.apache.hadoop.mapreduce.InputFormat} according to configuration. + * <p/> + * Priority: + * <ol> + * <li><tt>oozie.action.launcher.mapreduce.input-format.class-name</tt></li> + * <li><tt>OozieLauncherInputFormat</tt></li> + * </ol> + * + * @return + */ + Class<? extends InputFormat> locateOrGet() { + if (launcherInputFormatClass == null) { + launcherInputFormatClass = locate(); + } + + return launcherInputFormatClass; + } + + /** + * Get the {@code LauncherMapper} map only MR job's {@link org.apache.hadoop.mapreduce.InputFormat} according to configuration. + * <p/> + * Priority: + * <ol> + * <li><tt>oozie.action.launcher.mapreduce.input-format.class-name</tt></li> + * <li><tt>OozieLauncherInputFormat</tt></li> + * </ol> + * + * @return + */ + private Class<? extends InputFormat> locate() { + String inputFormatClassName = OozieLauncherInputFormat.class.getSimpleName(); + + final String configuredClassName = ConfigurationService.get( + JavaActionExecutor.OOZIE_ACTION_LAUNCHER_PREFIX + HADOOP_INPUT_FORMAT_CLASSNAME); + if (configuredClassName != null) { + inputFormatClassName = configuredClassName; + } + + try { + LOG.debug("Locating launcher input format class [{0}].", inputFormatClassName); + + final Class inputFormatClass = Class.forName(inputFormatClassName); + LOG.debug("Launcher input format class [{0}] located successfully.", inputFormatClassName); + + return inputFormatClass; + } catch (final ClassNotFoundException | ClassCastException e) { + LOG.warn("Could not load class [{0}], located [{1}] instead.", inputFormatClassName, + OozieLauncherInputFormat.class.getSimpleName()); + return OozieLauncherInputFormat.class; + } + } +} http://git-wip-us.apache.org/repos/asf/oozie/blob/11038820/core/src/test/java/org/apache/oozie/action/hadoop/TestLauncherInputFormatClassLocator.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/action/hadoop/TestLauncherInputFormatClassLocator.java b/core/src/test/java/org/apache/oozie/action/hadoop/TestLauncherInputFormatClassLocator.java new file mode 100644 index 0000000..a6ccc5a --- /dev/null +++ b/core/src/test/java/org/apache/oozie/action/hadoop/TestLauncherInputFormatClassLocator.java @@ -0,0 +1,57 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.oozie.action.hadoop; + +import org.apache.hadoop.mapreduce.InputFormat; +import org.apache.oozie.service.ConfigurationService; + +public class TestLauncherInputFormatClassLocator extends ActionExecutorTestCase { + + public void testInputFormatClassnameNotSet() { + ConfigurationService.set( + "oozie.action.launcher.mapreduce.input.format.class", + ""); + + final Class inputFormatClass = new LauncherInputFormatClassLocator().locateOrGet(); + + assertEquals("default launcher input format is found when configuration is left empty", + OozieLauncherInputFormat.class, + inputFormatClass); + } + + public void testInputFormatClassnameDefault() { + final Class inputFormatClass = new LauncherInputFormatClassLocator().locateOrGet(); + + assertEquals("default launcher input format is found when configuration is left on default", + OozieLauncherInputFormat.class, + inputFormatClass); + } + + public void testInputFormatClassnameSet() { + ConfigurationService.set( + "oozie.action.launcher.mapreduce.input.format.class", + InputFormat.class.getName()); + + final Class inputFormatClass = new LauncherInputFormatClassLocator().locateOrGet(); + + assertEquals("Hadoop launcher input format is found when configuration is set", + InputFormat.class, + inputFormatClass); + } +} \ No newline at end of file
