Repository: oozie Updated Branches: refs/heads/master 359d08d22 -> 4c28591b7
OOZIE-1713 Avoid creating dummy input file for each launcher job (puru via rohini) Project: http://git-wip-us.apache.org/repos/asf/oozie/repo Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/4c28591b Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/4c28591b Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/4c28591b Branch: refs/heads/master Commit: 4c28591b76e1927af3a2f41b4b295ca5cd3eeb00 Parents: 359d08d Author: Rohini Palaniswamy <[email protected]> Authored: Tue Mar 25 13:35:58 2014 -0700 Committer: Rohini Palaniswamy <[email protected]> Committed: Tue Mar 25 13:35:58 2014 -0700 ---------------------------------------------------------------------- .../oozie/action/hadoop/JavaActionExecutor.java | 1 + .../action/hadoop/LauncherMapperHelper.java | 9 +- release-log.txt | 1 + .../action/hadoop/OozieLauncherInputFormat.java | 118 +++++++++++++++++++ 4 files changed, 121 insertions(+), 8 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/4c28591b/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java index 860eb51..52a0953 100644 --- a/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java +++ b/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java @@ -134,6 +134,7 @@ public class JavaActionExecutor extends ActionExecutor { public static List<Class> getCommonLauncherClasses() { List<Class> classes = new ArrayList<Class>(); classes.add(LauncherMapper.class); + classes.add(OozieLauncherInputFormat.class); classes.addAll(Services.get().get(URIHandlerService.class).getClassesForLauncher()); return classes; } http://git-wip-us.apache.org/repos/asf/oozie/blob/4c28591b/core/src/main/java/org/apache/oozie/action/hadoop/LauncherMapperHelper.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/LauncherMapperHelper.java b/core/src/main/java/org/apache/oozie/action/hadoop/LauncherMapperHelper.java index eeda715..b893200 100644 --- a/core/src/main/java/org/apache/oozie/action/hadoop/LauncherMapperHelper.java +++ b/core/src/main/java/org/apache/oozie/action/hadoop/LauncherMapperHelper.java @@ -155,14 +155,7 @@ public class LauncherMapperHelper { OutputStream os = fs.create(new Path(actionDir, LauncherMapper.ACTION_CONF_XML)); actionConf.writeXml(os); os.close(); - - Path inputDir = new Path(actionDir, "input"); - fs.mkdirs(inputDir); - Writer writer = new OutputStreamWriter(fs.create(new Path(inputDir, "dummy.txt"))); - writer.write("dummy"); - writer.close(); - - launcherConf.set("mapred.input.dir", inputDir.toString()); + launcherConf.setInputFormat(OozieLauncherInputFormat.class); launcherConf.set("mapred.output.dir", new Path(actionDir, "output").toString()); } http://git-wip-us.apache.org/repos/asf/oozie/blob/4c28591b/release-log.txt ---------------------------------------------------------------------- diff --git a/release-log.txt b/release-log.txt index 1358824..0fbdde5 100644 --- a/release-log.txt +++ b/release-log.txt @@ -1,5 +1,6 @@ -- Oozie 4.1.0 release (trunk - unreleased) +OOZIE-1713 Avoid creating dummy input file for each launcher job (puru via rohini) OOZIE-1701 TestXTestCase#testWaitFor and TestXTestCase#testBaseDir have the errors (omaliuvanchuk via rohini) OOZIE-1751 Default authentication type using env variable for oozie CLI (puru via rohini) OOZIE-1753 Update create-release-artifact script for git (rkanter) http://git-wip-us.apache.org/repos/asf/oozie/blob/4c28591b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/OozieLauncherInputFormat.java ---------------------------------------------------------------------- diff --git a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/OozieLauncherInputFormat.java b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/OozieLauncherInputFormat.java new file mode 100644 index 0000000..cf43225 --- /dev/null +++ b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/OozieLauncherInputFormat.java @@ -0,0 +1,118 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.oozie.action.hadoop; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; + +import org.apache.hadoop.io.ObjectWritable; +import org.apache.hadoop.mapred.InputFormat; +import org.apache.hadoop.mapred.InputSplit; +import org.apache.hadoop.mapred.JobConf; +import org.apache.hadoop.mapred.RecordReader; +import org.apache.hadoop.mapred.Reporter; +/** + * Dummy input format implementation of Oozie launcher jobs. It returns only one record. + */ +public class OozieLauncherInputFormat implements InputFormat<Object, Object> { + + boolean isReadingDone = false; + + public RecordReader<Object, Object> getRecordReader(InputSplit arg0, JobConf arg1, Reporter arg2) + throws IOException { + return new RecordReader<Object, Object>() { + + @Override + public void close() throws IOException { + } + + @Override + public float getProgress() throws IOException { + if (isReadingDone) { + return 1.0f; + } + else + return 0.0f; + } + + @Override + public Object createKey() { + return new ObjectWritable(); + } + + @Override + public Object createValue() { + return new ObjectWritable(); + } + + @Override + public long getPos() throws IOException { + if (isReadingDone) { + return 1; + } + else { + return 0; + } + } + + @Override + public boolean next(Object arg0, Object arg1) throws IOException { + if (isReadingDone) { + return false; + } + else { + isReadingDone = true; + return true; + } + } + + }; + } + + @Override + public InputSplit[] getSplits(JobConf arg0, int arg1) throws IOException { + return new InputSplit[] { new EmptySplit() }; + } + + /** + * Empty Split implementation. + */ + public static class EmptySplit implements InputSplit { + + @Override + public void write(DataOutput out) throws IOException { + } + + @Override + public void readFields(DataInput in) throws IOException { + } + + @Override + public long getLength() { + return 0L; + } + + @Override + public String[] getLocations() { + return new String[0]; + } + } + +}
