Repository: hadoop Updated Branches: refs/heads/branch-2.6 3f200fddf -> 0a36b6808
Missing files from the previous commit of HADOOP-11007 Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0a36b680 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0a36b680 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0a36b680 Branch: refs/heads/branch-2.6 Commit: 0a36b6808fba84b4cc83d9e1c3a9a27b236c5054 Parents: 3f200fd Author: Kihwal Lee <kih...@apache.org> Authored: Tue Oct 7 16:30:53 2014 -0500 Committer: Kihwal Lee <kih...@apache.org> Committed: Tue Oct 7 16:32:55 2014 -0500 ---------------------------------------------------------------------- hadoop-tools/hadoop-ant/pom.xml | 56 +++++ .../java/org/apache/hadoop/ant/DfsTask.java | 210 +++++++++++++++++++ .../ant/condition/DfsBaseConditional.java | 68 ++++++ .../apache/hadoop/ant/condition/DfsExists.java | 24 +++ .../apache/hadoop/ant/condition/DfsIsDir.java | 24 +++ .../apache/hadoop/ant/condition/DfsZeroLen.java | 24 +++ .../resources/org/apache/hadoop/ant/antlib.xml | 29 +++ 7 files changed, 435 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a36b680/hadoop-tools/hadoop-ant/pom.xml ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-ant/pom.xml b/hadoop-tools/hadoop-ant/pom.xml new file mode 100644 index 0000000..197b527 --- /dev/null +++ b/hadoop-tools/hadoop-ant/pom.xml @@ -0,0 +1,56 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. See accompanying LICENSE file. +--> +<project xmlns="http://maven.apache.org/POM/4.0.0"; + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 + http://maven.apache.org/xsd/maven-4.0.0.xsd";> + <modelVersion>4.0.0</modelVersion> + <parent> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-project</artifactId> + <version>3.0.0-SNAPSHOT</version> + <relativePath>../../hadoop-project</relativePath> + </parent> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-ant</artifactId> + <version>3.0.0-SNAPSHOT</version> + <description>Apache Hadoop Ant Tasks</description> + <name>Apache Hadoop Ant Tasks</name> + <packaging>jar</packaging> + + <dependencies> + <dependency> + <groupId>org.apache.ant</groupId> + <artifactId>ant</artifactId> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-annotations</artifactId> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-common</artifactId> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-hdfs</artifactId> + <scope>provided</scope> + </dependency> + </dependencies> + +</project> http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a36b680/hadoop-tools/hadoop-ant/src/main/java/org/apache/hadoop/ant/DfsTask.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-ant/src/main/java/org/apache/hadoop/ant/DfsTask.java b/hadoop-tools/hadoop-ant/src/main/java/org/apache/hadoop/ant/DfsTask.java new file mode 100644 index 0000000..78cb360 --- /dev/null +++ b/hadoop-tools/hadoop-ant/src/main/java/org/apache/hadoop/ant/DfsTask.java @@ -0,0 +1,210 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ant; + +import java.io.ByteArrayOutputStream; +import java.io.OutputStream; +import java.io.PrintStream; +import java.security.AccessController; +import java.security.PrivilegedAction; +import java.util.LinkedList; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FsShell; +import org.apache.tools.ant.AntClassLoader; +import org.apache.tools.ant.BuildException; +import org.apache.tools.ant.Task; +import org.apache.hadoop.util.ToolRunner; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.hdfs.HdfsConfiguration; + +/** + * {@link org.apache.hadoop.fs.FsShell FsShell} wrapper for ant Task. + */ +@InterfaceAudience.Private +public class DfsTask extends Task { + + /** + * Default sink for {@link java.lang.System.out System.out} + * and {@link java.lang.System.err System.err}. + */ + private static final OutputStream nullOut = new OutputStream() { + public void write(int b) { /* ignore */ } + public String toString() { return ""; } + }; + private static final FsShell shell = new FsShell(); + + protected AntClassLoader confloader; + protected OutputStream out = nullOut; + protected OutputStream err = nullOut; + + // set by ant + protected String cmd; + protected final LinkedList<String> argv = new LinkedList<String>(); + protected String outprop; + protected String errprop; + protected boolean failonerror = true; + + // saved ant context + private PrintStream antOut; + private PrintStream antErr; + + /** + * Sets the command to run in {@link org.apache.hadoop.fs.FsShell FsShell}. + * @param cmd A valid command to FsShell, sans "-". + */ + public void setCmd(String cmd) { + this.cmd = "-" + cmd.trim(); + } + + /** + * Sets the argument list from a String of comma-separated values. + * @param args A String of comma-separated arguments to FsShell. + */ + public void setArgs(String args) { + for (String s : args.trim().split("\\s*,\\s*")) + argv.add(s); + } + + /** + * Sets the property into which System.out will be written. + * @param outprop The name of the property into which System.out is written. + * If the property is defined before this task is executed, it will not be updated. + */ + public void setOut(String outprop) { + this.outprop = outprop; + out = new ByteArrayOutputStream(); + if (outprop.equals(errprop)) + err = out; + } + + /** + * Sets the property into which System.err will be written. If this property + * has the same name as the property for System.out, the two will be interlaced. + * @param errprop The name of the property into which System.err is written. + * If the property is defined before this task is executed, it will not be updated. + */ + public void setErr(String errprop) { + this.errprop = errprop; + err = (errprop.equals(outprop)) ? err = out : new ByteArrayOutputStream(); + } + + /** + * Sets the path for the parent-last ClassLoader, intended to be used for + * {@link org.apache.hadoop.conf.Configuration Configuration}. + * @param confpath The path to search for resources, classes, etc. before + * parent ClassLoaders. + */ + public void setConf(String confpath) { + confloader = AccessController.doPrivileged( + new PrivilegedAction<AntClassLoader>() { + @Override + public AntClassLoader run() { + return new AntClassLoader(getClass().getClassLoader(), false); + } + }); + confloader.setProject(getProject()); + if (null != confpath) + confloader.addPathElement(confpath); + } + + /** + * Sets a property controlling whether or not a + * {@link org.apache.tools.ant.BuildException BuildException} will be thrown + * if the command returns a value less than zero or throws an exception. + * @param failonerror If true, throw a BuildException on error. + */ + public void setFailonerror(boolean failonerror) { + this.failonerror = failonerror; + } + + /** + * Save the current values of System.out, System.err and configure output + * streams for FsShell. + */ + protected void pushContext() { + antOut = System.out; + antErr = System.err; + System.setOut(new PrintStream(out)); + System.setErr(out == err ? System.out : new PrintStream(err)); + } + + /** + * Create the appropriate output properties with their respective output, + * restore System.out, System.err and release any resources from created + * ClassLoaders to aid garbage collection. + */ + protected void popContext() { + // write output to property, if applicable + if (outprop != null && !System.out.checkError()) + getProject().setNewProperty(outprop, out.toString()); + if (out != err && errprop != null && !System.err.checkError()) + getProject().setNewProperty(errprop, err.toString()); + + System.setErr(antErr); + System.setOut(antOut); + confloader.cleanup(); + confloader.setParent(null); + } + + // in case DfsTask is overridden + protected int postCmd(int exit_code) { + if ("-test".equals(cmd) && exit_code != 0) + outprop = null; + return exit_code; + } + + /** + * Invoke {@link org.apache.hadoop.fs.FsShell#doMain FsShell.doMain} after a + * few cursory checks of the configuration. + */ + public void execute() throws BuildException { + if (null == cmd) + throw new BuildException("Missing command (cmd) argument"); + argv.add(0, cmd); + + if (null == confloader) { + setConf(getProject().getProperty("hadoop.conf.dir")); + } + + int exit_code = 0; + try { + pushContext(); + + Configuration conf = new HdfsConfiguration(); + conf.setClassLoader(confloader); + exit_code = ToolRunner.run(conf, shell, + argv.toArray(new String[argv.size()])); + exit_code = postCmd(exit_code); + + if (0 > exit_code) { + StringBuilder msg = new StringBuilder(); + for (String s : argv) + msg.append(s + " "); + msg.append("failed: " + exit_code); + throw new Exception(msg.toString()); + } + } catch (Exception e) { + if (failonerror) + throw new BuildException(e); + } finally { + popContext(); + } + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a36b680/hadoop-tools/hadoop-ant/src/main/java/org/apache/hadoop/ant/condition/DfsBaseConditional.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-ant/src/main/java/org/apache/hadoop/ant/condition/DfsBaseConditional.java b/hadoop-tools/hadoop-ant/src/main/java/org/apache/hadoop/ant/condition/DfsBaseConditional.java new file mode 100644 index 0000000..33cf52b --- /dev/null +++ b/hadoop-tools/hadoop-ant/src/main/java/org/apache/hadoop/ant/condition/DfsBaseConditional.java @@ -0,0 +1,68 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ant.condition; + +import org.apache.tools.ant.taskdefs.condition.Condition; + +/** + * This wrapper around {@link org.apache.hadoop.ant.DfsTask} implements the + * Ant >1.5 + * {@link org.apache.tools.ant.taskdefs.condition.Condition Condition} + * interface for HDFS tests. So one can test conditions like this: + * {@code + * <condition property="precond"> + * <and> + * <hadoop:exists file="fileA" /> + * <hadoop:exists file="fileB" /> + * <hadoop:sizezero file="fileB" /> + * </and> + * </condition> + * } + * This will define the property precond if fileA exists and fileB has zero + * length. + */ +public abstract class DfsBaseConditional extends org.apache.hadoop.ant.DfsTask + implements Condition { + + protected boolean result; + String file; + + private void initArgs() { + setCmd("test"); + setArgs("-" + getFlag() + "," + file); + } + + public void setFile(String file) { + this.file = file; + } + + protected abstract char getFlag(); + + protected int postCmd(int exit_code) { + exit_code = super.postCmd(exit_code); + result = exit_code == 0; + return exit_code; + } + + public boolean eval() { + initArgs(); + execute(); + return result; + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a36b680/hadoop-tools/hadoop-ant/src/main/java/org/apache/hadoop/ant/condition/DfsExists.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-ant/src/main/java/org/apache/hadoop/ant/condition/DfsExists.java b/hadoop-tools/hadoop-ant/src/main/java/org/apache/hadoop/ant/condition/DfsExists.java new file mode 100644 index 0000000..8fbe872 --- /dev/null +++ b/hadoop-tools/hadoop-ant/src/main/java/org/apache/hadoop/ant/condition/DfsExists.java @@ -0,0 +1,24 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ant.condition; + +public class DfsExists extends DfsBaseConditional { + protected final char flag = 'e'; + protected char getFlag() { return flag; } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a36b680/hadoop-tools/hadoop-ant/src/main/java/org/apache/hadoop/ant/condition/DfsIsDir.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-ant/src/main/java/org/apache/hadoop/ant/condition/DfsIsDir.java b/hadoop-tools/hadoop-ant/src/main/java/org/apache/hadoop/ant/condition/DfsIsDir.java new file mode 100644 index 0000000..e146d06 --- /dev/null +++ b/hadoop-tools/hadoop-ant/src/main/java/org/apache/hadoop/ant/condition/DfsIsDir.java @@ -0,0 +1,24 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ant.condition; + +public class DfsIsDir extends DfsBaseConditional { + protected final char flag = 'd'; + protected char getFlag() { return flag; } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a36b680/hadoop-tools/hadoop-ant/src/main/java/org/apache/hadoop/ant/condition/DfsZeroLen.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-ant/src/main/java/org/apache/hadoop/ant/condition/DfsZeroLen.java b/hadoop-tools/hadoop-ant/src/main/java/org/apache/hadoop/ant/condition/DfsZeroLen.java new file mode 100644 index 0000000..caac378 --- /dev/null +++ b/hadoop-tools/hadoop-ant/src/main/java/org/apache/hadoop/ant/condition/DfsZeroLen.java @@ -0,0 +1,24 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ant.condition; + +public class DfsZeroLen extends DfsBaseConditional { + protected final char flag = 'z'; + protected char getFlag() { return flag; } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a36b680/hadoop-tools/hadoop-ant/src/main/resources/org/apache/hadoop/ant/antlib.xml ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-ant/src/main/resources/org/apache/hadoop/ant/antlib.xml b/hadoop-tools/hadoop-ant/src/main/resources/org/apache/hadoop/ant/antlib.xml new file mode 100644 index 0000000..968b899 --- /dev/null +++ b/hadoop-tools/hadoop-ant/src/main/resources/org/apache/hadoop/ant/antlib.xml @@ -0,0 +1,29 @@ +<?xml version="1.0"?> + +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +<antlib> + <taskdef name="hdfs" + classname="org.apache.hadoop.ant.DfsTask" /> + <taskdef name="exists" + classname="org.apache.hadoop.ant.condition.DfsExists" /> + <taskdef name="isdir" + classname="org.apache.hadoop.ant.condition.DfsIsDir" /> + <taskdef name="sizezero" + classname="org.apache.hadoop.ant.condition.DfsZeroLen" /> +</antlib>
