Author: suresh
Date: Tue Oct 23 20:23:18 2012
New Revision: 1401443
URL: http://svn.apache.org/viewvc?rev=1401443&view=rev
Log:
HADOOP-8947. Merge FileUtil and Shell changes from branch-1-win to
branch-trunk-win to enable initial test pass. Contributed by Raja Aluri, Davio
Lao, Sumadhur Reddy Bolli, Ahmed El Baz, Kanna Karanam, Chuan Liu, Ivan Mitic,
Chris Nauroth, and Bikas Saha.
Added:
hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DUHelper.java
Modified:
hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/CHANGES.branch-trunk-win.txt
hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DF.java
hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DU.java
hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java
hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java
hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java
hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java
hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java
hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestShell.java
Modified:
hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/CHANGES.branch-trunk-win.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/CHANGES.branch-trunk-win.txt?rev=1401443&r1=1401442&r2=1401443&view=diff
==============================================================================
---
hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/CHANGES.branch-trunk-win.txt
(original)
+++
hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/CHANGES.branch-trunk-win.txt
Tue Oct 23 20:23:18 2012
@@ -9,3 +9,8 @@ branch-trunk-win changes - unreleased
HADOOP-8946. winutils: compile codebase during Maven build on
branch-trunk-win. (Chris Nauroth via suresh)
+
+ HADOOP-8947. Merge FileUtil and Shell changes from branch-1-win to
+ branch-trunk-win to enable initial test pass. (Raja Aluri, Davio Lao,
+ Sumadhur Reddy Bolli, Ahmed El Baz, Kanna Karanam, Chuan Liu,
+ Ivan Mitic, Chris Nauroth, and Bikas Saha via suresh)
Modified:
hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DF.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DF.java?rev=1401443&r1=1401442&r2=1401443&view=diff
==============================================================================
---
hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DF.java
(original)
+++
hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DF.java
Tue Oct 23 20:23:18 2012
@@ -144,10 +144,22 @@ public class DF extends Shell {
}
@Override
+ protected void run() throws IOException {
+ if (WINDOWS) {
+ try {
+ this.mount = dirFile.getCanonicalPath().substring(0,2);
+ } catch (IOException e) {
+ }
+ return;
+ }
+ super.run();
+ }
+
+ @Override
protected String[] getExecString() {
// ignoring the error since the exit code it enough
- return new String[] {"bash","-c","exec 'df' '-k' '" + dirPath
- + "' 2>/dev/null"};
+ return (WINDOWS)? new String[]{"cmd", "/c", "df -k " + dirPath + " 2>nul"}:
+ new String[] {"bash","-c","exec 'df' '-k' '" + dirPath + "'
2>/dev/null"};
}
@Override
Modified:
hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DU.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DU.java?rev=1401443&r1=1401442&r2=1401443&view=diff
==============================================================================
---
hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DU.java
(original)
+++
hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DU.java
Tue Oct 23 20:23:18 2012
@@ -145,6 +145,20 @@ public class DU extends Shell {
public String getDirPath() {
return dirPath;
}
+
+
+ /**
+ * Override to hook in DUHelper class. Maybe this can be used more
+ * generally as well on Unix/Linux based systems
+ */
+ @Override
+ protected void run() throws IOException {
+ if (WINDOWS) {
+ used.set(DUHelper.getFolderUsage(dirPath));
+ return;
+ }
+ super.run();
+ }
/**
* Start the disk usage checking thread.
Added:
hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DUHelper.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DUHelper.java?rev=1401443&view=auto
==============================================================================
---
hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DUHelper.java
(added)
+++
hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DUHelper.java
Tue Oct 23 20:23:18 2012
@@ -0,0 +1,91 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs;
+
+import java.io.File;
+import org.apache.hadoop.util.Shell;
+
+public class DUHelper {
+
+ private int folderCount=0;
+ private int fileCount=0;
+ private double usage = 0;
+ private long folderSize = -1;
+
+ private DUHelper() {
+
+ }
+
+ public static long getFolderUsage(String folder) {
+ return new DUHelper().calculateFolderSize(folder);
+ }
+
+ private long calculateFolderSize(String folder) {
+ if (folder == null)
+ throw new IllegalArgumentException("folder");
+ File f = new File(folder);
+ return folderSize = getFileSize(f);
+ }
+
+ public String check(String folder) {
+ if (folder == null)
+ throw new IllegalArgumentException("folder");
+ File f = new File(folder);
+
+ folderSize = getFileSize(f);
+ usage = 1.0*(f.getTotalSpace() - f.getFreeSpace())/ f.getTotalSpace();
+ return String.format("used %d files %d disk in use %f", folderSize,
fileCount, usage);
+ }
+
+ public long getFileCount() {
+ return fileCount;
+ }
+
+ public double getUsage() {
+ return usage;
+ }
+
+ private long getFileSize(File folder) {
+
+ folderCount++;
+ //Counting the total folders
+ long foldersize = 0;
+ if (folder.isFile())
+ return folder.length();
+ File[] filelist = folder.listFiles();
+ if (filelist == null) {
+ return 0;
+ }
+ for (int i = 0; i < filelist.length; i++) {
+ if (filelist[i].isDirectory()) {
+ foldersize += getFileSize(filelist[i]);
+ } else {
+ fileCount++; //Counting the total files
+ foldersize += filelist[i].length();
+ }
+ }
+ return foldersize;
+ }
+
+ public static void main(String[] args) {
+ if (Shell.WINDOWS)
+ System.out.println("Windows: "+ DUHelper.getFolderUsage(args[0]));
+ else
+ System.out.println("Other: " + DUHelper.getFolderUsage(args[0]));
+ }
+}
\ No newline at end of file
Modified:
hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java?rev=1401443&r1=1401442&r2=1401443&view=diff
==============================================================================
---
hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java
(original)
+++
hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java
Tue Oct 23 20:23:18 2012
@@ -27,7 +27,11 @@ import java.util.zip.ZipFile;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.permission.FsAction;
+import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.io.nativeio.NativeIO;
+import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.util.Shell;
import org.apache.hadoop.util.Shell.ShellCommandExecutor;
@@ -43,6 +47,13 @@ public class FileUtil {
private static final Log LOG = LogFactory.getLog(FileUtil.class);
+ /* The error code is defined in winutils to indicate insufficient
+ * privilege to create symbolic links. This value need to keep in
+ * sync with the constant of the same name in:
+ * "src\winutils\common.h"
+ * */
+ public static final int SYMLINK_NO_PRIVILEGE = 2;
+
/**
* convert an array of FileStatus to an array of Path
*
@@ -436,11 +447,7 @@ public class FileUtil {
* @throws IOException on windows, there can be problems with the subprocess
*/
public static String makeShellPath(String filename) throws IOException {
- if (Path.WINDOWS) {
- return new CygPathCommand(filename).getResult();
- } else {
- return filename;
- }
+ return filename;
}
/**
@@ -568,21 +575,24 @@ public class FileUtil {
StringBuilder untarCommand = new StringBuilder();
boolean gzipped = inFile.toString().endsWith("gz");
if (gzipped) {
- untarCommand.append(" gzip -dc '");
+ untarCommand.append((Shell.WINDOWS) ? " gzip -dc \"" : " gzip -dc '");
untarCommand.append(FileUtil.makeShellPath(inFile));
- untarCommand.append("' | (");
+ untarCommand.append((Shell.WINDOWS) ? "\" | (" : "' | (");
}
- untarCommand.append("cd '");
+ untarCommand.append((Shell.WINDOWS) ? "cd \"" : "cd '");
untarCommand.append(FileUtil.makeShellPath(untarDir));
- untarCommand.append("' ; ");
- untarCommand.append("tar -xf ");
-
+ untarCommand.append((Shell.WINDOWS) ? "\" & " : "' ; ");
+
+ // Force the archive path as local on Windows as it can have a colon
+ untarCommand.append((Shell.WINDOWS) ? "tar --force-local -xf " : "tar -xf
");
+
if (gzipped) {
untarCommand.append(" -)");
} else {
untarCommand.append(FileUtil.makeShellPath(inFile));
}
- String[] shellCmd = { "bash", "-c", untarCommand.toString() };
+ String[] shellCmd = {(Shell.WINDOWS)?"cmd":"bash",
(Shell.WINDOWS)?"/c":"-c",
+ untarCommand.toString() };
ShellCommandExecutor shexec = new ShellCommandExecutor(shellCmd);
shexec.execute();
int exitcode = shexec.getExitCode();
@@ -606,21 +616,44 @@ public class FileUtil {
/**
* Create a soft link between a src and destination
- * only on a local disk. HDFS does not support this
+ * only on a local disk. HDFS does not support this.
+ * On Windows, when symlink creation fails due to security
+ * setting, we will log a warning. The return code in this
+ * case is 2.
* @param target the target for symlink
* @param linkname the symlink
* @return value returned by the command
*/
public static int symLink(String target, String linkname) throws IOException{
- String cmd = "ln -s " + target + " " + linkname;
- Process p = Runtime.getRuntime().exec(cmd, null);
- int returnVal = -1;
- try{
- returnVal = p.waitFor();
- } catch(InterruptedException e){
- //do nothing as of yet
+ // Run the input paths through Java's File so that they are converted to
the
+ // native OS form. FIXME: Long term fix is to expose symLink API that
+ // accepts File instead of String, as symlinks can only be created on the
+ // local FS.
+ String[] cmd = Shell.getSymlinkCommand(new File(target).getPath(),
+ new File(linkname).getPath());
+ ShellCommandExecutor shExec = new ShellCommandExecutor(cmd);
+ try {
+ shExec.execute();
+ } catch (Shell.ExitCodeException ec) {
+ int returnVal = ec.getExitCode();
+ if (Shell.WINDOWS && returnVal == SYMLINK_NO_PRIVILEGE) {
+ LOG.warn("Fail to create symbolic links on Windows. "
+ + "The default security settings in Windows disallow non-elevated "
+ + "administrators and all non-administrators from creating
symbolic links. "
+ + "This behavior can be changed in the Local Security Policy
management console");
+ } else if (returnVal != 0) {
+ LOG.warn("Command '" + StringUtils.join(" ", cmd) + "' failed "
+ + returnVal + " with: " + ec.getMessage());
+ }
+ return returnVal;
+ } catch (IOException e) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Error while create symlink " + linkname + " to " + target
+ + "." + " Exception: " + StringUtils.stringifyException(e));
+ }
+ throw e;
}
- return returnVal;
+ return shExec.getExitCode();
}
/**
@@ -644,30 +677,101 @@ public class FileUtil {
* @param recursive true, if permissions should be changed recursively
* @return the exit code from the command.
* @throws IOException
- * @throws InterruptedException
*/
public static int chmod(String filename, String perm, boolean recursive)
- throws IOException, InterruptedException {
- StringBuilder cmdBuf = new StringBuilder();
- cmdBuf.append("chmod ");
- if (recursive) {
- cmdBuf.append("-R ");
- }
- cmdBuf.append(perm).append(" ");
- cmdBuf.append(filename);
- String[] shellCmd = {"bash", "-c" ,cmdBuf.toString()};
- ShellCommandExecutor shExec = new ShellCommandExecutor(shellCmd);
+ throws IOException {
+ String [] cmd = Shell.getSetPermissionCommand(perm, recursive);
+ String[] args = new String[cmd.length + 1];
+ System.arraycopy(cmd, 0, args, 0, cmd.length);
+ args[cmd.length] = new File(filename).getPath();
+ ShellCommandExecutor shExec = new ShellCommandExecutor(args);
try {
shExec.execute();
- }catch(Exception e) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Error while changing permission : " + filename
- + " Exception: ", e);
+ }catch(IOException e) {
+ if(LOG.isDebugEnabled()) {
+ LOG.debug("Error while changing permission : " + filename
+ +" Exception: " + StringUtils.stringifyException(e));
}
}
return shExec.getExitCode();
}
+
+ /**
+ * Set permissions to the required value. Uses the java primitives instead
+ * of forking if group == other.
+ * @param f the file to change
+ * @param permission the new permissions
+ * @throws IOException
+ */
+ public static void setPermission(File f, FsPermission permission
+ ) throws IOException {
+ FsAction user = permission.getUserAction();
+ FsAction group = permission.getGroupAction();
+ FsAction other = permission.getOtherAction();
+
+ // use the native/fork if the group/other permissions are different
+ // or if the native is available or on Windows
+ if (group != other || NativeIO.isAvailable() || Shell.WINDOWS) {
+ execSetPermission(f, permission);
+ return;
+ }
+
+ boolean rv = true;
+
+ // read perms
+ rv = f.setReadable(group.implies(FsAction.READ), false);
+ checkReturnValue(rv, f, permission);
+ if (group.implies(FsAction.READ) != user.implies(FsAction.READ)) {
+ f.setReadable(user.implies(FsAction.READ), true);
+ checkReturnValue(rv, f, permission);
+ }
+
+ // write perms
+ rv = f.setWritable(group.implies(FsAction.WRITE), false);
+ checkReturnValue(rv, f, permission);
+ if (group.implies(FsAction.WRITE) != user.implies(FsAction.WRITE)) {
+ f.setWritable(user.implies(FsAction.WRITE), true);
+ checkReturnValue(rv, f, permission);
+ }
+
+ // exec perms
+ rv = f.setExecutable(group.implies(FsAction.EXECUTE), false);
+ checkReturnValue(rv, f, permission);
+ if (group.implies(FsAction.EXECUTE) != user.implies(FsAction.EXECUTE)) {
+ f.setExecutable(user.implies(FsAction.EXECUTE), true);
+ checkReturnValue(rv, f, permission);
+ }
+ }
+
+ private static void checkReturnValue(boolean rv, File p,
+ FsPermission permission
+ ) throws IOException {
+ if (!rv) {
+ throw new IOException("Failed to set permissions of path: " + p +
+ " to " +
+ String.format("%04o", permission.toShort()));
+ }
+ }
+
+ private static void execSetPermission(File f,
+ FsPermission permission
+ ) throws IOException {
+ if (NativeIO.isAvailable()) {
+ NativeIO.chmod(f.getCanonicalPath(), permission.toShort());
+ } else {
+ execCommand(f, Shell.getSetPermissionCommand(
+ String.format("%04o", permission.toShort()), false));
+ }
+ }
+ static String execCommand(File f, String... cmd) throws IOException {
+ String[] args = new String[cmd.length + 1];
+ System.arraycopy(cmd, 0, args, 0, cmd.length);
+ args[cmd.length] = f.getCanonicalPath();
+ String output = Shell.execCommand(args);
+ return output;
+ }
+
/**
* Create a tmp file for a base file.
* @param basefile the base file of the tmp
Modified:
hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java?rev=1401443&r1=1401442&r2=1401443&view=diff
==============================================================================
---
hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java
(original)
+++
hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java
Tue Oct 23 20:23:18 2012
@@ -36,7 +36,6 @@ import org.apache.hadoop.classification.
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.io.nativeio.NativeIO;
import org.apache.hadoop.util.Progressable;
import org.apache.hadoop.util.Shell;
import org.apache.hadoop.util.StringUtils;
@@ -529,8 +528,8 @@ public class RawLocalFileSystem extends
IOException e = null;
try {
StringTokenizer t = new StringTokenizer(
- execCommand(new File(getPath().toUri()),
- Shell.getGET_PERMISSION_COMMAND()));
+ FileUtil.execCommand(new File(getPath().toUri()),
+ Shell.getGetPermissionCommand()));
//expected format
//-rw------- 1 username groupname ...
String permission = t.nextToken();
@@ -539,7 +538,18 @@ public class RawLocalFileSystem extends
}
setPermission(FsPermission.valueOf(permission));
t.nextToken();
- setOwner(t.nextToken());
+
+ String owner = t.nextToken();
+ // If on windows domain, token format is DOMAIN\\user and we want to
+ // extract only the user name
+ if (Shell.WINDOWS) {
+ int i = owner.indexOf('\\');
+ if (i != -1)
+ owner = owner.substring(i + 1);
+ }
+ setOwner(owner);
+
+ // FIXME: Group names could have spaces on Windows
setGroup(t.nextToken());
} catch (Shell.ExitCodeException ioe) {
if (ioe.getExitCode() != 1) {
@@ -580,11 +590,11 @@ public class RawLocalFileSystem extends
}
if (username == null) {
- execCommand(pathToFile(p), Shell.SET_GROUP_COMMAND, groupname);
+ FileUtil.execCommand(pathToFile(p), Shell.SET_GROUP_COMMAND, groupname);
} else {
//OWNER[:[GROUP]]
String s = username + (groupname == null? "": ":" + groupname);
- execCommand(pathToFile(p), Shell.SET_OWNER_COMMAND, s);
+ FileUtil.execCommand(pathToFile(p), Shell.getSetOwnerCommand(s));
}
}
@@ -592,23 +602,7 @@ public class RawLocalFileSystem extends
* Use the command chmod to set permission.
*/
@Override
- public void setPermission(Path p, FsPermission permission)
- throws IOException {
- if (NativeIO.isAvailable()) {
- NativeIO.chmod(pathToFile(p).getCanonicalPath(),
- permission.toShort());
- } else {
- execCommand(pathToFile(p), Shell.SET_PERMISSION_COMMAND,
- String.format("%05o", permission.toShort()));
- }
- }
-
- private static String execCommand(File f, String... cmd) throws IOException {
- String[] args = new String[cmd.length + 1];
- System.arraycopy(cmd, 0, args, 0, cmd.length);
- args[cmd.length] = FileUtil.makeShellPath(f, true);
- String output = Shell.execCommand(args);
- return output;
+ public void setPermission(Path p, FsPermission permission) throws
IOException {
+ FileUtil.setPermission(pathToFile(p), permission);
}
-
-}
\ No newline at end of file
+}
Modified:
hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java?rev=1401443&r1=1401442&r2=1401443&view=diff
==============================================================================
---
hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java
(original)
+++
hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java
Tue Oct 23 20:23:18 2012
@@ -44,36 +44,76 @@ abstract public class Shell {
public static final Log LOG = LogFactory.getLog(Shell.class);
+ /** a Windows utility to emulate Unix commands */
+ public static final String WINUTILS = System.getenv("HADOOP_HOME")
+ + "\\bin\\winutils";
+
/** a Unix command to get the current user's name */
public final static String USER_NAME_COMMAND = "whoami";
+
+ /** Windows CreateProcess synchronization object */
+ public static final Object WindowsProcessLaunchLock = new Object();
+
/** a Unix command to get the current user's groups list */
public static String[] getGroupsCommand() {
- return new String[]{"bash", "-c", "groups"};
+ return (WINDOWS)? new String[]{"cmd", "/c", "groups"}
+ : new String[]{"bash", "-c", "groups"};
}
+
/** a Unix command to get a given user's groups list */
public static String[] getGroupsForUserCommand(final String user) {
//'groups username' command return is non-consistent across different
unixes
- return new String [] {"bash", "-c", "id -Gn " + user};
+ return (WINDOWS)? new String[] { WINUTILS, "groups", user}
+ : new String [] {"bash", "-c", "id -Gn " + user};
}
+
/** a Unix command to get a given netgroup's user list */
public static String[] getUsersForNetgroupCommand(final String netgroup) {
//'groups username' command return is non-consistent across different
unixes
- return new String [] {"bash", "-c", "getent netgroup " + netgroup};
+ return (WINDOWS)? new String [] {"cmd", "/c", "getent netgroup " +
netgroup}
+ : new String [] {"bash", "-c", "getent netgroup " +
netgroup};
+ }
+
+ /** Return a command to get permission information. */
+ public static String[] getGetPermissionCommand() {
+ return (WINDOWS) ? new String[] { WINUTILS, "ls" }
+ : new String[] { "/bin/ls", "-ld" };
+ }
+
+ /** Return a command to set permission */
+ public static String[] getSetPermissionCommand(String perm, boolean
recursive) {
+ if (recursive) {
+ return (WINDOWS) ? new String[] { WINUTILS, "chmod", "-R", perm }
+ : new String[] { "chmod", "-R", perm };
+ } else {
+ return (WINDOWS) ? new String[] { WINUTILS, "chmod", perm }
+ : new String[] { "chmod", perm };
+ }
}
+
+ /** Return a command to set owner */
+ public static String[] getSetOwnerCommand(String owner) {
+ return (WINDOWS) ? new String[] { WINUTILS, "chown", owner }
+ : new String[] { "chown", owner };
+ }
+
+ /** Return a command to create symbolic links */
+ public static String[] getSymlinkCommand(String target, String link) {
+ return WINDOWS ? new String[] { WINUTILS, "symlink", link, target }
+ : new String[] { "ln", "-s", target, link };
+ }
+
/** a Unix command to set permission */
public static final String SET_PERMISSION_COMMAND = "chmod";
/** a Unix command to set owner */
public static final String SET_OWNER_COMMAND = "chown";
+
+ /** a Unix command to set the change user's groups list */
public static final String SET_GROUP_COMMAND = "chgrp";
/** a Unix command to create a link */
public static final String LINK_COMMAND = "ln";
/** a Unix command to get a link target */
public static final String READ_LINK_COMMAND = "readlink";
- /** Return a Unix command to get permission information. */
- public static String[] getGET_PERMISSION_COMMAND() {
- //force /bin/ls, except on windows.
- return new String[] {(WINDOWS ? "ls" : "/bin/ls"), "-ld"};
- }
/**Time after which the executing script would be timedout*/
protected long timeOutInterval = 0L;
@@ -144,7 +184,19 @@ abstract public class Shell {
builder.directory(this.dir);
}
- process = builder.start();
+ if (Shell.WINDOWS) {
+ synchronized (WindowsProcessLaunchLock) {
+ // To workaround the race condition issue with child processes
+ // inheriting unintended handles during process launch that can
+ // lead to hangs on reading output and error streams, we
+ // serialize process creation. More info available at:
+ // http://support.microsoft.com/kb/315939
+ process = builder.start();
+ }
+ } else {
+ process = builder.start();
+ }
+
if (timeOutInterval > 0) {
timeOutTimer = new Timer("Shell command timeout");
timeoutTimerTask = new ShellTimeoutTimerTask(
Modified:
hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java?rev=1401443&r1=1401442&r2=1401443&view=diff
==============================================================================
---
hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java
(original)
+++
hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java
Tue Oct 23 20:23:18 2012
@@ -779,6 +779,28 @@ public class StringUtils {
}
/**
+ * Concatenates strings, using a separator.
+ *
+ * @param separator to join with
+ * @param strings to join
+ * @return the joined string
+ */
+ public static String join(CharSequence separator, String[] strings) {
+ // Ideally we don't have to duplicate the code here if array is iterable.
+ StringBuilder sb = new StringBuilder();
+ boolean first = true;
+ for (String s : strings) {
+ if (first) {
+ first = false;
+ } else {
+ sb.append(separator);
+ }
+ sb.append(s);
+ }
+ return sb.toString();
+ }
+
+ /**
* Convert SOME_STUFF to SomeStuff
*
* @param s input string
Modified:
hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java?rev=1401443&r1=1401442&r2=1401443&view=diff
==============================================================================
---
hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java
(original)
+++
hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java
Tue Oct 23 20:23:18 2012
@@ -19,6 +19,8 @@ package org.apache.hadoop.fs;
import java.io.BufferedReader;
import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
import java.io.FileReader;
import java.io.IOException;
import java.io.PrintWriter;
@@ -29,6 +31,7 @@ import java.util.List;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.util.Shell;
import org.junit.After;
import org.junit.Assert;
import org.junit.Test;
@@ -336,6 +339,10 @@ public class TestFileUtil {
@Test
public void testFailFullyDelete() throws IOException {
+ if(Shell.WINDOWS) {
+ // windows Dir.setWritable(false) does not work for directories
+ return;
+ }
LOG.info("Running test to verify failure of fullyDelete()");
setupDirsAndNonWritablePermissions();
boolean ret = FileUtil.fullyDelete(new MyFile(del));
@@ -402,6 +409,10 @@ public class TestFileUtil {
@Test
public void testFailFullyDeleteContents() throws IOException {
+ if(Shell.WINDOWS) {
+ // windows Dir.setWritable(false) does not work for directories
+ return;
+ }
LOG.info("Running test to verify failure of fullyDeleteContents()");
setupDirsAndNonWritablePermissions();
boolean ret = FileUtil.fullyDeleteContents(new MyFile(del));
@@ -477,4 +488,36 @@ public class TestFileUtil {
long expected = 2 * (3 + System.getProperty("line.separator").length());
Assert.assertEquals(expected, du);
}
+
+ @Test
+ public void testSymlink() throws Exception {
+ Assert.assertFalse(del.exists());
+ del.mkdirs();
+
+ byte[] data = "testSymLink".getBytes();
+
+ File file = new File(del, FILE);
+ File link = new File(del, "_link");
+
+ //write some data to the file
+ FileOutputStream os = new FileOutputStream(file);
+ os.write(data);
+ os.close();
+
+ //create the symlink
+ FileUtil.symLink(file.getAbsolutePath(), link.getAbsolutePath());
+
+ //ensure that symlink length is correctly reported by Java
+ Assert.assertEquals(data.length, file.length());
+ Assert.assertEquals(Shell.WINDOWS ? 0 : data.length, link.length());
+
+ //ensure that we can read from link.
+ FileInputStream in = new FileInputStream(link);
+ long len = 0;
+ while (in.read() > 0) {
+ len++;
+ }
+ in.close();
+ Assert.assertEquals(data.length, len);
+ }
}
Modified:
hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestShell.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestShell.java?rev=1401443&r1=1401442&r2=1401443&view=diff
==============================================================================
---
hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestShell.java
(original)
+++
hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestShell.java
Tue Oct 23 20:23:18 2012
@@ -81,6 +81,10 @@ public class TestShell extends TestCase
}
public void testShellCommandTimeout() throws Throwable {
+ if(Shell.WINDOWS) {
+ // setExecutable does not work on Windows
+ return;
+ }
String rootDir = new File(System.getProperty(
"test.build.data", "/tmp")).getAbsolutePath();
File shellFile = new File(rootDir, "timeout.sh");
