Repository: hadoop Updated Branches: refs/heads/HADOOP-12111 21e21b990 -> a6692bee0
HADOOP-7824. NativeIO.java flags and identifiers must be set correctly for each platform, not hardcoded to their Linux values (Martin Walsh via Colin P. McCabe) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/21d10ccc Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/21d10ccc Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/21d10ccc Branch: refs/heads/HADOOP-12111 Commit: 21d10ccc6e463cf250414264c78acb4a6e7c83e3 Parents: a890a31 Author: Colin Patrick Mccabe <cmcc...@cloudera.com> Authored: Fri Jul 31 14:55:14 2015 -0700 Committer: Colin Patrick Mccabe <cmcc...@cloudera.com> Committed: Fri Jul 31 15:01:34 2015 -0700 ---------------------------------------------------------------------- hadoop-common-project/hadoop-common/CHANGES.txt | 4 + .../org/apache/hadoop/io/ReadaheadPool.java | 4 +- .../org/apache/hadoop/io/nativeio/NativeIO.java | 83 +++++------ .../org/apache/hadoop/io/nativeio/NativeIO.c | 147 ++++++++++++------- .../apache/hadoop/io/nativeio/TestNativeIO.java | 90 ++++++++---- .../hdfs/server/datanode/BlockReceiver.java | 14 +- .../hdfs/server/datanode/BlockSender.java | 11 +- .../server/datanode/TestCachingStrategy.java | 5 +- .../hadoop/mapred/FadvisedChunkedFile.java | 5 +- .../hadoop/mapred/FadvisedFileRegion.java | 6 +- 10 files changed, 232 insertions(+), 137 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/21d10ccc/hadoop-common-project/hadoop-common/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 5020e91..675902d 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -1033,6 +1033,10 @@ Release 2.8.0 - UNRELEASED HADOOP-10945. 4-digit octal umask permissions throws a parse error (Chang Li via jlowe) + HADOOP-7824. NativeIO.java flags and identifiers must be set correctly for + each platform, not hardcoded to their Linux values (Martin Walsh via Colin + P. McCabe) + Release 2.7.2 - UNRELEASED INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/21d10ccc/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/ReadaheadPool.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/ReadaheadPool.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/ReadaheadPool.java index 18099db..a8c0690 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/ReadaheadPool.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/ReadaheadPool.java @@ -29,6 +29,8 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.io.nativeio.NativeIO; +import static org.apache.hadoop.io.nativeio.NativeIO.POSIX.POSIX_FADV_WILLNEED; + import com.google.common.base.Preconditions; import com.google.common.util.concurrent.ThreadFactoryBuilder; @@ -204,7 +206,7 @@ public class ReadaheadPool { // other FD, which may be wasted work, but won't cause a problem. try { NativeIO.POSIX.getCacheManipulator().posixFadviseIfPossible(identifier, - fd, off, len, NativeIO.POSIX.POSIX_FADV_WILLNEED); + fd, off, len, POSIX_FADV_WILLNEED); } catch (IOException ioe) { if (canceled) { // no big deal - the reader canceled the request and closed http://git-wip-us.apache.org/repos/asf/hadoop/blob/21d10ccc/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/NativeIO.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/NativeIO.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/NativeIO.java index 77a40ea..a123f18 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/NativeIO.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/NativeIO.java @@ -56,51 +56,54 @@ import com.google.common.annotations.VisibleForTesting; @InterfaceStability.Unstable public class NativeIO { public static class POSIX { - // Flags for open() call from bits/fcntl.h - public static final int O_RDONLY = 00; - public static final int O_WRONLY = 01; - public static final int O_RDWR = 02; - public static final int O_CREAT = 0100; - public static final int O_EXCL = 0200; - public static final int O_NOCTTY = 0400; - public static final int O_TRUNC = 01000; - public static final int O_APPEND = 02000; - public static final int O_NONBLOCK = 04000; - public static final int O_SYNC = 010000; - - // Flags for posix_fadvise() from bits/fcntl.h + // Flags for open() call from bits/fcntl.h - Set by JNI + public static int O_RDONLY = -1; + public static int O_WRONLY = -1; + public static int O_RDWR = -1; + public static int O_CREAT = -1; + public static int O_EXCL = -1; + public static int O_NOCTTY = -1; + public static int O_TRUNC = -1; + public static int O_APPEND = -1; + public static int O_NONBLOCK = -1; + public static int O_SYNC = -1; + + // Flags for posix_fadvise() from bits/fcntl.h - Set by JNI /* No further special treatment. */ - public static final int POSIX_FADV_NORMAL = 0; + public static int POSIX_FADV_NORMAL = -1; /* Expect random page references. */ - public static final int POSIX_FADV_RANDOM = 1; + public static int POSIX_FADV_RANDOM = -1; /* Expect sequential page references. */ - public static final int POSIX_FADV_SEQUENTIAL = 2; + public static int POSIX_FADV_SEQUENTIAL = -1; /* Will need these pages. */ - public static final int POSIX_FADV_WILLNEED = 3; + public static int POSIX_FADV_WILLNEED = -1; /* Don't need these pages. */ - public static final int POSIX_FADV_DONTNEED = 4; + public static int POSIX_FADV_DONTNEED = -1; /* Data will be accessed once. */ - public static final int POSIX_FADV_NOREUSE = 5; + public static int POSIX_FADV_NOREUSE = -1; + // Updated by JNI when supported by glibc. Leave defaults in case kernel + // supports sync_file_range, but glibc does not. /* Wait upon writeout of all pages in the range before performing the write. */ - public static final int SYNC_FILE_RANGE_WAIT_BEFORE = 1; + public static int SYNC_FILE_RANGE_WAIT_BEFORE = 1; /* Initiate writeout of all those dirty pages in the range which are not presently under writeback. */ - public static final int SYNC_FILE_RANGE_WRITE = 2; - + public static int SYNC_FILE_RANGE_WRITE = 2; /* Wait upon writeout of all pages in the range after performing the write. */ - public static final int SYNC_FILE_RANGE_WAIT_AFTER = 4; + public static int SYNC_FILE_RANGE_WAIT_AFTER = 4; private static final Log LOG = LogFactory.getLog(NativeIO.class); + // Set to true via JNI if possible + public static boolean fadvisePossible = false; + private static boolean nativeLoaded = false; - private static boolean fadvisePossible = true; private static boolean syncFileRangePossible = true; static final String WORKAROUND_NON_THREADSAFE_CALLS_KEY = @@ -262,8 +265,6 @@ public class NativeIO { if (nativeLoaded && fadvisePossible) { try { posix_fadvise(fd, offset, len, flags); - } catch (UnsupportedOperationException uoe) { - fadvisePossible = false; } catch (UnsatisfiedLinkError ule) { fadvisePossible = false; } @@ -344,21 +345,21 @@ public class NativeIO { private String owner, group; private int mode; - // Mode constants - public static final int S_IFMT = 0170000; /* type of file */ - public static final int S_IFIFO = 0010000; /* named pipe (fifo) */ - public static final int S_IFCHR = 0020000; /* character special */ - public static final int S_IFDIR = 0040000; /* directory */ - public static final int S_IFBLK = 0060000; /* block special */ - public static final int S_IFREG = 0100000; /* regular */ - public static final int S_IFLNK = 0120000; /* symbolic link */ - public static final int S_IFSOCK = 0140000; /* socket */ - public static final int S_ISUID = 0004000; /* set user id on execution */ - public static final int S_ISGID = 0002000; /* set group id on execution */ - public static final int S_ISVTX = 0001000; /* save swapped text even after use */ - public static final int S_IRUSR = 0000400; /* read permission, owner */ - public static final int S_IWUSR = 0000200; /* write permission, owner */ - public static final int S_IXUSR = 0000100; /* execute/search permission, owner */ + // Mode constants - Set by JNI + public static int S_IFMT = -1; /* type of file */ + public static int S_IFIFO = -1; /* named pipe (fifo) */ + public static int S_IFCHR = -1; /* character special */ + public static int S_IFDIR = -1; /* directory */ + public static int S_IFBLK = -1; /* block special */ + public static int S_IFREG = -1; /* regular */ + public static int S_IFLNK = -1; /* symbolic link */ + public static int S_IFSOCK = -1; /* socket */ + public static int S_ISUID = -1; /* set user id on execution */ + public static int S_ISGID = -1; /* set group id on execution */ + public static int S_ISVTX = -1; /* save swapped text even after use */ + public static int S_IRUSR = -1; /* read permission, owner */ + public static int S_IWUSR = -1; /* write permission, owner */ + public static int S_IXUSR = -1; /* execute/search permission, owner */ Stat(int ownerId, int groupId, int mode) { this.ownerId = ownerId; http://git-wip-us.apache.org/repos/asf/hadoop/blob/21d10ccc/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/NativeIO.c ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/NativeIO.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/NativeIO.c index bc78ab2..a716a02 100644 --- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/NativeIO.c +++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/NativeIO.c @@ -58,6 +58,15 @@ #define MMAP_PROT_WRITE org_apache_hadoop_io_nativeio_NativeIO_POSIX_MMAP_PROT_WRITE #define MMAP_PROT_EXEC org_apache_hadoop_io_nativeio_NativeIO_POSIX_MMAP_PROT_EXEC +#define NATIVE_IO_POSIX_CLASS "org/apache/hadoop/io/nativeio/NativeIO$POSIX" +#define NATIVE_IO_STAT_CLASS "org/apache/hadoop/io/nativeio/NativeIO$POSIX$Stat" + +#define SET_INT_OR_RETURN(E, C, F) \ + { \ + setStaticInt(E, C, #F, F); \ + if ((*E)->ExceptionCheck(E)) return; \ + } + // the NativeIO$POSIX$Stat inner class and its constructor static jclass stat_clazz; static jmethodID stat_ctor; @@ -101,12 +110,90 @@ static int workaround_non_threadsafe_calls(JNIEnv *env, jclass clazz) { return result; } +/** + * Sets a static boolean field to the specified value. + */ +static void setStaticBoolean(JNIEnv *env, jclass clazz, char *field, + jboolean val) { + jfieldID fid = (*env)->GetStaticFieldID(env, clazz, field, "Z"); + if (fid != NULL) { + (*env)->SetStaticBooleanField(env, clazz, fid, val); + } +} + +/** + * Sets a static int field to the specified value. + */ +static void setStaticInt(JNIEnv *env, jclass clazz, char *field, + jint val) { + jfieldID fid = (*env)->GetStaticFieldID(env, clazz, field, "I"); + if (fid != NULL) { + (*env)->SetStaticIntField(env, clazz, fid, val); + } +} + +/** + * Initialises a list of java constants that are platform specific. + * These are only initialized in UNIX. + * Any exceptions that occur will be dealt at the level above. +**/ +static void consts_init(JNIEnv *env) { + jclass clazz = (*env)->FindClass(env, NATIVE_IO_POSIX_CLASS); + if (clazz == NULL) { + return; // exception has been raised + } + SET_INT_OR_RETURN(env, clazz, O_RDONLY); + SET_INT_OR_RETURN(env, clazz, O_WRONLY); + SET_INT_OR_RETURN(env, clazz, O_RDWR); + SET_INT_OR_RETURN(env, clazz, O_CREAT); + SET_INT_OR_RETURN(env, clazz, O_EXCL); + SET_INT_OR_RETURN(env, clazz, O_NOCTTY); + SET_INT_OR_RETURN(env, clazz, O_TRUNC); + SET_INT_OR_RETURN(env, clazz, O_APPEND); + SET_INT_OR_RETURN(env, clazz, O_NONBLOCK); + SET_INT_OR_RETURN(env, clazz, O_SYNC); +#ifdef HAVE_POSIX_FADVISE + setStaticBoolean(env, clazz, "fadvisePossible", JNI_TRUE); + SET_INT_OR_RETURN(env, clazz, POSIX_FADV_NORMAL); + SET_INT_OR_RETURN(env, clazz, POSIX_FADV_RANDOM); + SET_INT_OR_RETURN(env, clazz, POSIX_FADV_SEQUENTIAL); + SET_INT_OR_RETURN(env, clazz, POSIX_FADV_WILLNEED); + SET_INT_OR_RETURN(env, clazz, POSIX_FADV_DONTNEED); + SET_INT_OR_RETURN(env, clazz, POSIX_FADV_NOREUSE); +#else + setStaticBoolean(env, clazz, "fadvisePossible", JNI_FALSE); +#endif +#ifdef HAVE_SYNC_FILE_RANGE + SET_INT_OR_RETURN(env, clazz, SYNC_FILE_RANGE_WAIT_BEFORE); + SET_INT_OR_RETURN(env, clazz, SYNC_FILE_RANGE_WRITE); + SET_INT_OR_RETURN(env, clazz, SYNC_FILE_RANGE_WAIT_AFTER); +#endif + clazz = (*env)->FindClass(env, NATIVE_IO_STAT_CLASS); + if (clazz == NULL) { + return; // exception has been raised + } + SET_INT_OR_RETURN(env, clazz, S_IFMT); + SET_INT_OR_RETURN(env, clazz, S_IFIFO); + SET_INT_OR_RETURN(env, clazz, S_IFCHR); + SET_INT_OR_RETURN(env, clazz, S_IFDIR); + SET_INT_OR_RETURN(env, clazz, S_IFBLK); + SET_INT_OR_RETURN(env, clazz, S_IFREG); + SET_INT_OR_RETURN(env, clazz, S_IFLNK); + SET_INT_OR_RETURN(env, clazz, S_IFSOCK); + SET_INT_OR_RETURN(env, clazz, S_ISUID); + SET_INT_OR_RETURN(env, clazz, S_ISGID); + SET_INT_OR_RETURN(env, clazz, S_ISVTX); + SET_INT_OR_RETURN(env, clazz, S_IRUSR); + SET_INT_OR_RETURN(env, clazz, S_IWUSR); + SET_INT_OR_RETURN(env, clazz, S_IXUSR); +} + static void stat_init(JNIEnv *env, jclass nativeio_class) { jclass clazz = NULL; jclass obj_class = NULL; jmethodID obj_ctor = NULL; // Init Stat - clazz = (*env)->FindClass(env, "org/apache/hadoop/io/nativeio/NativeIO$POSIX$Stat"); + clazz = (*env)->FindClass(env, NATIVE_IO_STAT_CLASS); if (!clazz) { return; // exception has been raised } @@ -181,39 +268,6 @@ static void nioe_deinit(JNIEnv *env) { } /* - * Compatibility mapping for fadvise flags. Return the proper value from fnctl.h. - * If the value is not known, return the argument unchanged. - */ -static int map_fadvise_flag(jint flag) { -#ifdef HAVE_POSIX_FADVISE - switch(flag) { - case org_apache_hadoop_io_nativeio_NativeIO_POSIX_POSIX_FADV_NORMAL: - return POSIX_FADV_NORMAL; - break; - case org_apache_hadoop_io_nativeio_NativeIO_POSIX_POSIX_FADV_RANDOM: - return POSIX_FADV_RANDOM; - break; - case org_apache_hadoop_io_nativeio_NativeIO_POSIX_POSIX_FADV_SEQUENTIAL: - return POSIX_FADV_SEQUENTIAL; - break; - case org_apache_hadoop_io_nativeio_NativeIO_POSIX_POSIX_FADV_WILLNEED: - return POSIX_FADV_WILLNEED; - break; - case org_apache_hadoop_io_nativeio_NativeIO_POSIX_POSIX_FADV_DONTNEED: - return POSIX_FADV_DONTNEED; - break; - case org_apache_hadoop_io_nativeio_NativeIO_POSIX_POSIX_FADV_NOREUSE: - return POSIX_FADV_NOREUSE; - break; - default: - return flag; - } -#else - return flag; -#endif -} - -/* * private static native void initNative(); * * We rely on this function rather than lazy initialization because @@ -223,6 +277,10 @@ static int map_fadvise_flag(jint flag) { JNIEXPORT void JNICALL Java_org_apache_hadoop_io_nativeio_NativeIO_initNative( JNIEnv *env, jclass clazz) { +#ifdef UNIX + consts_init(env); + PASS_EXCEPTIONS_GOTO(env, error); +#endif stat_init(env, clazz); PASS_EXCEPTIONS_GOTO(env, error); nioe_init(env); @@ -345,7 +403,7 @@ Java_org_apache_hadoop_io_nativeio_NativeIO_00024POSIX_posix_1fadvise( PASS_EXCEPTIONS(env); int err = 0; - if ((err = posix_fadvise(fd, (off_t)offset, (off_t)len, map_fadvise_flag(flags)))) { + if ((err = posix_fadvise(fd, (off_t)offset, (off_t)len, flags))) { #ifdef __FreeBSD__ throw_ioe(env, errno); #else @@ -448,22 +506,6 @@ Java_org_apache_hadoop_io_nativeio_NativeIO_00024POSIX_mlock_1native( #endif } -#ifdef __FreeBSD__ -static int toFreeBSDFlags(int flags) -{ - int rc = flags & 03; - if ( flags & 0100 ) rc |= O_CREAT; - if ( flags & 0200 ) rc |= O_EXCL; - if ( flags & 0400 ) rc |= O_NOCTTY; - if ( flags & 01000 ) rc |= O_TRUNC; - if ( flags & 02000 ) rc |= O_APPEND; - if ( flags & 04000 ) rc |= O_NONBLOCK; - if ( flags &010000 ) rc |= O_SYNC; - if ( flags &020000 ) rc |= O_ASYNC; - return rc; -} -#endif - /* * Class: org_apache_hadoop_io_nativeio_NativeIO_POSIX * Method: open @@ -479,9 +521,6 @@ Java_org_apache_hadoop_io_nativeio_NativeIO_00024POSIX_open( jint flags, jint mode) { #ifdef UNIX -#ifdef __FreeBSD__ - flags = toFreeBSDFlags(flags); -#endif jobject ret = NULL; const char *path = (*env)->GetStringUTFChars(env, j_path, NULL); http://git-wip-us.apache.org/repos/asf/hadoop/blob/21d10ccc/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/nativeio/TestNativeIO.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/nativeio/TestNativeIO.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/nativeio/TestNativeIO.java index bf3ece7..13fdbc1 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/nativeio/TestNativeIO.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/nativeio/TestNativeIO.java @@ -55,6 +55,9 @@ import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.util.NativeCodeLoader; import org.apache.hadoop.util.Time; +import static org.apache.hadoop.io.nativeio.NativeIO.POSIX.*; +import static org.apache.hadoop.io.nativeio.NativeIO.POSIX.Stat.*; + public class TestNativeIO { static final Log LOG = LogFactory.getLog(TestNativeIO.class); @@ -93,9 +96,8 @@ public class TestNativeIO { assertEquals(expectedOwner, owner); assertNotNull(stat.getGroup()); assertTrue(!stat.getGroup().isEmpty()); - assertEquals("Stat mode field should indicate a regular file", - NativeIO.POSIX.Stat.S_IFREG, - stat.getMode() & NativeIO.POSIX.Stat.S_IFMT); + assertEquals("Stat mode field should indicate a regular file", S_IFREG, + stat.getMode() & S_IFMT); } /** @@ -128,8 +130,7 @@ public class TestNativeIO { assertNotNull(stat.getGroup()); assertTrue(!stat.getGroup().isEmpty()); assertEquals("Stat mode field should indicate a regular file", - NativeIO.POSIX.Stat.S_IFREG, - stat.getMode() & NativeIO.POSIX.Stat.S_IFMT); + S_IFREG, stat.getMode() & S_IFMT); } catch (Throwable t) { thrown.set(t); } @@ -338,8 +339,7 @@ public class TestNativeIO { LOG.info("Open a missing file without O_CREAT and it should fail"); try { FileDescriptor fd = NativeIO.POSIX.open( - new File(TEST_DIR, "doesntexist").getAbsolutePath(), - NativeIO.POSIX.O_WRONLY, 0700); + new File(TEST_DIR, "doesntexist").getAbsolutePath(), O_WRONLY, 0700); fail("Able to open a new file without O_CREAT"); } catch (NativeIOException nioe) { LOG.info("Got expected exception", nioe); @@ -356,7 +356,7 @@ public class TestNativeIO { LOG.info("Test creating a file with O_CREAT"); FileDescriptor fd = NativeIO.POSIX.open( new File(TEST_DIR, "testWorkingOpen").getAbsolutePath(), - NativeIO.POSIX.O_WRONLY | NativeIO.POSIX.O_CREAT, 0700); + O_WRONLY | O_CREAT, 0700); assertNotNull(true); assertTrue(fd.valid()); FileOutputStream fos = new FileOutputStream(fd); @@ -369,7 +369,7 @@ public class TestNativeIO { try { fd = NativeIO.POSIX.open( new File(TEST_DIR, "testWorkingOpen").getAbsolutePath(), - NativeIO.POSIX.O_WRONLY | NativeIO.POSIX.O_CREAT | NativeIO.POSIX.O_EXCL, 0700); + O_WRONLY | O_CREAT | O_EXCL, 0700); fail("Was able to create existing file with O_EXCL"); } catch (NativeIOException nioe) { LOG.info("Got expected exception for failed exclusive create", nioe); @@ -390,7 +390,7 @@ public class TestNativeIO { for (int i = 0; i < 10000; i++) { FileDescriptor fd = NativeIO.POSIX.open( new File(TEST_DIR, "testNoFdLeak").getAbsolutePath(), - NativeIO.POSIX.O_WRONLY | NativeIO.POSIX.O_CREAT, 0700); + O_WRONLY | O_CREAT, 0700); assertNotNull(true); assertTrue(fd.valid()); FileOutputStream fos = new FileOutputStream(fd); @@ -436,8 +436,7 @@ public class TestNativeIO { FileInputStream fis = new FileInputStream("/dev/zero"); try { NativeIO.POSIX.posix_fadvise( - fis.getFD(), 0, 0, - NativeIO.POSIX.POSIX_FADV_SEQUENTIAL); + fis.getFD(), 0, 0, POSIX_FADV_SEQUENTIAL); } catch (UnsupportedOperationException uoe) { // we should just skip the unit test on machines where we don't // have fadvise support @@ -450,20 +449,14 @@ public class TestNativeIO { } try { - NativeIO.POSIX.posix_fadvise( - fis.getFD(), 0, 1024, - NativeIO.POSIX.POSIX_FADV_SEQUENTIAL); - + NativeIO.POSIX.posix_fadvise(fis.getFD(), 0, 1024, POSIX_FADV_SEQUENTIAL); fail("Did not throw on bad file"); } catch (NativeIOException nioe) { assertEquals(Errno.EBADF, nioe.getErrno()); } try { - NativeIO.POSIX.posix_fadvise( - null, 0, 1024, - NativeIO.POSIX.POSIX_FADV_SEQUENTIAL); - + NativeIO.POSIX.posix_fadvise(null, 0, 1024, POSIX_FADV_SEQUENTIAL); fail("Did not throw on null file"); } catch (NullPointerException npe) { // expected @@ -476,9 +469,8 @@ public class TestNativeIO { new File(TEST_DIR, "testSyncFileRange")); try { fos.write("foo".getBytes()); - NativeIO.POSIX.sync_file_range( - fos.getFD(), 0, 1024, - NativeIO.POSIX.SYNC_FILE_RANGE_WRITE); + NativeIO.POSIX.sync_file_range(fos.getFD(), 0, 1024, + SYNC_FILE_RANGE_WRITE); // no way to verify that this actually has synced, // but if it doesn't throw, we can assume it worked } catch (UnsupportedOperationException uoe) { @@ -489,9 +481,8 @@ public class TestNativeIO { fos.close(); } try { - NativeIO.POSIX.sync_file_range( - fos.getFD(), 0, 1024, - NativeIO.POSIX.SYNC_FILE_RANGE_WRITE); + NativeIO.POSIX.sync_file_range(fos.getFD(), 0, 1024, + SYNC_FILE_RANGE_WRITE); fail("Did not throw on bad file"); } catch (NativeIOException nioe) { assertEquals(Errno.EBADF, nioe.getErrno()); @@ -657,4 +648,51 @@ public class TestNativeIO { FileUtils.deleteQuietly(TEST_DIR); } } + + @Test (timeout=10000) + public void testNativePosixConsts() { + assumeTrue("Native POSIX constants not required for Windows", + !Path.WINDOWS); + assertTrue("Native 0_RDONLY const not set", O_RDONLY >= 0); + assertTrue("Native 0_WRONLY const not set", O_WRONLY >= 0); + assertTrue("Native 0_RDWR const not set", O_RDWR >= 0); + assertTrue("Native 0_CREAT const not set", O_CREAT >= 0); + assertTrue("Native 0_EXCL const not set", O_EXCL >= 0); + assertTrue("Native 0_NOCTTY const not set", O_NOCTTY >= 0); + assertTrue("Native 0_TRUNC const not set", O_TRUNC >= 0); + assertTrue("Native 0_APPEND const not set", O_APPEND >= 0); + assertTrue("Native 0_NONBLOCK const not set", O_NONBLOCK >= 0); + assertTrue("Native 0_SYNC const not set", O_SYNC >= 0); + assertTrue("Native S_IFMT const not set", S_IFMT >= 0); + assertTrue("Native S_IFIFO const not set", S_IFIFO >= 0); + assertTrue("Native S_IFCHR const not set", S_IFCHR >= 0); + assertTrue("Native S_IFDIR const not set", S_IFDIR >= 0); + assertTrue("Native S_IFBLK const not set", S_IFBLK >= 0); + assertTrue("Native S_IFREG const not set", S_IFREG >= 0); + assertTrue("Native S_IFLNK const not set", S_IFLNK >= 0); + assertTrue("Native S_IFSOCK const not set", S_IFSOCK >= 0); + assertTrue("Native S_ISUID const not set", S_ISUID >= 0); + assertTrue("Native S_ISGID const not set", S_ISGID >= 0); + assertTrue("Native S_ISVTX const not set", S_ISVTX >= 0); + assertTrue("Native S_IRUSR const not set", S_IRUSR >= 0); + assertTrue("Native S_IWUSR const not set", S_IWUSR >= 0); + assertTrue("Native S_IXUSR const not set", S_IXUSR >= 0); + } + + @Test (timeout=10000) + public void testNativeFadviseConsts() { + assumeTrue("Fadvise constants not supported", fadvisePossible); + assertTrue("Native POSIX_FADV_NORMAL const not set", + POSIX_FADV_NORMAL >= 0); + assertTrue("Native POSIX_FADV_RANDOM const not set", + POSIX_FADV_RANDOM >= 0); + assertTrue("Native POSIX_FADV_SEQUENTIAL const not set", + POSIX_FADV_SEQUENTIAL >= 0); + assertTrue("Native POSIX_FADV_WILLNEED const not set", + POSIX_FADV_WILLNEED >= 0); + assertTrue("Native POSIX_FADV_DONTNEED const not set", + POSIX_FADV_DONTNEED >= 0); + assertTrue("Native POSIX_FADV_NOREUSE const not set", + POSIX_FADV_NOREUSE >= 0); + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/21d10ccc/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java index 55c9d57..1cb308f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java @@ -58,6 +58,9 @@ import org.apache.hadoop.util.DataChecksum; import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.util.Time; +import static org.apache.hadoop.io.nativeio.NativeIO.POSIX.POSIX_FADV_DONTNEED; +import static org.apache.hadoop.io.nativeio.NativeIO.POSIX.SYNC_FILE_RANGE_WRITE; + import com.google.common.annotations.VisibleForTesting; /** A class that receives a block and writes to its own disk, meanwhile @@ -791,12 +794,12 @@ class BlockReceiver implements Closeable { this.datanode.getFSDataset().submitBackgroundSyncFileRangeRequest( block, outFd, lastCacheManagementOffset, offsetInBlock - lastCacheManagementOffset, - NativeIO.POSIX.SYNC_FILE_RANGE_WRITE); + SYNC_FILE_RANGE_WRITE); } else { NativeIO.POSIX.syncFileRangeIfPossible(outFd, - lastCacheManagementOffset, offsetInBlock - - lastCacheManagementOffset, - NativeIO.POSIX.SYNC_FILE_RANGE_WRITE); + lastCacheManagementOffset, + offsetInBlock - lastCacheManagementOffset, + SYNC_FILE_RANGE_WRITE); } } // @@ -812,8 +815,7 @@ class BlockReceiver implements Closeable { long dropPos = lastCacheManagementOffset - CACHE_DROP_LAG_BYTES; if (dropPos > 0 && dropCacheBehindWrites) { NativeIO.POSIX.getCacheManipulator().posixFadviseIfPossible( - block.getBlockName(), outFd, 0, dropPos, - NativeIO.POSIX.POSIX_FADV_DONTNEED); + block.getBlockName(), outFd, 0, dropPos, POSIX_FADV_DONTNEED); } lastCacheManagementOffset = offsetInBlock; long duration = Time.monotonicNow() - begin; http://git-wip-us.apache.org/repos/asf/hadoop/blob/21d10ccc/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockSender.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockSender.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockSender.java index 79f4dd7..fb8b132 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockSender.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockSender.java @@ -51,6 +51,9 @@ import org.apache.htrace.Sampler; import org.apache.htrace.Trace; import org.apache.htrace.TraceScope; +import static org.apache.hadoop.io.nativeio.NativeIO.POSIX.POSIX_FADV_DONTNEED; +import static org.apache.hadoop.io.nativeio.NativeIO.POSIX.POSIX_FADV_SEQUENTIAL; + import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; @@ -411,8 +414,7 @@ class BlockSender implements java.io.Closeable { try { NativeIO.POSIX.getCacheManipulator().posixFadviseIfPossible( block.getBlockName(), blockInFd, lastCacheDropOffset, - offset - lastCacheDropOffset, - NativeIO.POSIX.POSIX_FADV_DONTNEED); + offset - lastCacheDropOffset, POSIX_FADV_DONTNEED); } catch (Exception e) { LOG.warn("Unable to drop cache on file close", e); } @@ -729,8 +731,7 @@ class BlockSender implements java.io.Closeable { if (isLongRead() && blockInFd != null) { // Advise that this file descriptor will be accessed sequentially. NativeIO.POSIX.getCacheManipulator().posixFadviseIfPossible( - block.getBlockName(), blockInFd, 0, 0, - NativeIO.POSIX.POSIX_FADV_SEQUENTIAL); + block.getBlockName(), blockInFd, 0, 0, POSIX_FADV_SEQUENTIAL); } // Trigger readahead of beginning of file if configured. @@ -818,7 +819,7 @@ class BlockSender implements java.io.Closeable { long dropLength = offset - lastCacheDropOffset; NativeIO.POSIX.getCacheManipulator().posixFadviseIfPossible( block.getBlockName(), blockInFd, lastCacheDropOffset, - dropLength, NativeIO.POSIX.POSIX_FADV_DONTNEED); + dropLength, POSIX_FADV_DONTNEED); lastCacheDropOffset = offset; } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/21d10ccc/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestCachingStrategy.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestCachingStrategy.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestCachingStrategy.java index 709554a..bd1a777 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestCachingStrategy.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestCachingStrategy.java @@ -39,6 +39,9 @@ import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.io.nativeio.NativeIO; import org.apache.hadoop.io.nativeio.NativeIO.POSIX.CacheManipulator; import org.apache.hadoop.io.nativeio.NativeIOException; + +import static org.apache.hadoop.io.nativeio.NativeIO.POSIX.POSIX_FADV_DONTNEED; + import org.junit.Assert; import org.junit.BeforeClass; import org.junit.Test; @@ -78,7 +81,7 @@ public class TestCachingStrategy { synchronized void fadvise(int offset, int len, int flags) { LOG.debug("got fadvise(offset=" + offset + ", len=" + len + ",flags=" + flags + ")"); - if (flags == NativeIO.POSIX.POSIX_FADV_DONTNEED) { + if (flags == POSIX_FADV_DONTNEED) { for (int i = 0; i < len; i++) { dropped[(offset + i)] = true; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/21d10ccc/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/FadvisedChunkedFile.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/FadvisedChunkedFile.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/FadvisedChunkedFile.java index 70e6829..7e24e89 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/FadvisedChunkedFile.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/FadvisedChunkedFile.java @@ -27,6 +27,9 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.io.ReadaheadPool; import org.apache.hadoop.io.ReadaheadPool.ReadaheadRequest; import org.apache.hadoop.io.nativeio.NativeIO; + +import static org.apache.hadoop.io.nativeio.NativeIO.POSIX.POSIX_FADV_DONTNEED; + import org.jboss.netty.handler.stream.ChunkedFile; public class FadvisedChunkedFile extends ChunkedFile { @@ -72,7 +75,7 @@ public class FadvisedChunkedFile extends ChunkedFile { NativeIO.POSIX.getCacheManipulator().posixFadviseIfPossible(identifier, fd, getStartOffset(), getEndOffset() - getStartOffset(), - NativeIO.POSIX.POSIX_FADV_DONTNEED); + POSIX_FADV_DONTNEED); } catch (Throwable t) { LOG.warn("Failed to manage OS cache for " + identifier, t); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/21d10ccc/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/FadvisedFileRegion.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/FadvisedFileRegion.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/FadvisedFileRegion.java index f1acfff..6fd46a5 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/FadvisedFileRegion.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/FadvisedFileRegion.java @@ -30,6 +30,9 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.io.ReadaheadPool; import org.apache.hadoop.io.ReadaheadPool.ReadaheadRequest; import org.apache.hadoop.io.nativeio.NativeIO; + +import static org.apache.hadoop.io.nativeio.NativeIO.POSIX.POSIX_FADV_DONTNEED; + import org.jboss.netty.channel.DefaultFileRegion; import com.google.common.annotations.VisibleForTesting; @@ -155,8 +158,7 @@ public class FadvisedFileRegion extends DefaultFileRegion { if (manageOsCache && getCount() > 0) { try { NativeIO.POSIX.getCacheManipulator().posixFadviseIfPossible(identifier, - fd, getPosition(), getCount(), - NativeIO.POSIX.POSIX_FADV_DONTNEED); + fd, getPosition(), getCount(), POSIX_FADV_DONTNEED); } catch (Throwable t) { LOG.warn("Failed to manage OS cache for " + identifier, t); }
