Author: arp Date: Thu Nov 21 20:06:09 2013 New Revision: 1544306 URL: http://svn.apache.org/r1544306 Log: Merging r1543902 through r1544303 from trunk to branch HDFS-2832
Modified: hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/client/KerberosAuthenticator.java hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/CHANGES.txt (contents, props changed) hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/java/ (props changed) hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DU.java hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSOutputSummer.java hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer.java hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestDU.java Modified: hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/client/KerberosAuthenticator.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/client/KerberosAuthenticator.java?rev=1544306&r1=1544305&r2=1544306&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/client/KerberosAuthenticator.java (original) +++ hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/client/KerberosAuthenticator.java Thu Nov 21 20:06:09 2013 @@ -196,7 +196,13 @@ public class KerberosAuthenticator imple doSpnegoSequence(token); } else { LOG.debug("Using fallback authenticator sequence."); - getFallBackAuthenticator().authenticate(url, token); + Authenticator auth = getFallBackAuthenticator(); + // Make sure that the fall back authenticator have the same + // ConnectionConfigurator, since the method might be overridden. + // Otherwise the fall back authenticator might not have the information + // to make the connection (e.g., SSL certificates) + auth.setConnectionConfigurator(connConfigurator); + auth.authenticate(url, token); } } } Modified: hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1544306&r1=1544305&r2=1544306&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/CHANGES.txt (original) +++ hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/CHANGES.txt Thu Nov 21 20:06:09 2013 @@ -383,6 +383,11 @@ Release 2.3.0 - UNRELEASED HADOOP-10067. Missing POM dependency on jsr305 (Robert Rati via stevel) + HADOOP-10103. update commons-lang to 2.6 (Akira AJISAKA via stevel) + + HADOOP-10111. Allow DU to be initialized with an initial value (Kihwal Lee + via jeagles) + OPTIMIZATIONS HADOOP-9748. Reduce blocking on UGI.ensureInitialized (daryn) @@ -500,6 +505,9 @@ Release 2.2.1 - UNRELEASED HADOOP-10110. hadoop-auth has a build break due to missing dependency. (Chuan Liu via arp) + HADOOP-9114. After defined the dfs.checksum.type as the NULL, write file and hflush will + through java.lang.ArrayIndexOutOfBoundsException (Sathish via umamahesh) + Release 2.2.0 - 2013-10-13 INCOMPATIBLE CHANGES Propchange: hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/CHANGES.txt ------------------------------------------------------------------------------ Merged /hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt:r1544173 Merged /hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt:r1543613-1544303 Propchange: hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/java/ ------------------------------------------------------------------------------ Merged /hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java:r1543710-1544303 Modified: hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DU.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DU.java?rev=1544306&r1=1544305&r2=1544306&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DU.java (original) +++ hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DU.java Thu Nov 21 20:06:09 2013 @@ -47,17 +47,32 @@ public class DU extends Shell { * @throws IOException if we fail to refresh the disk usage */ public DU(File path, long interval) throws IOException { + this(path, interval, -1L); + } + + /** + * Keeps track of disk usage. + * @param path the path to check disk usage in + * @param interval refresh the disk usage at this interval + * @param initialUsed use this value until next refresh + * @throws IOException if we fail to refresh the disk usage + */ + public DU(File path, long interval, long initialUsed) throws IOException { super(0); - + //we set the Shell interval to 0 so it will always run our command //and use this one to set the thread sleep interval this.refreshInterval = interval; this.dirPath = path.getCanonicalPath(); - - //populate the used variable - run(); + + //populate the used variable if the initial value is not specified. + if (initialUsed < 0) { + run(); + } else { + this.used.set(initialUsed); + } } - + /** * Keeps track of disk usage. * @param path the path to check disk usage in @@ -65,9 +80,23 @@ public class DU extends Shell { * @throws IOException if we fail to refresh the disk usage */ public DU(File path, Configuration conf) throws IOException { + this(path, conf, -1L); + } + + /** + * Keeps track of disk usage. + * @param path the path to check disk usage in + * @param conf configuration object + * @param initialUsed use it until the next refresh. + * @throws IOException if we fail to refresh the disk usage + */ + public DU(File path, Configuration conf, long initialUsed) + throws IOException { this(path, conf.getLong(CommonConfigurationKeys.FS_DU_INTERVAL_KEY, - CommonConfigurationKeys.FS_DU_INTERVAL_DEFAULT)); + CommonConfigurationKeys.FS_DU_INTERVAL_DEFAULT), initialUsed); } + + /** * This thread refreshes the "used" variable. Modified: hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSOutputSummer.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSOutputSummer.java?rev=1544306&r1=1544305&r2=1544306&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSOutputSummer.java (original) +++ hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSOutputSummer.java Thu Nov 21 20:06:09 2013 @@ -183,10 +183,13 @@ abstract public class FSOutputSummer ext } static byte[] int2byte(int integer, byte[] bytes) { - bytes[0] = (byte)((integer >>> 24) & 0xFF); - bytes[1] = (byte)((integer >>> 16) & 0xFF); - bytes[2] = (byte)((integer >>> 8) & 0xFF); - bytes[3] = (byte)((integer >>> 0) & 0xFF); + if (bytes.length != 0) { + bytes[0] = (byte) ((integer >>> 24) & 0xFF); + bytes[1] = (byte) ((integer >>> 16) & 0xFF); + bytes[2] = (byte) ((integer >>> 8) & 0xFF); + bytes[3] = (byte) ((integer >>> 0) & 0xFF); + return bytes; + } return bytes; } Modified: hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer.java?rev=1544306&r1=1544305&r2=1544306&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer.java (original) +++ hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer.java Thu Nov 21 20:06:09 2013 @@ -472,7 +472,9 @@ public class HttpServer implements Filte if (conf.getBoolean( CommonConfigurationKeys.HADOOP_JETTY_LOGS_SERVE_ALIASES, CommonConfigurationKeys.DEFAULT_HADOOP_JETTY_LOGS_SERVE_ALIASES)) { - logContext.getInitParams().put( + @SuppressWarnings("unchecked") + Map<String, String> params = logContext.getInitParams(); + params.put( "org.mortbay.jetty.servlet.Default.aliases", "true"); } logContext.setDisplayName("logs"); Modified: hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestDU.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestDU.java?rev=1544306&r1=1544305&r2=1544306&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestDU.java (original) +++ hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestDU.java Thu Nov 21 20:06:09 2013 @@ -116,4 +116,19 @@ public class TestDU extends TestCase { long duSize = du.getUsed(); assertTrue(String.valueOf(duSize), duSize >= 0L); } + + public void testDUSetInitialValue() throws IOException { + File file = new File(DU_DIR, "dataX"); + createFile(file, 8192); + DU du = new DU(file, 3000, 1024); + du.start(); + assertTrue("Initial usage setting not honored", du.getUsed() == 1024); + + // wait until the first du runs. + try { + Thread.sleep(5000); + } catch (InterruptedException ie) {} + + assertTrue("Usage didn't get updated", du.getUsed() == 8192); + } }