Author: szetszwo Date: Fri Jul 18 02:21:21 2014 New Revision: 1611531 URL: http://svn.apache.org/r1611531 Log: Merge r1609845 through r1611528 from trunk.
Modified: hadoop/common/branches/HDFS-6584/ (props changed) hadoop/common/branches/HDFS-6584/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java hadoop/common/branches/HDFS-6584/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/metrics/AzureFileSystemMetricsSystem.java hadoop/common/branches/HDFS-6584/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/AzureBlobStorageTestAccount.java hadoop/common/branches/HDFS-6584/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/NativeAzureFileSystemBaseTest.java hadoop/common/branches/HDFS-6584/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptionSwitch.java hadoop/common/branches/HDFS-6584/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/OptionsParser.java Propchange: hadoop/common/branches/HDFS-6584/ ------------------------------------------------------------------------------ Merged /hadoop/common/trunk:r1610533-1611528 Modified: hadoop/common/branches/HDFS-6584/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java?rev=1611531&r1=1611530&r2=1611531&view=diff ============================================================================== --- hadoop/common/branches/HDFS-6584/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java (original) +++ hadoop/common/branches/HDFS-6584/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java Fri Jul 18 02:21:21 2014 @@ -373,6 +373,8 @@ public class NativeAzureFileSystem exten private Path workingDir; private long blockSize = MAX_AZURE_BLOCK_SIZE; private AzureFileSystemInstrumentation instrumentation; + private String metricsSourceName; + private boolean isClosed = false; private static boolean suppressRetryPolicy = false; // A counter to create unique (within-process) names for my metrics sources. private static AtomicInteger metricsSourceNameCounter = new AtomicInteger(); @@ -482,11 +484,10 @@ public class NativeAzureFileSystem exten // Make sure the metrics system is available before interacting with Azure AzureFileSystemMetricsSystem.fileSystemStarted(); - String sourceName = newMetricsSourceName(), - sourceDesc = "Azure Storage Volume File System metrics"; - instrumentation = DefaultMetricsSystem.instance().register(sourceName, - sourceDesc, new AzureFileSystemInstrumentation(conf)); - AzureFileSystemMetricsSystem.registerSource(sourceName, sourceDesc, + metricsSourceName = newMetricsSourceName(); + String sourceDesc = "Azure Storage Volume File System metrics"; + instrumentation = new AzureFileSystemInstrumentation(conf); + AzureFileSystemMetricsSystem.registerSource(metricsSourceName, sourceDesc, instrumentation); store.initialize(uri, conf, instrumentation); @@ -502,7 +503,6 @@ public class NativeAzureFileSystem exten LOG.debug(" blockSize = " + conf.getLong(AZURE_BLOCK_SIZE_PROPERTY_NAME, MAX_AZURE_BLOCK_SIZE)); } - } private NativeFileSystemStore createDefaultStore(Configuration conf) { @@ -1337,7 +1337,11 @@ public class NativeAzureFileSystem exten } @Override - public void close() throws IOException { + public synchronized void close() throws IOException { + if (isClosed) { + return; + } + // Call the base close() to close any resources there. super.close(); // Close the store @@ -1349,12 +1353,14 @@ public class NativeAzureFileSystem exten long startTime = System.currentTimeMillis(); + AzureFileSystemMetricsSystem.unregisterSource(metricsSourceName); AzureFileSystemMetricsSystem.fileSystemClosed(); if (LOG.isDebugEnabled()) { LOG.debug("Submitting metrics when file system closed took " + (System.currentTimeMillis() - startTime) + " ms."); } + isClosed = true; } /** @@ -1498,6 +1504,13 @@ public class NativeAzureFileSystem exten handleFilesWithDanglingTempData(root, new DanglingFileDeleter()); } + @Override + protected void finalize() throws Throwable { + LOG.debug("finalize() called."); + close(); + super.finalize(); + } + /** * Encode the key with a random prefix for load balancing in Azure storage. * Upload data to a random temporary file then do storage side renaming to Modified: hadoop/common/branches/HDFS-6584/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/metrics/AzureFileSystemMetricsSystem.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/metrics/AzureFileSystemMetricsSystem.java?rev=1611531&r1=1611530&r2=1611531&view=diff ============================================================================== --- hadoop/common/branches/HDFS-6584/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/metrics/AzureFileSystemMetricsSystem.java (original) +++ hadoop/common/branches/HDFS-6584/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/metrics/AzureFileSystemMetricsSystem.java Fri Jul 18 02:21:21 2014 @@ -44,21 +44,26 @@ public final class AzureFileSystemMetric } public static synchronized void fileSystemClosed() { - if (instance != null) { - instance.publishMetricsNow(); - } if (numFileSystems == 1) { + instance.publishMetricsNow(); instance.stop(); instance.shutdown(); instance = null; } numFileSystems--; } - + public static void registerSource(String name, String desc, MetricsSource source) { - // Register the source with the name appended with -WasbSystem - // so that the name is globally unique. - instance.register(name + "-WasbSystem", desc, source); + //caller has to use unique name to register source + instance.register(name, desc, source); + } + + public static synchronized void unregisterSource(String name) { + if (instance != null) { + //publish metrics before unregister a metrics source + instance.publishMetricsNow(); + instance.unregisterSource(name); + } } } Modified: hadoop/common/branches/HDFS-6584/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/AzureBlobStorageTestAccount.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/AzureBlobStorageTestAccount.java?rev=1611531&r1=1611530&r2=1611531&view=diff ============================================================================== --- hadoop/common/branches/HDFS-6584/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/AzureBlobStorageTestAccount.java (original) +++ hadoop/common/branches/HDFS-6584/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/AzureBlobStorageTestAccount.java Fri Jul 18 02:21:21 2014 @@ -324,9 +324,7 @@ public final class AzureBlobStorageTestA String sourceName = NativeAzureFileSystem.newMetricsSourceName(); String sourceDesc = "Azure Storage Volume File System metrics"; - AzureFileSystemInstrumentation instrumentation = - DefaultMetricsSystem.instance().register(sourceName, - sourceDesc, new AzureFileSystemInstrumentation(conf)); + AzureFileSystemInstrumentation instrumentation = new AzureFileSystemInstrumentation(conf); AzureFileSystemMetricsSystem.registerSource( sourceName, sourceDesc, instrumentation); Modified: hadoop/common/branches/HDFS-6584/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/NativeAzureFileSystemBaseTest.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/NativeAzureFileSystemBaseTest.java?rev=1611531&r1=1611530&r2=1611531&view=diff ============================================================================== --- hadoop/common/branches/HDFS-6584/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/NativeAzureFileSystemBaseTest.java (original) +++ hadoop/common/branches/HDFS-6584/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/NativeAzureFileSystemBaseTest.java Fri Jul 18 02:21:21 2014 @@ -516,6 +516,13 @@ public abstract class NativeAzureFileSys assertNotNull(status); } + @Test + public void testCloseFileSystemTwice() throws Exception { + //make sure close() can be called multiple times without doing any harm + fs.close(); + fs.close(); + } + private boolean testModifiedTime(Path testPath, long time) throws Exception { FileStatus fileStatus = fs.getFileStatus(testPath); final long errorMargin = modifiedTimeErrorMargin; Modified: hadoop/common/branches/HDFS-6584/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptionSwitch.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptionSwitch.java?rev=1611531&r1=1611530&r2=1611531&view=diff ============================================================================== --- hadoop/common/branches/HDFS-6584/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptionSwitch.java (original) +++ hadoop/common/branches/HDFS-6584/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptionSwitch.java Fri Jul 18 02:21:21 2014 @@ -162,6 +162,7 @@ public enum DistCpOptionSwitch { BANDWIDTH(DistCpConstants.CONF_LABEL_BANDWIDTH_MB, new Option("bandwidth", true, "Specify bandwidth per map in MB")); + static final String PRESERVE_STATUS_DEFAULT = "-prbugpc"; private final String confLabel; private final Option option; Modified: hadoop/common/branches/HDFS-6584/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/OptionsParser.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/OptionsParser.java?rev=1611531&r1=1611530&r2=1611531&view=diff ============================================================================== --- hadoop/common/branches/HDFS-6584/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/OptionsParser.java (original) +++ hadoop/common/branches/HDFS-6584/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/OptionsParser.java Fri Jul 18 02:21:21 2014 @@ -50,7 +50,7 @@ public class OptionsParser { protected String[] flatten(Options options, String[] arguments, boolean stopAtNonOption) { for (int index = 0; index < arguments.length; index++) { if (arguments[index].equals("-" + DistCpOptionSwitch.PRESERVE_STATUS.getSwitch())) { - arguments[index] = "-prbugpc"; + arguments[index] = DistCpOptionSwitch.PRESERVE_STATUS_DEFAULT; } } return super.flatten(options, arguments, stopAtNonOption);