Author: subru
Date: Fri Jul 25 20:33:09 2014
New Revision: 1613514
URL: http://svn.apache.org/r1613514
Log:
syncing YARN-1051 branch with trunk
Modified:
hadoop/common/branches/YARN-1051/ (props changed)
hadoop/common/branches/YARN-1051/hadoop-project/src/site/site.xml
hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/metrics/AzureFileSystemMetricsSystem.java
hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/AzureBlobStorageTestAccount.java
hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/NativeAzureFileSystemBaseTest.java
hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptionSwitch.java
hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/OptionsParser.java
hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/RumenToSLSConverter.java
hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java
hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/appmaster/AMSimulator.java
hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/appmaster/MRAMSimulator.java
hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/conf/SLSConfiguration.java
hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NMSimulator.java
hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java
hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/CapacitySchedulerMetrics.java
hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ContainerSimulator.java
hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/FairSchedulerMetrics.java
hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/FifoSchedulerMetrics.java
hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/NodeUpdateSchedulerEventWrapper.java
hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java
hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java
hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSCapacityScheduler.java
hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SchedulerMetrics.java
hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SchedulerWrapper.java
hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/TaskRunner.java
hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/utils/SLSUtils.java
hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/web/SLSWebApp.java
Propchange: hadoop/common/branches/YARN-1051/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk:r1609878-1613507
Modified: hadoop/common/branches/YARN-1051/hadoop-project/src/site/site.xml
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-project/src/site/site.xml?rev=1613514&r1=1613513&r2=1613514&view=diff
==============================================================================
--- hadoop/common/branches/YARN-1051/hadoop-project/src/site/site.xml (original)
+++ hadoop/common/branches/YARN-1051/hadoop-project/src/site/site.xml Fri Jul
25 20:33:09 2014
@@ -89,6 +89,7 @@
<item name="HDFS NFS Gateway"
href="hadoop-project-dist/hadoop-hdfs/HdfsNfsGateway.html"/>
<item name="HDFS Rolling Upgrade"
href="hadoop-project-dist/hadoop-hdfs/HdfsRollingUpgrade.html"/>
<item name="Extended Attributes"
href="hadoop-project-dist/hadoop-hdfs/ExtendedAttributes.html"/>
+ <item name="HDFS Support for Multihoming"
href="hadoop-project-dist/hadoop-hdfs/HdfsMultihoming.html"/>
</menu>
<menu name="MapReduce" inherit="top">
Modified:
hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java?rev=1613514&r1=1613513&r2=1613514&view=diff
==============================================================================
---
hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
(original)
+++
hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
Fri Jul 25 20:33:09 2014
@@ -373,6 +373,8 @@ public class NativeAzureFileSystem exten
private Path workingDir;
private long blockSize = MAX_AZURE_BLOCK_SIZE;
private AzureFileSystemInstrumentation instrumentation;
+ private String metricsSourceName;
+ private boolean isClosed = false;
private static boolean suppressRetryPolicy = false;
// A counter to create unique (within-process) names for my metrics sources.
private static AtomicInteger metricsSourceNameCounter = new AtomicInteger();
@@ -482,11 +484,10 @@ public class NativeAzureFileSystem exten
// Make sure the metrics system is available before interacting with Azure
AzureFileSystemMetricsSystem.fileSystemStarted();
- String sourceName = newMetricsSourceName(),
- sourceDesc = "Azure Storage Volume File System metrics";
- instrumentation = DefaultMetricsSystem.instance().register(sourceName,
- sourceDesc, new AzureFileSystemInstrumentation(conf));
- AzureFileSystemMetricsSystem.registerSource(sourceName, sourceDesc,
+ metricsSourceName = newMetricsSourceName();
+ String sourceDesc = "Azure Storage Volume File System metrics";
+ instrumentation = new AzureFileSystemInstrumentation(conf);
+ AzureFileSystemMetricsSystem.registerSource(metricsSourceName, sourceDesc,
instrumentation);
store.initialize(uri, conf, instrumentation);
@@ -502,7 +503,6 @@ public class NativeAzureFileSystem exten
LOG.debug(" blockSize = "
+ conf.getLong(AZURE_BLOCK_SIZE_PROPERTY_NAME,
MAX_AZURE_BLOCK_SIZE));
}
-
}
private NativeFileSystemStore createDefaultStore(Configuration conf) {
@@ -1337,7 +1337,11 @@ public class NativeAzureFileSystem exten
}
@Override
- public void close() throws IOException {
+ public synchronized void close() throws IOException {
+ if (isClosed) {
+ return;
+ }
+
// Call the base close() to close any resources there.
super.close();
// Close the store
@@ -1349,12 +1353,14 @@ public class NativeAzureFileSystem exten
long startTime = System.currentTimeMillis();
+ AzureFileSystemMetricsSystem.unregisterSource(metricsSourceName);
AzureFileSystemMetricsSystem.fileSystemClosed();
if (LOG.isDebugEnabled()) {
LOG.debug("Submitting metrics when file system closed took "
+ (System.currentTimeMillis() - startTime) + " ms.");
}
+ isClosed = true;
}
/**
@@ -1498,6 +1504,13 @@ public class NativeAzureFileSystem exten
handleFilesWithDanglingTempData(root, new DanglingFileDeleter());
}
+ @Override
+ protected void finalize() throws Throwable {
+ LOG.debug("finalize() called.");
+ close();
+ super.finalize();
+ }
+
/**
* Encode the key with a random prefix for load balancing in Azure storage.
* Upload data to a random temporary file then do storage side renaming to
Modified:
hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/metrics/AzureFileSystemMetricsSystem.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/metrics/AzureFileSystemMetricsSystem.java?rev=1613514&r1=1613513&r2=1613514&view=diff
==============================================================================
---
hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/metrics/AzureFileSystemMetricsSystem.java
(original)
+++
hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/metrics/AzureFileSystemMetricsSystem.java
Fri Jul 25 20:33:09 2014
@@ -44,21 +44,26 @@ public final class AzureFileSystemMetric
}
public static synchronized void fileSystemClosed() {
- if (instance != null) {
- instance.publishMetricsNow();
- }
if (numFileSystems == 1) {
+ instance.publishMetricsNow();
instance.stop();
instance.shutdown();
instance = null;
}
numFileSystems--;
}
-
+
public static void registerSource(String name, String desc,
MetricsSource source) {
- // Register the source with the name appended with -WasbSystem
- // so that the name is globally unique.
- instance.register(name + "-WasbSystem", desc, source);
+ //caller has to use unique name to register source
+ instance.register(name, desc, source);
+ }
+
+ public static synchronized void unregisterSource(String name) {
+ if (instance != null) {
+ //publish metrics before unregister a metrics source
+ instance.publishMetricsNow();
+ instance.unregisterSource(name);
+ }
}
}
Modified:
hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/AzureBlobStorageTestAccount.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/AzureBlobStorageTestAccount.java?rev=1613514&r1=1613513&r2=1613514&view=diff
==============================================================================
---
hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/AzureBlobStorageTestAccount.java
(original)
+++
hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/AzureBlobStorageTestAccount.java
Fri Jul 25 20:33:09 2014
@@ -324,9 +324,7 @@ public final class AzureBlobStorageTestA
String sourceName = NativeAzureFileSystem.newMetricsSourceName();
String sourceDesc = "Azure Storage Volume File System metrics";
- AzureFileSystemInstrumentation instrumentation =
- DefaultMetricsSystem.instance().register(sourceName,
- sourceDesc, new AzureFileSystemInstrumentation(conf));
+ AzureFileSystemInstrumentation instrumentation = new
AzureFileSystemInstrumentation(conf);
AzureFileSystemMetricsSystem.registerSource(
sourceName, sourceDesc, instrumentation);
Modified:
hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/NativeAzureFileSystemBaseTest.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/NativeAzureFileSystemBaseTest.java?rev=1613514&r1=1613513&r2=1613514&view=diff
==============================================================================
---
hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/NativeAzureFileSystemBaseTest.java
(original)
+++
hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/NativeAzureFileSystemBaseTest.java
Fri Jul 25 20:33:09 2014
@@ -516,6 +516,13 @@ public abstract class NativeAzureFileSys
assertNotNull(status);
}
+ @Test
+ public void testCloseFileSystemTwice() throws Exception {
+ //make sure close() can be called multiple times without doing any harm
+ fs.close();
+ fs.close();
+ }
+
private boolean testModifiedTime(Path testPath, long time) throws Exception {
FileStatus fileStatus = fs.getFileStatus(testPath);
final long errorMargin = modifiedTimeErrorMargin;
Modified:
hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptionSwitch.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptionSwitch.java?rev=1613514&r1=1613513&r2=1613514&view=diff
==============================================================================
---
hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptionSwitch.java
(original)
+++
hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptionSwitch.java
Fri Jul 25 20:33:09 2014
@@ -162,6 +162,7 @@ public enum DistCpOptionSwitch {
BANDWIDTH(DistCpConstants.CONF_LABEL_BANDWIDTH_MB,
new Option("bandwidth", true, "Specify bandwidth per map in MB"));
+ static final String PRESERVE_STATUS_DEFAULT = "-prbugpc";
private final String confLabel;
private final Option option;
Modified:
hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/OptionsParser.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/OptionsParser.java?rev=1613514&r1=1613513&r2=1613514&view=diff
==============================================================================
---
hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/OptionsParser.java
(original)
+++
hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/OptionsParser.java
Fri Jul 25 20:33:09 2014
@@ -50,7 +50,7 @@ public class OptionsParser {
protected String[] flatten(Options options, String[] arguments, boolean
stopAtNonOption) {
for (int index = 0; index < arguments.length; index++) {
if (arguments[index].equals("-" +
DistCpOptionSwitch.PRESERVE_STATUS.getSwitch())) {
- arguments[index] = "-prbugpc";
+ arguments[index] = DistCpOptionSwitch.PRESERVE_STATUS_DEFAULT;
}
}
return super.flatten(options, arguments, stopAtNonOption);
Modified:
hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/RumenToSLSConverter.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/RumenToSLSConverter.java?rev=1613514&r1=1613513&r2=1613514&view=diff
==============================================================================
---
hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/RumenToSLSConverter.java
(original)
+++
hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/RumenToSLSConverter.java
Fri Jul 25 20:33:09 2014
@@ -21,6 +21,8 @@ import org.apache.commons.cli.CommandLin
import org.apache.commons.cli.CommandLineParser;
import org.apache.commons.cli.GnuParser;
import org.apache.commons.cli.Options;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.codehaus.jackson.JsonFactory;
import org.codehaus.jackson.map.ObjectMapper;
import org.codehaus.jackson.map.ObjectWriter;
@@ -42,6 +44,8 @@ import java.util.Set;
import java.util.TreeMap;
import java.util.TreeSet;
+@Private
+@Unstable
public class RumenToSLSConverter {
private static final String EOL = System.getProperty("line.separator");
Modified:
hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java?rev=1613514&r1=1613513&r2=1613514&view=diff
==============================================================================
---
hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java
(original)
+++
hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java
Fri Jul 25 20:33:09 2014
@@ -32,6 +32,8 @@ import java.util.Iterator;
import java.util.Random;
import java.util.Arrays;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.tools.rumen.JobTraceReader;
import org.apache.hadoop.tools.rumen.LoggedJob;
@@ -66,6 +68,8 @@ import org.apache.log4j.Logger;
import org.codehaus.jackson.JsonFactory;
import org.codehaus.jackson.map.ObjectMapper;
+@Private
+@Unstable
public class SLSRunner {
// RM, Runner
private ResourceManager rm;
Modified:
hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/appmaster/AMSimulator.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/appmaster/AMSimulator.java?rev=1613514&r1=1613513&r2=1613514&view=diff
==============================================================================
---
hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/appmaster/AMSimulator.java
(original)
+++
hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/appmaster/AMSimulator.java
Fri Jul 25 20:33:09 2014
@@ -29,6 +29,8 @@ import java.util.Map;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.LinkedBlockingQueue;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest;
@@ -70,6 +72,8 @@ import org.apache.hadoop.yarn.sls.SLSRun
import org.apache.hadoop.yarn.sls.scheduler.TaskRunner;
import org.apache.hadoop.yarn.sls.utils.SLSUtils;
+@Private
+@Unstable
public abstract class AMSimulator extends TaskRunner.Task {
// resource manager
protected ResourceManager rm;
Modified:
hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/appmaster/MRAMSimulator.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/appmaster/MRAMSimulator.java?rev=1613514&r1=1613513&r2=1613514&view=diff
==============================================================================
---
hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/appmaster/MRAMSimulator.java
(original)
+++
hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/appmaster/MRAMSimulator.java
Fri Jul 25 20:33:09 2014
@@ -27,6 +27,8 @@ import java.util.LinkedList;
import java.util.List;
import java.util.Map;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest;
@@ -45,6 +47,8 @@ import org.apache.hadoop.yarn.sls.schedu
import org.apache.hadoop.yarn.sls.SLSRunner;
import org.apache.log4j.Logger;
+@Private
+@Unstable
public class MRAMSimulator extends AMSimulator {
/*
Vocabulary Used:
Modified:
hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/conf/SLSConfiguration.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/conf/SLSConfiguration.java?rev=1613514&r1=1613513&r2=1613514&view=diff
==============================================================================
---
hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/conf/SLSConfiguration.java
(original)
+++
hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/conf/SLSConfiguration.java
Fri Jul 25 20:33:09 2014
@@ -18,6 +18,11 @@
package org.apache.hadoop.yarn.sls.conf;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+
+@Private
+@Unstable
public class SLSConfiguration {
// sls
public static final String PREFIX = "yarn.sls.";
Modified:
hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NMSimulator.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NMSimulator.java?rev=1613514&r1=1613513&r2=1613514&view=diff
==============================================================================
---
hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NMSimulator.java
(original)
+++
hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NMSimulator.java
Fri Jul 25 20:33:09 2014
@@ -27,6 +27,8 @@ import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.DelayQueue;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerExitStatus;
@@ -54,6 +56,8 @@ import org.apache.hadoop.yarn.sls.schedu
import org.apache.hadoop.yarn.sls.scheduler.TaskRunner;
import org.apache.hadoop.yarn.sls.utils.SLSUtils;
+@Private
+@Unstable
public class NMSimulator extends TaskRunner.Task {
// node resource
private RMNode node;
Modified:
hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java?rev=1613514&r1=1613513&r2=1613514&view=diff
==============================================================================
---
hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java
(original)
+++
hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java
Fri Jul 25 20:33:09 2014
@@ -21,6 +21,8 @@ package org.apache.hadoop.yarn.sls.nodem
import java.util.ArrayList;
import java.util.List;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.net.Node;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerExitStatus;
@@ -36,6 +38,8 @@ import org.apache.hadoop.yarn.server.res
import org.apache.hadoop.yarn.server.resourcemanager.rmnode
.UpdatedContainerInfo;
+@Private
+@Unstable
public class NodeInfo {
private static int NODE_ID = 0;
@@ -43,6 +47,8 @@ public class NodeInfo {
return NodeId.newInstance(host, port);
}
+ @Private
+ @Unstable
private static class FakeRMNodeImpl implements RMNode {
private NodeId nodeId;
private String hostName;
@@ -164,7 +170,7 @@ public class NodeInfo {
perNode = resourceOption;
}
}
-
+
public static RMNode newNodeInfo(String rackName, String hostName,
final ResourceOption resourceOption, int port) {
final NodeId nodeId = newNodeID(hostName, port);
Modified:
hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/CapacitySchedulerMetrics.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/CapacitySchedulerMetrics.java?rev=1613514&r1=1613513&r2=1613514&view=diff
==============================================================================
---
hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/CapacitySchedulerMetrics.java
(original)
+++
hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/CapacitySchedulerMetrics.java
Fri Jul 25 20:33:09 2014
@@ -18,6 +18,11 @@
package org.apache.hadoop.yarn.sls.scheduler;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+
+@Private
+@Unstable
public class CapacitySchedulerMetrics extends SchedulerMetrics {
public CapacitySchedulerMetrics() {
Modified:
hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ContainerSimulator.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ContainerSimulator.java?rev=1613514&r1=1613513&r2=1613514&view=diff
==============================================================================
---
hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ContainerSimulator.java
(original)
+++
hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ContainerSimulator.java
Fri Jul 25 20:33:09 2014
@@ -21,9 +21,13 @@ package org.apache.hadoop.yarn.sls.sched
import java.util.concurrent.Delayed;
import java.util.concurrent.TimeUnit;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.Resource;
+@Private
+@Unstable
public class ContainerSimulator implements Delayed {
// id
private ContainerId id;
Modified:
hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/FairSchedulerMetrics.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/FairSchedulerMetrics.java?rev=1613514&r1=1613513&r2=1613514&view=diff
==============================================================================
---
hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/FairSchedulerMetrics.java
(original)
+++
hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/FairSchedulerMetrics.java
Fri Jul 25 20:33:09 2014
@@ -18,6 +18,8 @@
package org.apache.hadoop.yarn.sls.scheduler;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair
.AppSchedulable;
@@ -28,6 +30,8 @@ import org.apache.hadoop.yarn.server.res
import com.codahale.metrics.Gauge;
import org.apache.hadoop.yarn.sls.SLSRunner;
+@Private
+@Unstable
public class FairSchedulerMetrics extends SchedulerMetrics {
private int totalMemoryMB = Integer.MAX_VALUE;
Modified:
hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/FifoSchedulerMetrics.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/FifoSchedulerMetrics.java?rev=1613514&r1=1613513&r2=1613514&view=diff
==============================================================================
---
hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/FifoSchedulerMetrics.java
(original)
+++
hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/FifoSchedulerMetrics.java
Fri Jul 25 20:33:09 2014
@@ -18,12 +18,16 @@
package org.apache.hadoop.yarn.sls.scheduler;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.yarn.api.records.QueueInfo;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo
.FifoScheduler;
import com.codahale.metrics.Gauge;
+@Private
+@Unstable
public class FifoSchedulerMetrics extends SchedulerMetrics {
public FifoSchedulerMetrics() {
Modified:
hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/NodeUpdateSchedulerEventWrapper.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/NodeUpdateSchedulerEventWrapper.java?rev=1613514&r1=1613513&r2=1613514&view=diff
==============================================================================
---
hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/NodeUpdateSchedulerEventWrapper.java
(original)
+++
hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/NodeUpdateSchedulerEventWrapper.java
Fri Jul 25 20:33:09 2014
@@ -18,9 +18,13 @@
package org.apache.hadoop.yarn.sls.scheduler;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event
.NodeUpdateSchedulerEvent;
+@Private
+@Unstable
public class NodeUpdateSchedulerEventWrapper extends NodeUpdateSchedulerEvent {
public NodeUpdateSchedulerEventWrapper(NodeUpdateSchedulerEvent event) {
Modified:
hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java?rev=1613514&r1=1613513&r2=1613514&view=diff
==============================================================================
---
hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java
(original)
+++
hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java
Fri Jul 25 20:33:09 2014
@@ -18,6 +18,8 @@
package org.apache.hadoop.yarn.sls.scheduler;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.net.Node;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerId;
@@ -33,6 +35,8 @@ import org.apache.hadoop.yarn.server.res
import java.util.Collections;
import java.util.List;
+@Private
+@Unstable
public class RMNodeWrapper implements RMNode {
private RMNode node;
private List<UpdatedContainerInfo> updates;
Modified:
hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java?rev=1613514&r1=1613513&r2=1613514&view=diff
==============================================================================
---
hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java
(original)
+++
hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java
Fri Jul 25 20:33:09 2014
@@ -36,6 +36,7 @@ import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceAudience.LimitedPrivate;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.conf.Configurable;
@@ -92,6 +93,8 @@ import com.codahale.metrics.MetricRegist
import com.codahale.metrics.SlidingWindowReservoir;
import com.codahale.metrics.Timer;
+@Private
+@Unstable
final public class ResourceSchedulerWrapper
extends AbstractYarnScheduler<SchedulerApplicationAttempt, SchedulerNode>
implements SchedulerWrapper, ResourceScheduler, Configurable {
Modified:
hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSCapacityScheduler.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSCapacityScheduler.java?rev=1613514&r1=1613513&r2=1613514&view=diff
==============================================================================
---
hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSCapacityScheduler.java
(original)
+++
hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSCapacityScheduler.java
Fri Jul 25 20:33:09 2014
@@ -17,6 +17,8 @@
*/
package org.apache.hadoop.yarn.sls.scheduler;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.util.ShutdownHookManager;
import org.apache.hadoop.yarn.sls.SLSRunner;
import org.apache.hadoop.yarn.sls.conf.SLSConfiguration;
@@ -100,6 +102,8 @@ import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
+@Private
+@Unstable
public class SLSCapacityScheduler extends CapacityScheduler implements
SchedulerWrapper,Configurable {
private static final String EOL = System.getProperty("line.separator");
Modified:
hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SchedulerMetrics.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SchedulerMetrics.java?rev=1613514&r1=1613513&r2=1613514&view=diff
==============================================================================
---
hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SchedulerMetrics.java
(original)
+++
hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SchedulerMetrics.java
Fri Jul 25 20:33:09 2014
@@ -21,6 +21,8 @@ package org.apache.hadoop.yarn.sls.sched
import java.util.HashSet;
import java.util.Set;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler
.ResourceScheduler;
@@ -30,6 +32,8 @@ import org.apache.hadoop.yarn.server.res
import com.codahale.metrics.Gauge;
import com.codahale.metrics.MetricRegistry;
+@Private
+@Unstable
public abstract class SchedulerMetrics {
protected ResourceScheduler scheduler;
protected Set<String> trackedQueues;
Modified:
hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SchedulerWrapper.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SchedulerWrapper.java?rev=1613514&r1=1613513&r2=1613514&view=diff
==============================================================================
---
hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SchedulerWrapper.java
(original)
+++
hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SchedulerWrapper.java
Fri Jul 25 20:33:09 2014
@@ -19,11 +19,15 @@ package org.apache.hadoop.yarn.sls.sched
import java.util.Set;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import com.codahale.metrics.MetricRegistry;
+@Private
+@Unstable
public interface SchedulerWrapper {
public MetricRegistry getMetrics();
Modified:
hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/TaskRunner.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/TaskRunner.java?rev=1613514&r1=1613513&r2=1613514&view=diff
==============================================================================
---
hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/TaskRunner.java
(original)
+++
hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/TaskRunner.java
Fri Jul 25 20:33:09 2014
@@ -25,9 +25,15 @@ import java.util.concurrent.Delayed;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.yarn.exceptions.YarnException;
+@Private
+@Unstable
public class TaskRunner {
+ @Private
+ @Unstable
public abstract static class Task implements Runnable, Delayed {
private long start;
private long end;
Modified:
hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/utils/SLSUtils.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/utils/SLSUtils.java?rev=1613514&r1=1613513&r2=1613514&view=diff
==============================================================================
---
hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/utils/SLSUtils.java
(original)
+++
hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/utils/SLSUtils.java
Fri Jul 25 20:33:09 2014
@@ -17,6 +17,8 @@
*/
package org.apache.hadoop.yarn.sls.utils;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.tools.rumen.JobTraceReader;
@@ -36,6 +38,8 @@ import java.util.Map;
import java.util.List;
import java.util.Iterator;
+@Private
+@Unstable
public class SLSUtils {
public static String[] getRackHostName(String hostname) {
Modified:
hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/web/SLSWebApp.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/web/SLSWebApp.java?rev=1613514&r1=1613513&r2=1613514&view=diff
==============================================================================
---
hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/web/SLSWebApp.java
(original)
+++
hadoop/common/branches/YARN-1051/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/web/SLSWebApp.java
Fri Jul 25 20:33:09 2014
@@ -30,6 +30,8 @@ import javax.servlet.http.HttpServletReq
import javax.servlet.http.HttpServletResponse;
import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event
.SchedulerEventType;
import org.mortbay.jetty.Handler;
@@ -49,6 +51,8 @@ import com.codahale.metrics.Histogram;
import com.codahale.metrics.MetricRegistry;
import org.mortbay.jetty.handler.ResourceHandler;
+@Private
+@Unstable
public class SLSWebApp extends HttpServlet {
private static final long serialVersionUID = 1905162041950251407L;
private transient Server server;
