[ https://issues.apache.org/jira/browse/YARN-1021?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=13780781#comment-13780781 ]
Hudson commented on YARN-1021: ------------------------------ FAILURE: Integrated in Hadoop-Yarn-trunk #346 (See [https://builds.apache.org/job/Hadoop-Yarn-trunk/346/]) YARN-1021. Yarn Scheduler Load Simulator. (ywskycn via tucu) (tucu: http://svn.apache.org/viewcvs.cgi/?root=Apache-SVN&view=rev&rev=1527059) * /hadoop/common/trunk/hadoop-assemblies/src/main/resources/assemblies/hadoop-sls.xml * /hadoop/common/trunk/hadoop-assemblies/src/main/resources/assemblies/hadoop-tools.xml * /hadoop/common/trunk/hadoop-project/pom.xml * /hadoop/common/trunk/hadoop-project/src/site/site.xml * /hadoop/common/trunk/hadoop-tools/hadoop-openstack * /hadoop/common/trunk/hadoop-tools/hadoop-sls * /hadoop/common/trunk/hadoop-tools/hadoop-sls/README * /hadoop/common/trunk/hadoop-tools/hadoop-sls/dev-support * /hadoop/common/trunk/hadoop-tools/hadoop-sls/dev-support/findbugs-exclude.xml * /hadoop/common/trunk/hadoop-tools/hadoop-sls/pom.xml * /hadoop/common/trunk/hadoop-tools/hadoop-sls/src * /hadoop/common/trunk/hadoop-tools/hadoop-sls/src/main * /hadoop/common/trunk/hadoop-tools/hadoop-sls/src/main/assemblies * /hadoop/common/trunk/hadoop-tools/hadoop-sls/src/main/assemblies/sls.xml * /hadoop/common/trunk/hadoop-tools/hadoop-sls/src/main/bin * /hadoop/common/trunk/hadoop-tools/hadoop-sls/src/main/bin/rumen2sls.sh * /hadoop/common/trunk/hadoop-tools/hadoop-sls/src/main/bin/slsrun.sh * /hadoop/common/trunk/hadoop-tools/hadoop-sls/src/main/data * /hadoop/common/trunk/hadoop-tools/hadoop-sls/src/main/data/2jobs2min-rumen-jh.json * /hadoop/common/trunk/hadoop-tools/hadoop-sls/src/main/html * /hadoop/common/trunk/hadoop-tools/hadoop-sls/src/main/html/css * /hadoop/common/trunk/hadoop-tools/hadoop-sls/src/main/html/css/bootstrap-responsive.min.css * /hadoop/common/trunk/hadoop-tools/hadoop-sls/src/main/html/css/bootstrap.min.css * /hadoop/common/trunk/hadoop-tools/hadoop-sls/src/main/html/js * /hadoop/common/trunk/hadoop-tools/hadoop-sls/src/main/html/js/thirdparty * /hadoop/common/trunk/hadoop-tools/hadoop-sls/src/main/html/js/thirdparty/bootstrap.min.js * /hadoop/common/trunk/hadoop-tools/hadoop-sls/src/main/html/js/thirdparty/d3-LICENSE * /hadoop/common/trunk/hadoop-tools/hadoop-sls/src/main/html/js/thirdparty/d3.v3.js * /hadoop/common/trunk/hadoop-tools/hadoop-sls/src/main/html/js/thirdparty/jquery.js * /hadoop/common/trunk/hadoop-tools/hadoop-sls/src/main/html/showSimulationTrace.html * /hadoop/common/trunk/hadoop-tools/hadoop-sls/src/main/html/simulate.html.template * /hadoop/common/trunk/hadoop-tools/hadoop-sls/src/main/html/simulate.info.html.template * /hadoop/common/trunk/hadoop-tools/hadoop-sls/src/main/html/track.html.template * /hadoop/common/trunk/hadoop-tools/hadoop-sls/src/main/java * /hadoop/common/trunk/hadoop-tools/hadoop-sls/src/main/java/org * /hadoop/common/trunk/hadoop-tools/hadoop-sls/src/main/java/org/apache * /hadoop/common/trunk/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop * /hadoop/common/trunk/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn * /hadoop/common/trunk/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls * /hadoop/common/trunk/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/RumenToSLSConverter.java * /hadoop/common/trunk/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java * /hadoop/common/trunk/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/appmaster * /hadoop/common/trunk/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/appmaster/AMSimulator.java * /hadoop/common/trunk/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/appmaster/MRAMSimulator.java * /hadoop/common/trunk/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/conf * /hadoop/common/trunk/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/conf/SLSConfiguration.java * /hadoop/common/trunk/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager * /hadoop/common/trunk/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NMSimulator.java * /hadoop/common/trunk/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java * /hadoop/common/trunk/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler * /hadoop/common/trunk/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/CapacitySchedulerMetrics.java * /hadoop/common/trunk/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ContainerSimulator.java * /hadoop/common/trunk/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/FairSchedulerMetrics.java * /hadoop/common/trunk/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/FifoSchedulerMetrics.java * /hadoop/common/trunk/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/NodeUpdateSchedulerEventWrapper.java * /hadoop/common/trunk/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java * /hadoop/common/trunk/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java * /hadoop/common/trunk/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SchedulerMetrics.java * /hadoop/common/trunk/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/TaskRunner.java * /hadoop/common/trunk/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/utils * /hadoop/common/trunk/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/utils/SLSUtils.java * /hadoop/common/trunk/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/web * /hadoop/common/trunk/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/web/SLSWebApp.java * /hadoop/common/trunk/hadoop-tools/hadoop-sls/src/main/sample-conf * /hadoop/common/trunk/hadoop-tools/hadoop-sls/src/main/sample-conf/capacity-scheduler.xml * /hadoop/common/trunk/hadoop-tools/hadoop-sls/src/main/sample-conf/fair-scheduler-allocation.xml * /hadoop/common/trunk/hadoop-tools/hadoop-sls/src/main/sample-conf/fair-scheduler.xml * /hadoop/common/trunk/hadoop-tools/hadoop-sls/src/main/sample-conf/log4j.properties * /hadoop/common/trunk/hadoop-tools/hadoop-sls/src/main/sample-conf/sls-runner.xml * /hadoop/common/trunk/hadoop-tools/hadoop-sls/src/main/sample-conf/yarn-site.xml * /hadoop/common/trunk/hadoop-tools/hadoop-sls/src/site * /hadoop/common/trunk/hadoop-tools/hadoop-sls/src/site/apt * /hadoop/common/trunk/hadoop-tools/hadoop-sls/src/site/apt/SchedulerLoadSimulator.apt.vm * /hadoop/common/trunk/hadoop-tools/hadoop-sls/src/site/resources * /hadoop/common/trunk/hadoop-tools/hadoop-sls/src/site/resources/css * /hadoop/common/trunk/hadoop-tools/hadoop-sls/src/site/resources/css/site.css * /hadoop/common/trunk/hadoop-tools/hadoop-sls/src/site/resources/images * /hadoop/common/trunk/hadoop-tools/hadoop-sls/src/site/resources/images/sls_JVM.png * /hadoop/common/trunk/hadoop-tools/hadoop-sls/src/site/resources/images/sls_arch.png * /hadoop/common/trunk/hadoop-tools/hadoop-sls/src/site/resources/images/sls_cluster_memory.png * /hadoop/common/trunk/hadoop-tools/hadoop-sls/src/site/resources/images/sls_cluster_vcores.png * /hadoop/common/trunk/hadoop-tools/hadoop-sls/src/site/resources/images/sls_queue_allocated_memory.png * /hadoop/common/trunk/hadoop-tools/hadoop-sls/src/site/resources/images/sls_queue_allocated_vcores.png * /hadoop/common/trunk/hadoop-tools/hadoop-sls/src/site/resources/images/sls_running_apps_containers.png * /hadoop/common/trunk/hadoop-tools/hadoop-sls/src/site/resources/images/sls_scheduler_operation_timecost.png * /hadoop/common/trunk/hadoop-tools/hadoop-sls/src/site/resources/images/sls_track_job.png * /hadoop/common/trunk/hadoop-tools/hadoop-sls/src/site/resources/images/sls_track_queue.png * /hadoop/common/trunk/hadoop-tools/hadoop-sls/src/test * /hadoop/common/trunk/hadoop-tools/hadoop-sls/src/test/java * /hadoop/common/trunk/hadoop-tools/hadoop-sls/src/test/java/org * /hadoop/common/trunk/hadoop-tools/hadoop-sls/src/test/java/org/apache * /hadoop/common/trunk/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop * /hadoop/common/trunk/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn * /hadoop/common/trunk/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls * /hadoop/common/trunk/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/TestSLSRunner.java * /hadoop/common/trunk/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/scheduler * /hadoop/common/trunk/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/scheduler/TestTaskRunner.java * /hadoop/common/trunk/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/utils * /hadoop/common/trunk/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/utils/TestSLSUtils.java * /hadoop/common/trunk/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/web * /hadoop/common/trunk/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/web/TestSLSWebApp.java * /hadoop/common/trunk/hadoop-tools/hadoop-sls/src/test/resources * /hadoop/common/trunk/hadoop-tools/hadoop-sls/src/test/resources/fair-scheduler-allocation.xml * /hadoop/common/trunk/hadoop-tools/hadoop-sls/src/test/resources/fair-scheduler.xml * /hadoop/common/trunk/hadoop-tools/hadoop-sls/src/test/resources/simulate.html.template * /hadoop/common/trunk/hadoop-tools/hadoop-sls/src/test/resources/simulate.info.html.template * /hadoop/common/trunk/hadoop-tools/hadoop-sls/src/test/resources/sls-runner.xml * /hadoop/common/trunk/hadoop-tools/hadoop-sls/src/test/resources/track.html.template * /hadoop/common/trunk/hadoop-tools/hadoop-sls/src/test/resources/yarn-site.xml * /hadoop/common/trunk/hadoop-tools/hadoop-tools-dist/pom.xml * /hadoop/common/trunk/hadoop-tools/pom.xml * /hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt > Yarn Scheduler Load Simulator > ----------------------------- > > Key: YARN-1021 > URL: https://issues.apache.org/jira/browse/YARN-1021 > Project: Hadoop YARN > Issue Type: New Feature > Components: scheduler > Reporter: Wei Yan > Assignee: Wei Yan > Fix For: 2.3.0 > > Attachments: YARN-1021-demo.tar.gz, YARN-1021-images.tar.gz, > YARN-1021.patch, YARN-1021.patch, YARN-1021.patch, YARN-1021.patch, > YARN-1021.patch, YARN-1021.patch, YARN-1021.patch, YARN-1021.patch, > YARN-1021.patch, YARN-1021.patch, YARN-1021.patch, YARN-1021.patch, > YARN-1021.patch, YARN-1021.patch, YARN-1021.patch, YARN-1021.pdf > > > The Yarn Scheduler is a fertile area of interest with different > implementations, e.g., Fifo, Capacity and Fair schedulers. Meanwhile, > several optimizations are also made to improve scheduler performance for > different scenarios and workload. Each scheduler algorithm has its own set of > features, and drives scheduling decisions by many factors, such as fairness, > capacity guarantee, resource availability, etc. It is very important to > evaluate a scheduler algorithm very well before we deploy it in a production > cluster. Unfortunately, currently it is non-trivial to evaluate a scheduling > algorithm. Evaluating in a real cluster is always time and cost consuming, > and it is also very hard to find a large-enough cluster. Hence, a simulator > which can predict how well a scheduler algorithm for some specific workload > would be quite useful. > We want to build a Scheduler Load Simulator to simulate large-scale Yarn > clusters and application loads in a single machine. This would be invaluable > in furthering Yarn by providing a tool for researchers and developers to > prototype new scheduler features and predict their behavior and performance > with reasonable amount of confidence, there-by aiding rapid innovation. > The simulator will exercise the real Yarn ResourceManager removing the > network factor by simulating NodeManagers and ApplicationMasters via handling > and dispatching NM/AMs heartbeat events from within the same JVM. > To keep tracking of scheduler behavior and performance, a scheduler wrapper > will wrap the real scheduler. > The simulator will produce real time metrics while executing, including: > * Resource usages for whole cluster and each queue, which can be utilized to > configure cluster and queue's capacity. > * The detailed application execution trace (recorded in relation to simulated > time), which can be analyzed to understand/validate the scheduler behavior > (individual jobs turn around time, throughput, fairness, capacity guarantee, > etc). > * Several key metrics of scheduler algorithm, such as time cost of each > scheduler operation (allocate, handle, etc), which can be utilized by Hadoop > developers to find the code spots and scalability limits. > The simulator will provide real time charts showing the behavior of the > scheduler and its performance. > A short demo is available http://www.youtube.com/watch?v=6thLi8q0qLE, showing > how to use simulator to simulate Fair Scheduler and Capacity Scheduler. -- This message is automatically generated by JIRA. If you think it was sent incorrectly, please contact your JIRA administrators For more information on JIRA, see: http://www.atlassian.com/software/jira