Repository: storm Updated Branches: refs/heads/master 318c9e26b -> 09bfca337
http://git-wip-us.apache.org/repos/asf/storm/blob/f5ad9288/storm-core/src/ui/public/templates/supervisor-page-template.html ---------------------------------------------------------------------- diff --git a/storm-core/src/ui/public/templates/supervisor-page-template.html b/storm-core/src/ui/public/templates/supervisor-page-template.html new file mode 100644 index 0000000..9e6fadc --- /dev/null +++ b/storm-core/src/ui/public/templates/supervisor-page-template.html @@ -0,0 +1,145 @@ +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<script id="supervisor-summary-template" type="text/html"> + <table class="table table-striped compact" id="supervisor-summary-table"> + <thead> + <tr> + <th> + <span data-toggle="tooltip" data-placement="top" title="The hostname reported by the remote host. (Note that this hostname is not the result of a reverse lookup at the Nimbus node.)"> + Host + </span> + </th> + <th> + <span data-toggle="tooltip" data-placement="top" title="A unique identifier given to a Supervisor when it joins the cluster."> + Supervisor Id + </span> + </th> + <th> + <span data-toggle="tooltip" data-placement="top" title="The length of time a Supervisor has been registered to the cluster."> + Uptime + </span> + </th> + <th> + <span data-toggle="tooltip" data-placement="top" title="Slots are Workers (processes)."> + Slots + </span> + </th> + <th> + <span data-toggle="tooltip" data-placement="top" title="Slots are Workers (processes)."> + Used slots + </span> + </th> + <th> + <span data-toggle="tooltip" data-placement="top" title="Memory capacity of a supervisor."> + Total Mem (MB) + </span> + </th> + <th> + <span data-toggle="tooltip" data-placement="top" title="Memory that has been allocated."> + Used Mem (MB) + </span> + </th> + {{#schedulerDisplayResource}} + <th> + <span data-toggle="tooltip" data-placement="top" title="CPU capacity of a supervisor. Every 100 means one core."> + Total CPU (%) + </span> + </th> + <th> + <span data-toggle="tooltip" data-placement="top" title="CPU that has been allocated. Every 100 means one core"> + Used CPU (%) + </span> + </th> + {{/schedulerDisplayResource}} + <th> + <span data-toggle="tooltip" data-placement="top" title="Version"> + Version + </span> + </th> + </tr> + </thead> + <tbody> + {{#supervisors}} + <tr> + <td><a href="/supervisor.html?host={{host}}">{{host}} (<a href="{{logLink}}" title="View log">log</a>)</a></td> + <td><a href="/supervisor.html?id={{id}}">{{id}}</td> + <td>{{uptime}}</td> + <td>{{slotsTotal}}</td> + <td>{{slotsUsed}}</td> + <td>{{totalMem}}</td> + <td>{{usedMem}}</td> + {{#schedulerDisplayResource}} + <td>{{totalCpu}}</td> + <td>{{usedCpu}}</td> + {{/schedulerDisplayResource}} + <td>{{version}}</td> + </tr> + {{/supervisors}} + </tbody> + </table> +</script> +<script id="worker-stats-template" type="text/html"> + <table class="table table-striped compact" id="worker-stats-table"> + <thead> + <tr> + <th class="header headerSortDown"> + <span data-toggle="tooltip" data-placement="top" title="The name given to the topology by when it was submitted. Click the name to view the Topology's information."> + Topology Name + </span> + </th> + <th class="header"> + <span data-original-title="The hostname reported by the remote host. (Note that this hostname is not the result of a reverse lookup at the Nimbus node.)" data-toggle="tooltip" data-placement="top"> + Host + </span> + </th> + <th class="header"> + <span data-toggle="tooltip" data-placement="top" title="The port number used by the Worker. Click on the port number to open the logviewer page for this Worker."> + Port + </span> + </th> + <th class="header"> + <span data-toggle="tooltip" data-placement="top" title="The length of time a Worker has been alive."> + Uptime + </span> + </th> + <th class="header"> + <span data-toggle="tooltip" data-placement="top" title="The number of executors."> + Num executors + </span> + </th> + <th class="header"> + <span data-toggle="tooltip" data-placement="top" title="Assigned Total Memory by Scheduler."> + Assigned Mem (MB) + </span> + </th> + {{#schedulerDisplayResource}} + <th class="header"> + <span data-toggle="tooltip" data-placement="top" title="Assigned Total CPU by Scheduler. Every 100 means 1 core."> + Assigned CPU (%) + </span> + </th> + {{/schedulerDisplayResource}} + <th class="header"> + <span data-toggle="tooltip" data-placement="top" title="The components running in this worker and the number of tasks per component."> + Components + </span> + </th> + </tr></thead> + <tbody> + </tbody> + </table> +</script> http://git-wip-us.apache.org/repos/asf/storm/blob/f5ad9288/storm-core/src/ui/public/templates/topology-page-template.html ---------------------------------------------------------------------- diff --git a/storm-core/src/ui/public/templates/topology-page-template.html b/storm-core/src/ui/public/templates/topology-page-template.html index da1796c..ebdf3aa 100644 --- a/storm-core/src/ui/public/templates/topology-page-template.html +++ b/storm-core/src/ui/public/templates/topology-page-template.html @@ -29,45 +29,52 @@ </span> </th> <th> - <span data-toggle="tooltip" data-placement="above" title="The user that submitted the Topology, if authentication is enabled."> + <span data-toggle="tooltip" data-placement="top" title="The user that submitted the Topology, if authentication is enabled."> Owner </span> </th> <th> - <span data-toggle="tooltip" data-placement="above" title="The status can be one of ACTIVE, INACTIVE, KILLED, or REBALANCING."> + <span data-toggle="tooltip" data-placement="top" title="The status can be one of ACTIVE, INACTIVE, KILLED, or REBALANCING."> Status </span> </th> <th> - <span data-toggle="tooltip" data-placement="above" title="The time since the Topology was submitted."> + <span data-toggle="tooltip" data-placement="top" title="The time since the Topology was submitted."> Uptime </span> </th> <th> - <span data-toggle="tooltip" data-placement="above" title="The number of Workers (processes)."> + <span data-toggle="tooltip" data-placement="top" title="The number of Workers (processes)."> Num workers </span> </th> <th> - <span data-toggle="tooltip" data-placement="above" title="Executors are threads in a Worker process."> + <span data-toggle="tooltip" data-placement="top" title="Executors are threads in a Worker process."> Num executors </span> </th> <th> - <span data-toggle="tooltip" data-placement="above" title="A Task is an instance of a Bolt or Spout. The number of Tasks is almost always equal to the number of Executors."> + <span data-toggle="tooltip" data-placement="top" title="A Task is an instance of a Bolt or Spout. The number of Tasks is almost always equal to the number of Executors."> Num tasks </span> </th> <th> - <span cdata-toggle="tooltip" data-placement="above" title="Number of nimbus hosts on which this topology's code is replicated. "> + <span cdata-toggle="tooltip" data-placement="top" title="Number of nimbus hosts on which this topology's code is replicated. "> Replication count </span> </th> <th> - <span data-toggle="tooltip" data-placement="above" title="Assigned Total Memory by Scheduler."> + <span data-toggle="tooltip" data-placement="top" title="Assigned Total Memory by Scheduler."> Assigned Mem (MB) </span> </th> + {{#schedulerDisplayResource}} + <th> + <span data-toggle="tooltip" data-placement="top" title="Assigned Total CPU by Scheduler. Every 100 means 1 core."> + Assigned CPU (%) + </span> + </th> + {{/schedulerDisplayResource}} <th> <span data-toggle="tooltip" data-placement="left" title="This shows information from the scheduler about the latest attempt to schedule the Topology on the cluster."> Scheduler Info @@ -87,6 +94,9 @@ <td>{{tasksTotal}}</td> <td>{{replicationCount}}</td> <td>{{assignedTotalMem}}</td> + {{#schedulerDisplayResource}} + <td>{{assignedCpu}}</td> + {{/schedulerDisplayResource}} <td>{{schedulerInfo}}</td> </tr> </tbody> @@ -95,48 +105,48 @@ <script id="topology-resources-template" type="text/html"> <table id="topology-resources-table" class="table compact"> <thead> - <tr> - <th> - <span data-toggle="tooltip" data-placement="right" title="The name given to the topology by when it was submitted."> - Name - </span> - </th> - <th> - <span data-toggle="tooltip" data-placement="right" title="The unique ID given to a Topology each time it is launched."> - Id - </span> - </th> - <th> - <span data-toggle="tooltip" data-placement="above" title="Requested Total On-Heap Memory by User."> - Requested On-Heap Memory (MB) - </span> - </th> - <th> - <span data-toggle="tooltip" data-placement="above" title="Assigned Total On-Heap Memory by Scheduler."> - Assigned On-Heap Memory (MB) - </span> - </th> - <th> - <span data-toggle="tooltip" data-placement="above" title="Requested Total Off-Heap Memory by User."> - Requested Off-Heap Memory (MB) - </span> - </th> - <th> - <span data-toggle="tooltip" data-placement="above" title="Assigned Total Off-Heap Memory by Scheduler."> - Assigned Off-Heap Memory (MB) - </span> - </th> - <th> - <span data-toggle="tooltip" data-placement="above" title="Requested Total CPU by User. Every 100 means 1 core."> - Requested CPU (%) - </span> - </th> - <th> - <span data-toggle="tooltip" data-placement="left" title="Assigned Total CPU by Scheduler. Every 100 means 1 core."> - Assigned CPU (%) - </span> - </th> - </tr> + <tr> + <th> + <span data-toggle="tooltip" data-placement="right" title="The name given to the topology by when it was submitted."> + Name + </span> + </th> + <th> + <span data-toggle="tooltip" data-placement="right" title="The unique ID given to a Topology each time it is launched."> + Id + </span> + </th> + <th> + <span data-toggle="tooltip" data-placement="top" title="Requested Total On-Heap Memory by User."> + Requested On-Heap Memory (MB) + </span> + </th> + <th> + <span data-toggle="tooltip" data-placement="top" title="Assigned Total On-Heap Memory by Scheduler."> + Assigned On-Heap Memory (MB) + </span> + </th> + <th> + <span data-toggle="tooltip" data-placement="top" title="Requested Total Off-Heap Memory by User."> + Requested Off-Heap Memory (MB) + </span> + </th> + <th> + <span data-toggle="tooltip" data-placement="top" title="Assigned Total Off-Heap Memory by Scheduler."> + Assigned Off-Heap Memory (MB) + </span> + </th> + <th> + <span data-toggle="tooltip" data-placement="top" title="Requested Total CPU by User. Every 100 means 1 core."> + Requested CPU (%) + </span> + </th> + <th> + <span data-toggle="tooltip" data-placement="left" title="Assigned Total CPU by Scheduler. Every 100 means 1 core."> + Assigned CPU (%) + </span> + </th> + </tr> </thead> <tbody> <tr> @@ -163,22 +173,22 @@ </span> </th> <th> - <span data-toggle="tooltip" data-placement="above" title="The number of Tuples emitted."> + <span data-toggle="tooltip" data-placement="top" title="The number of Tuples emitted."> Emitted </span> </th> <th> - <span data-toggle="tooltip" data-placement="above" title="The number of Tuples emitted that sent to one or more bolts."> + <span data-toggle="tooltip" data-placement="top" title="The number of Tuples emitted that sent to one or more bolts."> Transferred </span> </th> <th> - <span data-toggle="tooltip" data-placement="above" title="The average time a Tuple "tree" takes to be completely processed by the Topology. A value of 0 is expected if no acking is done."> + <span data-toggle="tooltip" data-placement="top" title="The average time a Tuple "tree" takes to be completely processed by the Topology. A value of 0 is expected if no acking is done."> Complete latency (ms) </span> </th> <th> - <span data-toggle="tooltip" data-placement="above" title="The number of Tuple "trees" successfully processed. A value of 0 is expected if no acking is done."> + <span data-toggle="tooltip" data-placement="top" title="The number of Tuple "trees" successfully processed. A value of 0 is expected if no acking is done."> Acked </span> </th> @@ -215,22 +225,22 @@ </span> </th> <th> - <span data-toggle="tooltip" data-placement="above" title="Topic"> + <span data-toggle="tooltip" data-placement="top" title="Topic"> Topic </span> </th> <th> - <span data-toggle="tooltip" data-placement="above" title="Partition"> + <span data-toggle="tooltip" data-placement="top" title="Partition"> Partition </span> </th> <th> - <span data-toggle="tooltip" data-placement="above" title="Latest Offset"> + <span data-toggle="tooltip" data-placement="top" title="Latest Offset"> Latest Offset </span> </th> <th> - <span data-toggle="tooltip" data-placement="above" title="Offset of last spout message successfully acked"> + <span data-toggle="tooltip" data-placement="top" title="Offset of last spout message successfully acked"> Spout Committed Offset </span> </th> @@ -267,7 +277,7 @@ </span> </th> <th> - <span data-toggle="tooltip" data-placement="above" title="Type of spout"> + <span data-toggle="tooltip" data-placement="top" title="Type of spout"> Type </span> </th> @@ -353,32 +363,32 @@ </span> </th> <th class="header"> - <span data-toggle="tooltip" data-placement="above" title="A Task is an instance of a Bolt or Spout. The number of Tasks is almost always equal to the number of Executors."> + <span data-toggle="tooltip" data-placement="top" title="A Task is an instance of a Bolt or Spout. The number of Tasks is almost always equal to the number of Executors."> Tasks </span> </th> <th class="header"> - <span data-toggle="tooltip" data-placement="above" title="The number of Tuples emitted."> + <span data-toggle="tooltip" data-placement="top" title="The number of Tuples emitted."> Emitted </span> </th> <th class="header"> - <span data-toggle="tooltip" data-placement="above" title="The number of Tuples emitted that sent to one or more bolts."> + <span data-toggle="tooltip" data-placement="top" title="The number of Tuples emitted that sent to one or more bolts."> Transferred </span> </th> <th class="header"> - <span data-toggle="tooltip" data-placement="above" title="The average time a Tuple "tree" takes to be completely processed by the Topology. A value of 0 is expected if no acking is done."> + <span data-toggle="tooltip" data-placement="top" title="The average time a Tuple "tree" takes to be completely processed by the Topology. A value of 0 is expected if no acking is done."> Complete latency (ms) </span> </th> <th class="header"> - <span data-toggle="tooltip" data-placement="above" title="The number of Tuple "trees" successfully processed. A value of 0 is expected if no acking is done."> + <span data-toggle="tooltip" data-placement="top" title="The number of Tuple "trees" successfully processed. A value of 0 is expected if no acking is done."> Acked </span> </th> <th class="header"> - <span data-toggle="tooltip" data-placement="above" title="The number of Tuple "trees" that were explicitly failed or timed out before acking was completed. A value of 0 is expected if no acking is done."> + <span data-toggle="tooltip" data-placement="top" title="The number of Tuple "trees" that were explicitly failed or timed out before acking was completed. A value of 0 is expected if no acking is done."> Failed </span> </th> @@ -432,42 +442,42 @@ </span> </th> <th class="header"> - <span data-toggle="tooltip" data-placement="above" title="A Task is an instance of a Bolt or Spout. The number of Tasks is almost always equal to the number of Executors."> + <span data-toggle="tooltip" data-placement="top" title="A Task is an instance of a Bolt or Spout. The number of Tasks is almost always equal to the number of Executors."> Tasks </span> </th> <th class="header"> - <span data-toggle="tooltip" data-placement="above" title="The number of Tuples emitted."> + <span data-toggle="tooltip" data-placement="top" title="The number of Tuples emitted."> Emitted </span> </th> <th class="header"> - <span data-toggle="tooltip" data-placement="above" title="The number of Tuples emitted that sent to one or more bolts."> + <span data-toggle="tooltip" data-placement="top" title="The number of Tuples emitted that sent to one or more bolts."> Transferred </span> </th> <th class="header"> - <span data-original-title="If this is around 1.0, the corresponding Bolt is running as fast as it can, so you may want to increase the Bolt's parallelism. This is (number executed * average execute latency) / measurement time." data-toggle="tooltip" data-placement="above"> + <span data-original-title="If this is around 1.0, the corresponding Bolt is running as fast as it can, so you may want to increase the Bolt's parallelism. This is (number executed * average execute latency) / measurement time." data-toggle="tooltip" data-placement="top"> Capacity (last 10m) </span> </th> <th class="header"> - <span data-toggle="tooltip" data-placement="above" title="The average time a Tuple spends in the execute method. The execute method may complete without sending an Ack for the tuple."> + <span data-toggle="tooltip" data-placement="top" title="The average time a Tuple spends in the execute method. The execute method may complete without sending an Ack for the tuple."> Execute latency (ms) </span> </th> <th class="header"> - <span data-toggle="tooltip" data-placement="above" title="The number of incoming Tuples processed."> + <span data-toggle="tooltip" data-placement="top" title="The number of incoming Tuples processed."> Executed </span> </th> <th class="header"> - <span data-toggle="tooltip" data-placement="above" title="The average time it takes to Ack a Tuple after it is first received. Bolts that join, aggregate or batch may not Ack a tuple until a number of other Tuples have been received."> + <span data-toggle="tooltip" data-placement="top" title="The average time it takes to Ack a Tuple after it is first received. Bolts that join, aggregate or batch may not Ack a tuple until a number of other Tuples have been received."> Process latency (ms) </span> </th> <th class="header"> - <span data-toggle="tooltip" data-placement="above" title="The number of Tuples acknowledged by this Bolt."> + <span data-toggle="tooltip" data-placement="top" title="The number of Tuples acknowledged by this Bolt."> Acked </span> </th> @@ -512,7 +522,57 @@ {{/bolts}} </tbody> </script> - +<script id="worker-stats-template" type="text/html"> + <h2>Worker Resources</h2> + <table class="table table-striped compact" id="worker-stats-table"> + <thead> + <tr> + <th class="header headerSortDown"> + <span data-original-title="The hostname reported by the remote host. (Note that this hostname is not the result of a reverse lookup at the Nimbus node.)" data-toggle="tooltip" data-placement="top"> + Host + </span> + </th> + <th class="header"> + <span data-toggle="tooltip" data-placement="top" title="A unique identifier given to a Supervisor when it joins the cluster."> + Supervisor Id + </span> + </th> + <th class="header"> + <span data-toggle="tooltip" data-placement="top" title="The port number used by the Worker. Click on the port number to open the logviewer page for this Worker."> + Port + </span> + </th> + <th class="header"> + <span data-toggle="tooltip" data-placement="top" title="The length of time a Worker has been alive."> + Uptime + </span> + </th> + <th class="header"> + <span data-toggle="tooltip" data-placement="top" title="The number of executors"> + Num executors + </span> + </th> + <th class="header"> + <span data-toggle="tooltip" data-placement="top" title="Assigned Total Memory by Scheduler."> + Assigned Mem (MB) + </span> + </th> + {{#schedulerDisplayResource}} + <th class="header"> + <span data-toggle="tooltip" data-placement="top" title="Assigned Total CPU by Scheduler. Every 100 means 1 core."> + Assigned CPU (%) + </span> + </th> + {{/schedulerDisplayResource}} + <th class="header"> + <span data-toggle="tooltip" data-placement="top" title="The components running in this worker and the number of tasks per component."> + Components + </span> + </th> + </tr></thead> + <tbody> + </tbody> +</script> <script id="topology-actions-template" type="text/html"> <h2>Topology actions</h2> <p id="topology-actions"> http://git-wip-us.apache.org/repos/asf/storm/blob/f5ad9288/storm-core/src/ui/public/topology.html ---------------------------------------------------------------------- diff --git a/storm-core/src/ui/public/topology.html b/storm-core/src/ui/public/topology.html index 0f7cf9a..9b60dd1 100644 --- a/storm-core/src/ui/public/topology.html +++ b/storm-core/src/ui/public/topology.html @@ -79,6 +79,9 @@ <div id="bolt-stats" class="col-md-12"></div> </div> <div class="row"> + <div id="worker-stats" class="col-md-12"></div> + </div> + <div class="row"> <div id="topology-visualization" class="col-md-12"></div> </div> <div class="row"> @@ -283,6 +286,7 @@ $(document).ready(function() { var topologySpoutsLag = $("#topology-spouts-lag"); var spoutStats = $("#spout-stats"); var boltStats = $("#bolt-stats"); + var workerStats = $("#worker-stats"); var config = $("#topology-configuration"); var topologyActions = $("#topology-actions"); var topologyVisualization = $("#topology-visualization") @@ -331,6 +335,12 @@ $(document).ready(function() { ] }); + jsError(function() { + workerStats.append(Mustache.render($(template).filter("#worker-stats-template").html(),response)); + makeTopologyWorkerStatsTable (response, '#worker-stats-table', '#worker-stats'); + }); + + jsError(function() { topologyVisualization.append(Mustache.render($(template).filter("#topology-visualization-template").html(), response)); $("#show-hide-visualization").click(function () { show_visualization(null) }); @@ -438,9 +448,9 @@ $(document).ready(function() { } } }); + }); }}); }); - }); </script> </html> http://git-wip-us.apache.org/repos/asf/storm/blob/f5ad9288/storm-core/test/clj/org/apache/storm/nimbus_test.clj ---------------------------------------------------------------------- diff --git a/storm-core/test/clj/org/apache/storm/nimbus_test.clj b/storm-core/test/clj/org/apache/storm/nimbus_test.clj index 96e629d..be60396 100644 --- a/storm-core/test/clj/org/apache/storm/nimbus_test.clj +++ b/storm-core/test/clj/org/apache/storm/nimbus_test.clj @@ -33,7 +33,7 @@ (:import [org.apache.storm.generated Credentials NotAliveException SubmitOptions TopologyInitialStatus TopologyStatus AlreadyAliveException KillOptions RebalanceOptions InvalidTopologyException AuthorizationException - LogConfig LogLevel LogLevelAction]) + LogConfig LogLevel LogLevelAction Assignment NodeInfo]) (:import [java.util HashMap]) (:import [java.io File]) (:import [org.apache.storm.utils Time Utils Utils$UptimeComputer ConfigUtils IPredicate StormCommonInstaller] @@ -1357,7 +1357,45 @@ nimbus/check-authorization! [1 2 3] expected-name expected-conf expected-operation) (verify-first-call-args-for-indices - nimbus/try-read-storm-topology [0] "fake-id")))))))))) + nimbus/try-read-storm-topology [0] "fake-id")))))) + + (testing "getSupervisorPageInfo only calls check-authorization as getTopology" + (let [expected-operation "getTopology" + assignment (doto (Assignment.) + (.set_executor_node_port {[1 1] (NodeInfo. "super1" #{1}), + [2 2] (NodeInfo. "super2" #{2})})) + clojurified-assignment (clojurify-assignment assignment) + topo-assignment {expected-name assignment} + check-auth-state (atom []) + mock-check-authorization (fn [nimbus storm-name storm-conf operation] + (swap! check-auth-state conj {:nimbus nimbus + :storm-name storm-name + :storm-conf storm-conf + :operation operation}))] + (stubbing [nimbus/check-authorization! mock-check-authorization + nimbus/try-read-storm-conf expected-conf + nimbus/try-read-storm-topology nil + nimbus/get-clojurified-task-info {} + nimbus/all-supervisor-info {"super1" {:hostname "host1", :meta [1234], :uptime-secs 123} + "super2" {:hostname "host2", :meta [1234], :uptime-secs 123}} + clojurify-assignment clojurified-assignment + nimbus/topology-assignments topo-assignment + nimbus/get-launch-time-secs 0] + ;; not called yet + (verify-call-times-for nimbus/check-authorization! 0) + (.getSupervisorPageInfo nimbus "super1" nil true) + + ;; afterwards, it should get called twice + (verify-call-times-for nimbus/check-authorization! 2) + (let [first-call (nth @check-auth-state 0) + second-call (nth @check-auth-state 1)] + (is (= expected-name (:storm-name first-call))) + (is (= expected-conf (:storm-conf first-call))) + (is (= "getTopology" (:operation first-call))) + + (is (= expected-name (:storm-name second-call))) + (is (= expected-conf (:storm-conf second-call))) + (is (= "getSupervisorPageInfo" (:operation second-call))))))))))) (deftest test-nimbus-iface-getTopology-methods-throw-correctly (with-local-cluster [cluster] @@ -1414,7 +1452,8 @@ :status {:type bogus-type}} } ] - (stubbing [nimbus/nimbus-topology-bases bogus-bases + (stubbing [nimbus/get-resources-for-topology nil + nimbus/nimbus-topology-bases bogus-bases nimbus/get-blob-replication-count 1] (let [topos (.get_topologies (.getClusterInfo nimbus))] ; The number of topologies in the summary is correct. @@ -1769,3 +1808,37 @@ (is (= (count @hb-cache) 2)) (is (contains? @hb-cache "topo1")) (is (contains? @hb-cache "topo2")))))) + + +(deftest user-topologies-for-supervisor + (let [assignment (doto (Assignment.) + (.set_executor_node_port {[1 1] (NodeInfo. "super1" #{1}), + [2 2] (NodeInfo. "super2" #{2})})) + assignment2 (doto (Assignment.) + (.set_executor_node_port {[1 1] (NodeInfo. "super2" #{2}), + [2 2] (NodeInfo. "super2" #{2})})) + assignments {"topo1" assignment, "topo2" assignment2}] + (stubbing [nimbus/is-authorized? true] + (let [topos1 (nimbus/user-and-supervisor-topos nil nil nil assignments "super1") + topos2 (nimbus/user-and-supervisor-topos nil nil nil assignments "super2")] + (is (= (list "topo1") (:supervisor-topologies topos1))) + (is (= #{"topo1"} (:user-topologies topos1))) + (is (= (list "topo1" "topo2") (:supervisor-topologies topos2))) + (is (= #{"topo1" "topo2"} (:user-topologies topos2))))))) + +(defn- mock-check-auth + [nimbus conf blob-store op topo-name] + (= topo-name "authorized")) + +(deftest user-topologies-for-supervisor-with-unauthorized-user + (let [assignment (doto (Assignment.) + (.set_executor_node_port {[1 1] (NodeInfo. "super1" #{1}), + [2 2] (NodeInfo. "super2" #{2})})) + assignment2 (doto (Assignment.) + (.set_executor_node_port {[1 1] (NodeInfo. "super1" #{2}), + [2 2] (NodeInfo. "super2" #{2})})) + assignments {"topo1" assignment, "authorized" assignment2}] + (stubbing [nimbus/is-authorized? mock-check-auth] + (let [topos (nimbus/user-and-supervisor-topos nil nil nil assignments "super1")] + (is (= (list "topo1" "authorized") (:supervisor-topologies topos))) + (is (= #{"authorized"} (:user-topologies topos))))))) http://git-wip-us.apache.org/repos/asf/storm/blob/f5ad9288/storm-core/test/jvm/org/apache/storm/stats/TestStatsUtil.java ---------------------------------------------------------------------- diff --git a/storm-core/test/jvm/org/apache/storm/stats/TestStatsUtil.java b/storm-core/test/jvm/org/apache/storm/stats/TestStatsUtil.java new file mode 100644 index 0000000..78b032c --- /dev/null +++ b/storm-core/test/jvm/org/apache/storm/stats/TestStatsUtil.java @@ -0,0 +1,266 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.storm.stats; + +import java.util.Arrays; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.HashMap; + +import org.apache.storm.scheduler.WorkerSlot; +import org.apache.storm.generated.WorkerResources; +import org.apache.storm.generated.WorkerSummary; + +import org.junit.Before; +import org.junit.Test; +import org.junit.Assert; + +public class TestStatsUtil { + + /* + * aggWorkerStats tests + */ + private Map<Integer, String> task2Component = new HashMap<Integer, String>(); + private Map<List<Integer>, Map<String, Object>> beats = new HashMap<List<Integer>, Map<String, Object>>(); + private Map<List<Long>, List<Object>> exec2NodePort = new HashMap<List<Long>, List<Object>>(); + private Map<String, String> nodeHost = new HashMap<String, String>(); + private Map<WorkerSlot, WorkerResources> worker2Resources = new HashMap<WorkerSlot, WorkerResources>(); + + private List<Long> makeExecutorId(int firstTask, int lastTask){ + return Arrays.asList(new Long(firstTask), new Long(lastTask)); + } + + public void makeTopoInfo() { + List<Object> hostPort = new ArrayList<Object>(); + hostPort.add("node1"); + hostPort.add(new Long(1)); + + exec2NodePort.put(makeExecutorId(1,1), hostPort); + + nodeHost.put("node1", "host1"); + nodeHost.put("node2", "host2"); + nodeHost.put("node3", "host3"); + + List<Integer> exec1 = new ArrayList<Integer>(); + exec1.add(1); + exec1.add(1); + HashMap<String, Object> exec1Beat = new HashMap<String, Object>(); + exec1Beat.put("uptime", 100); + + // should not be returned since this executor is not part of the topology's assignment + List<Integer> exec2 = new ArrayList<Integer>(); + exec2.add(2); + exec2.add(4); + HashMap<String, Object> exec2Beat = new HashMap<String, Object>(); + exec2Beat.put("uptime", 200); + + Map<String, Object> beat1 = new HashMap<String, Object>(); + beat1.put("heartbeat", exec1Beat); + + Map<String, Object> beat2 = new HashMap<String, Object>(); + beat2.put("heartbeat", exec2Beat); + + beats.put(exec1, beat1); + beats.put(exec2, beat2); + + task2Component.put(1, "my-component"); + task2Component.put(2, "__sys1"); + task2Component.put(3, "__sys2"); + task2Component.put(4, "__sys3"); + task2Component.put(5, "__sys4"); + task2Component.put(6, "__sys4"); + task2Component.put(7, "my-component2"); + + WorkerResources ws1 = new WorkerResources(); + ws1.set_mem_on_heap(1); + ws1.set_mem_off_heap(2); + ws1.set_cpu(3); + worker2Resources.put(new WorkerSlot("node1", 1), ws1); + + WorkerResources ws2 = new WorkerResources(); + ws2.set_mem_on_heap(4); + ws2.set_mem_off_heap(8); + ws2.set_cpu(12); + worker2Resources.put(new WorkerSlot("node2", 2), ws2); + + WorkerResources ws3 = new WorkerResources(); + ws3.set_mem_on_heap(16); + ws3.set_mem_off_heap(32); + ws3.set_cpu(48); + worker2Resources.put(new WorkerSlot("node3", 3), ws3); + } + + private void makeTopoInfoWithSysWorker() { + makeTopoInfo(); + + List<Object> secondWorker = new ArrayList<Object>(); + secondWorker.add("node2"); + secondWorker.add(new Long(2)); + exec2NodePort.put(makeExecutorId(2, 4), secondWorker); + } + + private void makeTopoInfoWithMissingBeats() { + makeTopoInfo(); + + List<Object> thirdWorker = new ArrayList<Object>(); + thirdWorker.add("node3"); + thirdWorker.add(new Long(3)); + exec2NodePort.put(makeExecutorId(5, 7), thirdWorker); + } + + private List<WorkerSummary> checkWorkerStats(boolean includeSys, boolean userAuthorized, String filterSupervisor) { + List<WorkerSummary> summaries = + StatsUtil.aggWorkerStats("my-storm-id", "my-storm-name", + task2Component, beats, exec2NodePort, nodeHost, worker2Resources, + includeSys, userAuthorized, filterSupervisor); + for (WorkerSummary ws : summaries) { + String host = ws.get_host(); + int port = ws.get_port(); + Assert.assertEquals("my-storm-id", ws.get_topology_id()); + Assert.assertEquals("my-storm-name", ws.get_topology_name()); + boolean includeSupervisor = filterSupervisor == null || filterSupervisor.equals(host); + switch (port) { + case 1: + Assert.assertEquals("host1", ws.get_host()); + Assert.assertEquals("node1", ws.get_supervisor_id()); + Assert.assertEquals(1, ws.get_num_executors()); + Assert.assertEquals(100, ws.get_uptime_secs()); + Assert.assertEquals(1.0, ws.get_assigned_memonheap(), 0.001); + Assert.assertEquals(2.0, ws.get_assigned_memoffheap(), 0.001); + Assert.assertEquals(3.0, ws.get_assigned_cpu(), 0.001); + break; + case 2: + Assert.assertEquals("host2", ws.get_host()); + Assert.assertEquals("node2", ws.get_supervisor_id()); + Assert.assertEquals(1, ws.get_num_executors()); + Assert.assertEquals(200, ws.get_uptime_secs()); + Assert.assertEquals(4.0, ws.get_assigned_memonheap(), 0.001); + Assert.assertEquals(8.0, ws.get_assigned_memoffheap(), 0.001); + Assert.assertEquals(12.0, ws.get_assigned_cpu(), 0.001); + break; + case 3: + Assert.assertEquals("host3", ws.get_host()); + Assert.assertEquals("node3", ws.get_supervisor_id()); + Assert.assertEquals(1, ws.get_num_executors()); + // no heartbeat for this one, should be 0 + Assert.assertEquals(0, ws.get_uptime_secs()); + Assert.assertEquals(16.0, ws.get_assigned_memonheap(), 0.001); + Assert.assertEquals(32.0, ws.get_assigned_memoffheap(), 0.001); + Assert.assertEquals(48.0, ws.get_assigned_cpu(), 0.001); + break; + } + } + + // get the worker count back s.t. we can assert in each test function + return summaries; + } + + private WorkerSummary getWorkerSummaryForPort(List<WorkerSummary> summaries, int port) { + //iterate of WorkerSummary and find the one with the port + for (WorkerSummary ws : summaries) { + if (ws.get_port() == port) { + return ws; + } + } + return null; + } + + @Test + public void aggWorkerStats() { + makeTopoInfo(); + List<WorkerSummary> summaries = checkWorkerStats(true /*include sys*/, + true /*user authorized*/, + null /*filter supervisor*/); + WorkerSummary ws = getWorkerSummaryForPort(summaries, 1); + Assert.assertEquals(1, ws.get_component_to_num_tasks().size()); + Assert.assertEquals(1, ws.get_component_to_num_tasks().get("my-component").intValue()); + Assert.assertEquals(1, summaries.size()); + } + + @Test + public void aggWorkerStatsWithSystemComponents() { + makeTopoInfoWithSysWorker(); + List<WorkerSummary> summaries = checkWorkerStats(true /*include sys*/, + true /*user authorized*/, + null /*filter supervisor*/); + WorkerSummary ws = getWorkerSummaryForPort(summaries, 2); + // since we made sys components visible, the component map has all system components + Assert.assertEquals(3, ws.get_component_to_num_tasks().size()); + Assert.assertEquals(1, ws.get_component_to_num_tasks().get("__sys1").intValue()); + Assert.assertEquals(1, ws.get_component_to_num_tasks().get("__sys2").intValue()); + Assert.assertEquals(1, ws.get_component_to_num_tasks().get("__sys3").intValue()); + Assert.assertEquals(2, summaries.size()); + } + + @Test + public void aggWorkerStatsWithHiddenSystemComponents() { + makeTopoInfoWithSysWorker(); + List<WorkerSummary> summaries = checkWorkerStats(false /*DON'T include sys*/, + true /*user authorized*/, + null /*filter supervisor*/); + WorkerSummary ws1 = getWorkerSummaryForPort(summaries, 1); + WorkerSummary ws2 = getWorkerSummaryForPort(summaries, 2); + Assert.assertEquals(1, ws1.get_component_to_num_tasks().size()); + // since we made sys components hidden, the component map is empty for this worker + Assert.assertEquals(0, ws2.get_component_to_num_tasks().size()); + Assert.assertEquals(2, summaries.size()); + } + + @Test + public void aggWorkerStatsForUnauthorizedUser() { + makeTopoInfoWithSysWorker(); + List<WorkerSummary> summaries = checkWorkerStats(true /*include sys (should not matter)*/, + false /*user NOT authorized*/, + null /*filter supervisor*/); + WorkerSummary ws1 = getWorkerSummaryForPort(summaries, 1); + WorkerSummary ws2 = getWorkerSummaryForPort(summaries, 2); + // since we made user not authorized, component map is empty + Assert.assertEquals(0, ws1.get_component_to_num_tasks().size()); + Assert.assertEquals(0, ws2.get_component_to_num_tasks().size()); + Assert.assertEquals(2, summaries.size()); + } + + @Test + public void aggWorkerStatsFilterSupervisor() { + makeTopoInfoWithMissingBeats(); + List<WorkerSummary> summaries = checkWorkerStats(true /*include sys*/, + true /*user authorized*/, + "node3" /*filter supervisor*/); + WorkerSummary ws = getWorkerSummaryForPort(summaries, 3); + // only host3 should be returned given filter + Assert.assertEquals(2, ws.get_component_to_num_tasks().size()); + Assert.assertEquals(2, ws.get_component_to_num_tasks().get("__sys4").intValue()); + Assert.assertEquals(1, ws.get_component_to_num_tasks().get("my-component2").intValue()); + Assert.assertEquals(1, summaries.size()); + } + + @Test + public void aggWorkerStatsFilterSupervisorAndHideSystemComponents() { + makeTopoInfoWithMissingBeats(); + List<WorkerSummary> summaries = checkWorkerStats(false /*DON'T include sys*/, + true /*user authorized*/, + "node3" /*filter supervisor*/); + + WorkerSummary ws = getWorkerSummaryForPort(summaries, 3); + // hidden sys component + Assert.assertEquals(1, ws.get_component_to_num_tasks().size()); + Assert.assertEquals(1, ws.get_component_to_num_tasks().get("my-component2").intValue()); + Assert.assertEquals(1, summaries.size()); + } +}
