Repository: aurora Updated Branches: refs/heads/master f729fd4d2 -> 663d13f23
Create an ELB-friendly endpoint to detect the leading scheduler. Reviewed at https://reviews.apache.org/r/45511/ Project: http://git-wip-us.apache.org/repos/asf/aurora/repo Commit: http://git-wip-us.apache.org/repos/asf/aurora/commit/663d13f2 Tree: http://git-wip-us.apache.org/repos/asf/aurora/tree/663d13f2 Diff: http://git-wip-us.apache.org/repos/asf/aurora/diff/663d13f2 Branch: refs/heads/master Commit: 663d13f232a7de73892729449c576ad6ff234aaa Parents: f729fd4 Author: Ashwin Murthy <[email protected]> Authored: Tue Apr 5 09:52:40 2016 -0700 Committer: Bill Farner <[email protected]> Committed: Tue Apr 5 09:52:40 2016 -0700 ---------------------------------------------------------------------- RELEASE-NOTES.md | 4 +- docs/reference/scheduler-endpoints.md | 19 ++++ .../scheduler/http/JettyServerModule.java | 1 + .../aurora/scheduler/http/LeaderHealth.java | 93 ++++++++++++++++++++ src/main/resources/scheduler/assets/index.html | 1 + .../aurora/scheduler/http/LeaderHealthTest.java | 63 +++++++++++++ .../sh/org/apache/aurora/e2e/test_end_to_end.sh | 1 + 7 files changed, 181 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/aurora/blob/663d13f2/RELEASE-NOTES.md ---------------------------------------------------------------------- diff --git a/RELEASE-NOTES.md b/RELEASE-NOTES.md index 1df0345..2f935da 100644 --- a/RELEASE-NOTES.md +++ b/RELEASE-NOTES.md @@ -1,9 +1,11 @@ -0.13.0 +0.13.0 (Not yet released) ------ ### New/updated: - Upgraded Mesos to 0.26.0 +- Added a new health endpoint (/leaderhealth) which can be used for load balancer health + checks to always forward requests to the leading scheduler. - Added a new `aurora job add` client command to scale out an existing job. - Upgraded the scheduler ZooKeeper client from 3.4.6 to 3.4.8. - Added support for dedicated constraints not exclusive to a particular role. http://git-wip-us.apache.org/repos/asf/aurora/blob/663d13f2/docs/reference/scheduler-endpoints.md ---------------------------------------------------------------------- diff --git a/docs/reference/scheduler-endpoints.md b/docs/reference/scheduler-endpoints.md new file mode 100644 index 0000000..d302e90 --- /dev/null +++ b/docs/reference/scheduler-endpoints.md @@ -0,0 +1,19 @@ +# HTTP endpoints + +There are a number of HTTP endpoints that the Aurora scheduler exposes. These allow various +operational tasks to be performed on the scheduler. Below is the list of all such endpoints +and a brief explanation of what they do. + +## Leader health +The /leaderhealth endpoint enables performing health checks on the scheduler instances inorder +to forward requests to the leading scheduler. This is typically used by a load balancer such as +HAProxy or AWS ELB. + +When a HTTP GET request is issued on this endpoint, it responds as follows: + +- If the instance that received the GET request is the leading scheduler, a HTTP status code of + 200 (OK) is returned. +- If the instance that received the GET request is not the leading scheduler but a leader does + exist, a HTTP status code of 503 (SERVICE_UNAVAILABLE) is returned. +- If no leader currently exists or the leader is unknown, a HTTP status code of 502 + (BAD_GATEWAY) is returned. \ No newline at end of file http://git-wip-us.apache.org/repos/asf/aurora/blob/663d13f2/src/main/java/org/apache/aurora/scheduler/http/JettyServerModule.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/aurora/scheduler/http/JettyServerModule.java b/src/main/java/org/apache/aurora/scheduler/http/JettyServerModule.java index df649ff..a5446e3 100644 --- a/src/main/java/org/apache/aurora/scheduler/http/JettyServerModule.java +++ b/src/main/java/org/apache/aurora/scheduler/http/JettyServerModule.java @@ -216,6 +216,7 @@ public class JettyServerModule extends AbstractModule { .put(ContentionPrinter.class, "contention") .put(Cron.class, "cron") .put(HealthHandler.class, "health") + .put(LeaderHealth.class, "leaderhealth") .put(Locks.class, "locks") .put(LogConfig.class, "logconfig") .put(Maintenance.class, "maintenance") http://git-wip-us.apache.org/repos/asf/aurora/blob/663d13f2/src/main/java/org/apache/aurora/scheduler/http/LeaderHealth.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/aurora/scheduler/http/LeaderHealth.java b/src/main/java/org/apache/aurora/scheduler/http/LeaderHealth.java new file mode 100644 index 0000000..8a7886f --- /dev/null +++ b/src/main/java/org/apache/aurora/scheduler/http/LeaderHealth.java @@ -0,0 +1,93 @@ +/** + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.aurora.scheduler.http; + +import java.util.Map; +import java.util.Objects; + +import javax.inject.Inject; +import javax.ws.rs.GET; +import javax.ws.rs.Path; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Joiner; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Maps; + +import org.apache.aurora.common.collections.Pair; + +import static javax.servlet.http.HttpServletResponse.SC_BAD_GATEWAY; +import static javax.servlet.http.HttpServletResponse.SC_OK; +import static javax.servlet.http.HttpServletResponse.SC_SERVICE_UNAVAILABLE; + +import static org.apache.aurora.scheduler.http.LeaderRedirect.LeaderStatus; +import static org.apache.aurora.scheduler.http.LeaderRedirect.LeaderStatus.LEADING; +import static org.apache.aurora.scheduler.http.LeaderRedirect.LeaderStatus.NOT_LEADING; +import static org.apache.aurora.scheduler.http.LeaderRedirect.LeaderStatus.NO_LEADER; + +/** + * Servlet that exposes the leader health. It is meant to be used by load balancers such as + * ELB to always forward to the leading scheduler. + */ +@Path("/leaderhealth") +public class LeaderHealth { + + private static final Map<LeaderStatus, Pair<Integer, String>> RESPONSE_CODES = ImmutableMap.of( + LEADING, Pair.of(SC_OK, "This is the leading scheduler."), + NOT_LEADING, Pair.of(SC_SERVICE_UNAVAILABLE, "Another instance is the leading scheduler."), + NO_LEADER, Pair.of(SC_BAD_GATEWAY, "There is no leading scheduler or the leader is unknown.") + ); + + private static final String HELP = generateHelpText(); + + private final LeaderRedirect leaderRedirect; + + @Inject + LeaderHealth(LeaderRedirect leaderRedirect) { + this.leaderRedirect = Objects.requireNonNull(leaderRedirect); + } + + @VisibleForTesting + static String getHelpText(LeaderStatus status) { + return "Current status: " + RESPONSE_CODES.get(status).getSecond() + + "\n\n" + + HELP; + } + + /** + * Gets the leadership status of the scheduler. + * + * @return HTTP response. + */ + @GET + public Response get() { + LeaderStatus leaderStatus = leaderRedirect.getLeaderStatus(); + return Response.status(RESPONSE_CODES.get(leaderStatus).getFirst()) + .type(MediaType.TEXT_PLAIN_TYPE) + .entity(getHelpText(leaderStatus)) + .build(); + } + + private static String generateHelpText() { + Map<Integer, String> details = Maps.transformValues( + Maps.uniqueIndex(RESPONSE_CODES.values(), Pair::getFirst), + Pair::getSecond); + + return "This endpoint can indicate to a load balancer which among a group of schedulers " + + "is leading.\n\nThe response codes are:\n" + + Joiner.on("\n").withKeyValueSeparator(": ").join(details); + } +} http://git-wip-us.apache.org/repos/asf/aurora/blob/663d13f2/src/main/resources/scheduler/assets/index.html ---------------------------------------------------------------------- diff --git a/src/main/resources/scheduler/assets/index.html b/src/main/resources/scheduler/assets/index.html index 861d551..5679b19 100644 --- a/src/main/resources/scheduler/assets/index.html +++ b/src/main/resources/scheduler/assets/index.html @@ -76,6 +76,7 @@ <h4> Scheduler Internals</h4> <ul class="list-unstyled group-summary"> <li><a href='/contention'>/contention</a></li> + <li><a href='/leaderhealth'>/leaderhealth</a></li> <li><a href='/locks'>/locks</a></li> <li><a href='/logconfig'>/logconfig</a></li> <li><a href='/services'>/services</a></li> http://git-wip-us.apache.org/repos/asf/aurora/blob/663d13f2/src/test/java/org/apache/aurora/scheduler/http/LeaderHealthTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/aurora/scheduler/http/LeaderHealthTest.java b/src/test/java/org/apache/aurora/scheduler/http/LeaderHealthTest.java new file mode 100644 index 0000000..d21a38e --- /dev/null +++ b/src/test/java/org/apache/aurora/scheduler/http/LeaderHealthTest.java @@ -0,0 +1,63 @@ +/** + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.aurora.scheduler.http; + +import javax.servlet.http.HttpServletResponse; +import javax.ws.rs.core.Response; + +import org.apache.aurora.common.net.pool.DynamicHostSet.MonitorException; +import org.apache.aurora.common.testing.easymock.EasyMockTest; +import org.apache.aurora.scheduler.http.LeaderRedirect.LeaderStatus; +import org.junit.Before; +import org.junit.Test; + +import static org.easymock.EasyMock.expect; +import static org.junit.Assert.assertEquals; + +public class LeaderHealthTest extends EasyMockTest { + + private LeaderRedirect leaderRedirect; + private LeaderHealth leaderHealth; + + @Before + public void setUp() throws MonitorException { + leaderRedirect = createMock(LeaderRedirect.class); + leaderHealth = new LeaderHealth(leaderRedirect); + } + + private void expectResponse(LeaderStatus status, int responseCode) { + expect(leaderRedirect.getLeaderStatus()).andReturn(status); + + control.replay(); + + Response response = leaderHealth.get(); + assertEquals(responseCode, response.getStatus()); + assertEquals(LeaderHealth.getHelpText(status), response.getEntity().toString()); + } + + @Test + public void testLeader() throws Exception { + expectResponse(LeaderStatus.LEADING, HttpServletResponse.SC_OK); + } + + @Test + public void testNotLeader() throws Exception { + expectResponse(LeaderStatus.NOT_LEADING, HttpServletResponse.SC_SERVICE_UNAVAILABLE); + } + + @Test + public void testNoLeaders() throws Exception { + expectResponse(LeaderStatus.NO_LEADER, HttpServletResponse.SC_BAD_GATEWAY); + } +} http://git-wip-us.apache.org/repos/asf/aurora/blob/663d13f2/src/test/sh/org/apache/aurora/e2e/test_end_to_end.sh ---------------------------------------------------------------------- diff --git a/src/test/sh/org/apache/aurora/e2e/test_end_to_end.sh b/src/test/sh/org/apache/aurora/e2e/test_end_to_end.sh index 77a4c36..3471756 100755 --- a/src/test/sh/org/apache/aurora/e2e/test_end_to_end.sh +++ b/src/test/sh/org/apache/aurora/e2e/test_end_to_end.sh @@ -116,6 +116,7 @@ test_scheduler_ui() { # Check that scheduler UI pages shown base_url="localhost:8081" + check_url_live "$base_url/leaderhealth" check_url_live "$base_url/scheduler" check_url_live "$base_url/scheduler/$_role" check_url_live "$base_url/scheduler/$_role/$_env/$_job"
