Repository: aurora Updated Branches: refs/heads/master e67c6a732 -> 9c316d324
Display reservations and persistent volumes in /offers debug http endpoint Bugs closed: AURORA-1736 Reviewed at https://reviews.apache.org/r/50052/ Project: http://git-wip-us.apache.org/repos/asf/aurora/repo Commit: http://git-wip-us.apache.org/repos/asf/aurora/commit/9c316d32 Tree: http://git-wip-us.apache.org/repos/asf/aurora/tree/9c316d32 Diff: http://git-wip-us.apache.org/repos/asf/aurora/diff/9c316d32 Branch: refs/heads/master Commit: 9c316d324c88281ba97b332a427901f69bf705df Parents: e67c6a7 Author: Mehrdad Nurolahzade <[email protected]> Authored: Sat Jul 23 18:48:48 2016 +0200 Committer: Stephan Erb <[email protected]> Committed: Sat Jul 23 18:48:48 2016 +0200 ---------------------------------------------------------------------- RELEASE-NOTES.md | 12 ++ config/legacy_untested_classes.txt | 1 - .../apache/aurora/scheduler/http/Offers.java | 93 +++-------- .../aurora/scheduler/http/OffersTest.java | 156 +++++++++++++++++++ 4 files changed, 186 insertions(+), 76 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/aurora/blob/9c316d32/RELEASE-NOTES.md ---------------------------------------------------------------------- diff --git a/RELEASE-NOTES.md b/RELEASE-NOTES.md index 29d224d..ca98d7a 100644 --- a/RELEASE-NOTES.md +++ b/RELEASE-NOTES.md @@ -1,3 +1,15 @@ +0.16.0 (Not yet released) +====== + +### New/updated: + +- The `/offers` endpoint has been modified to display attributes of resource offers as received + from Mesos. This has affected rendering of some of the existing attributes. Furthermore, it now + dumps additional offer attributes including [reservations](http://mesos.apache.org/documentation/latest/reservation/) + and [persistent volumes](http://mesos.apache.org/documentation/latest/persistent-volume/). + +### Deprecations and removals: + 0.15.0 ====== http://git-wip-us.apache.org/repos/asf/aurora/blob/9c316d32/config/legacy_untested_classes.txt ---------------------------------------------------------------------- diff --git a/config/legacy_untested_classes.txt b/config/legacy_untested_classes.txt index 1ea2183..ee4d3d7 100644 --- a/config/legacy_untested_classes.txt +++ b/config/legacy_untested_classes.txt @@ -26,7 +26,6 @@ org/apache/aurora/scheduler/http/api/security/Kerberos5Realm org/apache/aurora/scheduler/http/JerseyTemplateServlet org/apache/aurora/scheduler/http/LogConfig org/apache/aurora/scheduler/http/LogConfig$LoggerConfig -org/apache/aurora/scheduler/http/Offers org/apache/aurora/scheduler/http/Offers$1 org/apache/aurora/scheduler/http/Offers$2 org/apache/aurora/scheduler/http/Offers$3 http://git-wip-us.apache.org/repos/asf/aurora/blob/9c316d32/src/main/java/org/apache/aurora/scheduler/http/Offers.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/aurora/scheduler/http/Offers.java b/src/main/java/org/apache/aurora/scheduler/http/Offers.java index 80f0824..f22ca6e 100644 --- a/src/main/java/org/apache/aurora/scheduler/http/Offers.java +++ b/src/main/java/org/apache/aurora/scheduler/http/Offers.java @@ -13,9 +13,9 @@ */ package org.apache.aurora.scheduler.http; -import java.util.Map; import java.util.Objects; - +import java.util.stream.Collectors; +import java.util.stream.StreamSupport; import javax.inject.Inject; import javax.ws.rs.GET; import javax.ws.rs.Path; @@ -23,18 +23,13 @@ import javax.ws.rs.Produces; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; -import com.google.common.base.Function; -import com.google.common.collect.FluentIterable; -import com.google.common.collect.ImmutableMap; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.PropertyNamingStrategy; +import com.hubspot.jackson.datatype.protobuf.ProtobufModule; -import org.apache.aurora.scheduler.HostOffer; import org.apache.aurora.scheduler.offers.OfferManager; -import org.apache.mesos.Protos.Attribute; -import org.apache.mesos.Protos.ExecutorID; -import org.apache.mesos.Protos.Resource; -import org.apache.mesos.Protos.Value.Range; - -import static org.apache.mesos.Protos.Offer; /** * Servlet that exposes resource offers that the scheduler is currently retaining. @@ -43,10 +38,15 @@ import static org.apache.mesos.Protos.Offer; public class Offers { private final OfferManager offerManager; + private final ObjectMapper mapper; @Inject Offers(OfferManager offerManager) { this.offerManager = Objects.requireNonNull(offerManager); + mapper = new ObjectMapper() + .registerModule(new ProtobufModule()) + .setPropertyNamingStrategy(PropertyNamingStrategy.CAMEL_CASE_TO_LOWER_CASE_WITH_UNDERSCORES) + .setSerializationInclusion(JsonInclude.Include.NON_NULL); } /** @@ -56,69 +56,12 @@ public class Offers { */ @GET @Produces(MediaType.APPLICATION_JSON) - public Response getOffers() { + public Response getOffers() throws JsonProcessingException { return Response.ok( - FluentIterable.from(offerManager.getOffers()).transform(TO_BEAN).toList()).build(); + mapper.writeValueAsString( + StreamSupport.stream(offerManager.getOffers().spliterator(), false) + .map(o -> o.getOffer()) + .collect(Collectors.toList()))) + .build(); } - - private static final Function<ExecutorID, String> EXECUTOR_ID_TOSTRING = ExecutorID::getValue; - - private static final Function<Range, Object> RANGE_TO_BEAN = - range -> range.getBegin() + "-" + range.getEnd(); - - private static final Function<Attribute, Object> ATTRIBUTE_TO_BEAN = - attr -> { - ImmutableMap.Builder<String, Object> builder = ImmutableMap.builder(); - builder.put("name", attr.getName()); - if (attr.hasScalar()) { - builder.put("scalar", attr.getScalar().getValue()); - } - if (attr.hasRanges()) { - builder.put("ranges", immutable(attr.getRanges().getRangeList(), RANGE_TO_BEAN)); - } - if (attr.hasSet()) { - builder.put("set", attr.getSet().getItemList()); - } - if (attr.hasText()) { - builder.put("text", attr.getText().getValue()); - } - return builder.build(); - }; - - private static final Function<Resource, Object> RESOURCE_TO_BEAN = - resource -> { - ImmutableMap.Builder<String, Object> builder = ImmutableMap.builder(); - builder.put("name", resource.getName()); - if (resource.hasScalar()) { - builder.put("scalar", resource.getScalar().getValue()); - } - if (resource.hasRanges()) { - builder.put("ranges", immutable(resource.getRanges().getRangeList(), RANGE_TO_BEAN)); - } - if (resource.hasSet()) { - builder.put("set", resource.getSet().getItemList()); - } - if (resource.hasRevocable()) { - builder.put("revocable", "true"); - } - return builder.build(); - }; - - private static <A, B> Iterable<B> immutable(Iterable<A> iterable, Function<A, B> transform) { - return FluentIterable.from(iterable).transform(transform).toList(); - } - - private static final Function<HostOffer, Map<String, ?>> TO_BEAN = - hostOffer -> { - Offer offer = hostOffer.getOffer(); - return ImmutableMap.<String, Object>builder() - .put("id", offer.getId().getValue()) - .put("framework_id", offer.getFrameworkId().getValue()) - .put("slave_id", offer.getSlaveId().getValue()) - .put("hostname", offer.getHostname()) - .put("resources", immutable(offer.getResourcesList(), RESOURCE_TO_BEAN)) - .put("attributes", immutable(offer.getAttributesList(), ATTRIBUTE_TO_BEAN)) - .put("executor_ids", immutable(offer.getExecutorIdsList(), EXECUTOR_ID_TOSTRING)) - .build(); - }; } http://git-wip-us.apache.org/repos/asf/aurora/blob/9c316d32/src/test/java/org/apache/aurora/scheduler/http/OffersTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/aurora/scheduler/http/OffersTest.java b/src/test/java/org/apache/aurora/scheduler/http/OffersTest.java new file mode 100644 index 0000000..9e35732 --- /dev/null +++ b/src/test/java/org/apache/aurora/scheduler/http/OffersTest.java @@ -0,0 +1,156 @@ +/** + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.aurora.scheduler.http; + +import java.util.List; +import javax.servlet.http.HttpServletResponse; +import javax.ws.rs.core.Response; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.PropertyNamingStrategy; +import com.google.common.collect.ImmutableSet; +import com.google.gson.Gson; +import com.hubspot.jackson.datatype.protobuf.ProtobufModule; + +import org.apache.aurora.common.testing.easymock.EasyMockTest; +import org.apache.aurora.gen.HostAttributes; +import org.apache.aurora.scheduler.HostOffer; +import org.apache.aurora.scheduler.offers.OfferManager; +import org.apache.aurora.scheduler.storage.entities.IHostAttributes; +import org.apache.mesos.Protos; +import org.junit.Before; +import org.junit.Test; + +import static org.apache.aurora.gen.MaintenanceMode.NONE; +import static org.easymock.EasyMock.expect; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +public class OffersTest extends EasyMockTest { + private Offers offers; + private OfferManager offerManager; + + @Before + public void setUp() { + offerManager = createMock(OfferManager.class); + offers = new Offers(offerManager); + } + + @Test + public void testNoOffers() throws Exception { + expect(offerManager.getOffers()).andReturn(ImmutableSet.of()); + + control.replay(); + + Response response = offers.getOffers(); + assertEquals(HttpServletResponse.SC_OK, response.getStatus()); + assertTrue(toList(response.getEntity().toString()).isEmpty()); + } + + @Test + public void testOneOffer() throws Exception { + HostOffer offer = new HostOffer( + Protos.Offer.newBuilder() + .setId(Protos.OfferID.newBuilder().setValue("offer_id")) + .setFrameworkId(Protos.FrameworkID.newBuilder().setValue("framework_id")) + .setSlaveId(Protos.SlaveID.newBuilder().setValue("slave_id")) + .setHostname("host_name") + .addResources(Protos.Resource.newBuilder() + .setName("cpus") + .setType(Protos.Value.Type.SCALAR) + .setScalar(Protos.Value.Scalar.newBuilder().setValue(1.0).build()) + .setReservation(Protos.Resource.ReservationInfo.newBuilder() + .setLabels(Protos.Labels.newBuilder() + .addLabels(Protos.Label.newBuilder() + .setKey("key") + .setValue("value")) + .build()) + .build()) + .build()) + .addResources(Protos.Resource.newBuilder() + .setName("mem") + .setType(Protos.Value.Type.SCALAR) + .setScalar(Protos.Value.Scalar.newBuilder().setValue(128.0).build()) + .setReservation(Protos.Resource.ReservationInfo.newBuilder().build()) + .build()) + .addResources(Protos.Resource.newBuilder() + .setName("disk") + .setType(Protos.Value.Type.SCALAR) + .setScalar(Protos.Value.Scalar.newBuilder().setValue(128.0).build()) + .setReservation(Protos.Resource.ReservationInfo.newBuilder() + .setLabels(Protos.Labels.newBuilder() + .addLabels(Protos.Label.newBuilder() + .setKey("key")) + .build()) + .build()) + .setDisk(Protos.Resource.DiskInfo.newBuilder() + .setPersistence(Protos.Resource.DiskInfo.Persistence.newBuilder() + .setId("volume") + .build()) + .setVolume(Protos.Volume.newBuilder() + .setContainerPath("path") + .setMode(Protos.Volume.Mode.RW) + .setImage(Protos.Image.newBuilder() + .setType(Protos.Image.Type.DOCKER) + .setDocker(Protos.Image.Docker.newBuilder() + .setName("image") + .build()) + .build()) + .build()) + .setSource(Protos.Resource.DiskInfo.Source.newBuilder() + .setType(Protos.Resource.DiskInfo.Source.Type.PATH) + .setPath(Protos.Resource.DiskInfo.Source.Path.newBuilder() + .setRoot("root") + .build()) + .build()) + .build()) + .build()) + .addResources(Protos.Resource.newBuilder() + .setName("gpus") + .setType(Protos.Value.Type.SCALAR) + .setScalar(Protos.Value.Scalar.newBuilder().setValue(4.0).build()) + .build()) + .addResources(Protos.Resource.newBuilder() + .setName("ports") + .setType(Protos.Value.Type.RANGES) + .setRanges(Protos.Value.Ranges.newBuilder() + .addRange(Protos.Value.Range.newBuilder() + .setBegin(31000) + .setEnd(32000) + .build()) + .build()) + .build()) + .build(), + IHostAttributes.build(new HostAttributes().setMode(NONE))); + + expect(offerManager.getOffers()).andReturn(ImmutableSet.of(offer)); + + control.replay(); + + Response response = offers.getOffers(); + assertEquals(HttpServletResponse.SC_OK, response.getStatus()); + ObjectMapper mapper = new ObjectMapper() + .registerModule(new ProtobufModule()) + .setPropertyNamingStrategy( + PropertyNamingStrategy.CAMEL_CASE_TO_LOWER_CASE_WITH_UNDERSCORES); + assertEquals( + offer.getOffer(), + mapper.readValue(response.getEntity().toString(), Protos.Offer.class)); + } + + private static List toList(String json) { + return new Gson().fromJson(json, List.class); + } + +}
