This is an automated email from the ASF dual-hosted git repository.
yaliang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-heron.git
The following commit(s) were added to refs/heads/master by this push:
new 9f3def0 Add json format for dry-run (#2842)
9f3def0 is described below
commit 9f3def0238f5d845a8217bc131d0da849e38c98a
Author: Andrew Jorgensen <ajorgen...@users.noreply.github.com>
AuthorDate: Wed Apr 4 18:39:03 2018 -0400
Add json format for dry-run (#2842)
This adds JSON as an optional format for the dry-run output. JSON is
more easily machine readable than the other output formats and can be
used to make deploy decisions based on the resources required and the
resources available at deploy time.
---
.../heron/common/basics/DryRunFormatType.java | 3 +-
heron/scheduler-core/src/java/BUILD | 15 ++--
.../heron/scheduler/dryrun/JsonFormatterUtils.java | 87 ++++++++++++++++++++++
.../scheduler/dryrun/SubmitJsonDryRunRenderer.java | 44 +++++++++++
.../scheduler/dryrun/UpdateJsonDryRunRenderer.java | 57 ++++++++++++++
.../heron/scheduler/utils/DryRunRenders.java | 7 ++
heron/scheduler-core/tests/java/BUILD | 71 +++++++++---------
.../scheduler/dryrun/JsonFormatterUtilsTest.java | 70 +++++++++++++++++
.../resources/JsonFormatterUtilsExpectedJson.txt | 1 +
heron/tools/cli/src/python/args.py | 2 +-
10 files changed, 313 insertions(+), 44 deletions(-)
diff --git
a/heron/common/src/java/com/twitter/heron/common/basics/DryRunFormatType.java
b/heron/common/src/java/com/twitter/heron/common/basics/DryRunFormatType.java
index ff85d4f..407ac4c 100644
---
a/heron/common/src/java/com/twitter/heron/common/basics/DryRunFormatType.java
+++
b/heron/common/src/java/com/twitter/heron/common/basics/DryRunFormatType.java
@@ -20,7 +20,8 @@ package com.twitter.heron.common.basics;
public enum DryRunFormatType {
RAW,
TABLE,
- COLORED_TABLE;
+ COLORED_TABLE,
+ JSON;
public static DryRunFormatType getDryRunFormatType(String dryRunFormatType) {
return DryRunFormatType.valueOf(dryRunFormatType.toUpperCase());
diff --git a/heron/scheduler-core/src/java/BUILD
b/heron/scheduler-core/src/java/BUILD
index 30aa30b..b471af4 100644
--- a/heron/scheduler-core/src/java/BUILD
+++ b/heron/scheduler-core/src/java/BUILD
@@ -1,13 +1,14 @@
package(default_visibility = ["//visibility:public"])
common_deps_files = [
- "//heron/common/src/java:basics-java",
- "//heron/common/src/java:config-java",
- "//heron/common/src/java:utils-java",
- "//heron/api/src/java:classification",
- "@commons_cli_commons_cli//jar",
- "@com_google_guava_guava//jar",
- "@org_apache_commons_commons_lang3//jar",
+ "//heron/common/src/java:basics-java",
+ "//heron/common/src/java:config-java",
+ "//heron/common/src/java:utils-java",
+ "//heron/api/src/java:classification",
+ "@commons_cli_commons_cli//jar",
+ "@com_google_guava_guava//jar",
+ "@org_apache_commons_commons_lang3//jar",
+ "//third_party/java:jackson",
]
spi_deps_files = [
diff --git
a/heron/scheduler-core/src/java/com/twitter/heron/scheduler/dryrun/JsonFormatterUtils.java
b/heron/scheduler-core/src/java/com/twitter/heron/scheduler/dryrun/JsonFormatterUtils.java
new file mode 100644
index 0000000..c2c0b6c
--- /dev/null
+++
b/heron/scheduler-core/src/java/com/twitter/heron/scheduler/dryrun/JsonFormatterUtils.java
@@ -0,0 +1,87 @@
+// Copyright 2018 Twitter. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.twitter.heron.scheduler.dryrun;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ArrayNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+
+import com.twitter.heron.spi.packing.PackingPlan;
+import com.twitter.heron.spi.packing.Resource;
+
+public class JsonFormatterUtils {
+
+ private final ObjectMapper mapper;
+
+ public JsonFormatterUtils() {
+ mapper = new ObjectMapper();
+ }
+
+ public String renderPackingPlan(String topologyName, String packingClass,
+ PackingPlan packingPlan) throws
JsonProcessingException {
+ ObjectNode topLevel = mapper.createObjectNode();
+ topLevel.put("packingClass", packingClass);
+ topLevel.put("topology", topologyName);
+
+ ArrayNode containers = mapper.createArrayNode();
+
+ for (PackingPlan.ContainerPlan containerPlan :
packingPlan.getContainers()) {
+ ObjectNode planNode = mapper.createObjectNode();
+ planNode.put("taskId", containerPlan.getId());
+ containers.add(renderContainerPlan(containerPlan));
+ }
+
+ topLevel.set("containers", containers);
+
+ return mapper.writeValueAsString(topLevel);
+ }
+
+ ObjectNode renderContainerPlan(PackingPlan.ContainerPlan containerPlan) {
+ Resource requiredResources = containerPlan.getRequiredResource();
+ ObjectNode resources = mapper.createObjectNode();
+ resources.put("cpu", requiredResources.getCpu());
+ resources.put("ram", requiredResources.getRam().asBytes());
+ resources.put("disk", requiredResources.getDisk().asBytes());
+
+ ObjectNode containerNode = mapper.createObjectNode();
+ containerNode.put("id", containerPlan.getId());
+ containerNode.set("resources", resources);
+
+ ArrayNode components = mapper.createArrayNode();
+ for (PackingPlan.InstancePlan instancePlan : containerPlan.getInstances())
{
+ components.add(renderInstancePlan(instancePlan));
+ }
+
+ containerNode.set("components", components);
+
+ return containerNode;
+ }
+
+ ObjectNode renderInstancePlan(PackingPlan.InstancePlan instancePlan) {
+ Resource resources = instancePlan.getResource();
+ ObjectNode resourcesNode = mapper.createObjectNode();
+ resourcesNode.put("cpu", resources.getCpu());
+ resourcesNode.put("ram", resources.getRam().asBytes());
+ resourcesNode.put("disk", resources.getDisk().asBytes());
+
+ ObjectNode instancePlanNode = mapper.createObjectNode();
+ instancePlanNode.put("component", instancePlan.getComponentName());
+ instancePlanNode.put("id", instancePlan.getTaskId());
+ instancePlanNode.put("index", instancePlan.getComponentIndex());
+ instancePlanNode.set("resources", resourcesNode);
+
+ return instancePlanNode;
+ }
+}
diff --git
a/heron/scheduler-core/src/java/com/twitter/heron/scheduler/dryrun/SubmitJsonDryRunRenderer.java
b/heron/scheduler-core/src/java/com/twitter/heron/scheduler/dryrun/SubmitJsonDryRunRenderer.java
new file mode 100644
index 0000000..65cb6d6
--- /dev/null
+++
b/heron/scheduler-core/src/java/com/twitter/heron/scheduler/dryrun/SubmitJsonDryRunRenderer.java
@@ -0,0 +1,44 @@
+// Copyright 2016 Twitter. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.twitter.heron.scheduler.dryrun;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+
+import com.twitter.heron.spi.common.Context;
+import com.twitter.heron.spi.packing.PackingPlan;
+
+/**
+ * Dry-run renderer that renders submit dry-run response in raw format
+ */
+public class SubmitJsonDryRunRenderer {
+ private final SubmitDryRunResponse response;
+ private final JsonFormatterUtils formatter;
+
+ public SubmitJsonDryRunRenderer(SubmitDryRunResponse response) {
+ this.response = response;
+ this.formatter = new JsonFormatterUtils();
+ }
+
+ public String render() {
+ String topologyName = response.getTopology().getName();
+ String packingClassName = Context.packingClass(response.getConfig());
+ PackingPlan packingPlan = response.getPackingPlan();
+
+ try {
+ return formatter.renderPackingPlan(topologyName, packingClassName,
packingPlan);
+ } catch (JsonProcessingException e) {
+ return "ERROR: " + e.getMessage();
+ }
+ }
+}
diff --git
a/heron/scheduler-core/src/java/com/twitter/heron/scheduler/dryrun/UpdateJsonDryRunRenderer.java
b/heron/scheduler-core/src/java/com/twitter/heron/scheduler/dryrun/UpdateJsonDryRunRenderer.java
new file mode 100644
index 0000000..05bc24b
--- /dev/null
+++
b/heron/scheduler-core/src/java/com/twitter/heron/scheduler/dryrun/UpdateJsonDryRunRenderer.java
@@ -0,0 +1,57 @@
+// Copyright 2018 Twitter. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.twitter.heron.scheduler.dryrun;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+
+import com.twitter.heron.spi.common.Context;
+import com.twitter.heron.spi.packing.PackingPlan;
+
+/**
+ * Dry-run renderer that renders submit dry-run response in raw format
+ */
+public class UpdateJsonDryRunRenderer {
+ private final UpdateDryRunResponse response;
+
+ public UpdateJsonDryRunRenderer(UpdateDryRunResponse response) {
+ this.response = response;
+ }
+
+ public String render() {
+ StringBuilder builder = new StringBuilder();
+ JsonFormatterUtils newFormatter = new JsonFormatterUtils();
+ JsonFormatterUtils oldFormatter = new JsonFormatterUtils();
+
+ String topologyName = response.getTopology().getName();
+ String packingClassName = Context.packingClass(response.getConfig());
+ PackingPlan newPackingPlan = response.getPackingPlan();
+ PackingPlan oldPackingPlan = response.getOldPackingPlan();
+
+ try {
+ String newPackingPlanJson = newFormatter.renderPackingPlan(topologyName,
packingClassName,
+ newPackingPlan);
+ String oldPackingPlanJson = oldFormatter.renderPackingPlan(topologyName,
packingClassName,
+ oldPackingPlan);
+
+ builder.append("New packing plan:\n");
+ builder.append(newPackingPlanJson).append("\n");
+ builder.append("Old packing plan:\n");
+ builder.append(oldPackingPlanJson).append("\n");
+
+ return builder.toString();
+ } catch (JsonProcessingException e) {
+ return "ERROR: " + e.getMessage();
+ }
+ }
+}
diff --git
a/heron/scheduler-core/src/java/com/twitter/heron/scheduler/utils/DryRunRenders.java
b/heron/scheduler-core/src/java/com/twitter/heron/scheduler/utils/DryRunRenders.java
index d62fdfa..fde99f5 100644
---
a/heron/scheduler-core/src/java/com/twitter/heron/scheduler/utils/DryRunRenders.java
+++
b/heron/scheduler-core/src/java/com/twitter/heron/scheduler/utils/DryRunRenders.java
@@ -14,10 +14,13 @@
package com.twitter.heron.scheduler.utils;
import com.twitter.heron.common.basics.DryRunFormatType;
+
import com.twitter.heron.scheduler.dryrun.SubmitDryRunResponse;
+import com.twitter.heron.scheduler.dryrun.SubmitJsonDryRunRenderer;
import com.twitter.heron.scheduler.dryrun.SubmitRawDryRunRenderer;
import com.twitter.heron.scheduler.dryrun.SubmitTableDryRunRenderer;
import com.twitter.heron.scheduler.dryrun.UpdateDryRunResponse;
+import com.twitter.heron.scheduler.dryrun.UpdateJsonDryRunRenderer;
import com.twitter.heron.scheduler.dryrun.UpdateRawDryRunRenderer;
import com.twitter.heron.scheduler.dryrun.UpdateTableDryRunRenderer;
@@ -31,6 +34,8 @@ public final class DryRunRenders {
return new SubmitTableDryRunRenderer(response, false).render();
case COLORED_TABLE:
return new SubmitTableDryRunRenderer(response, true).render();
+ case JSON:
+ return new SubmitJsonDryRunRenderer(response).render();
default: throw new IllegalArgumentException(
String.format("Unexpected rendering format: %s", formatType));
}
@@ -44,6 +49,8 @@ public final class DryRunRenders {
return new UpdateTableDryRunRenderer(response, false).render();
case COLORED_TABLE:
return new UpdateTableDryRunRenderer(response, true).render();
+ case JSON:
+ return new UpdateJsonDryRunRenderer(response).render();
default: throw new IllegalArgumentException(
String.format("Unexpected rendering format: %s", formatType));
}
diff --git a/heron/scheduler-core/tests/java/BUILD
b/heron/scheduler-core/tests/java/BUILD
index 1fd405a..a4fc8e3 100644
--- a/heron/scheduler-core/tests/java/BUILD
+++ b/heron/scheduler-core/tests/java/BUILD
@@ -11,47 +11,48 @@ common_deps_files = [
]
spi_deps_files = [
- "//heron/spi/src/java:common-spi-java",
- "//heron/spi/src/java:statemgr-spi-java",
- "//heron/spi/src/java:uploader-spi-java",
- "//heron/spi/src/java:packing-spi-java",
- "//heron/spi/src/java:scheduler-spi-java",
- "//heron/spi/src/java:utils-spi-java",
+ "//heron/spi/src/java:common-spi-java",
+ "//heron/spi/src/java:statemgr-spi-java",
+ "//heron/spi/src/java:uploader-spi-java",
+ "//heron/spi/src/java:packing-spi-java",
+ "//heron/spi/src/java:scheduler-spi-java",
+ "//heron/spi/src/java:utils-spi-java",
]
scheduler_deps_files = \
- common_deps_files + \
- heron_java_proto_files() + \
- spi_deps_files
+ common_deps_files + \
+ heron_java_proto_files() + \
+ spi_deps_files
java_library(
- name = "scheduler-core-tests",
- srcs = glob(["**/*.java"]),
- deps = scheduler_deps_files + [
- "//heron/statemgrs/src/java:null-statemgr-java",
- "//heron/schedulers/src/java:null-scheduler-java",
- "//heron/packing/src/java:roundrobin-packing",
- ],
+ name = "scheduler-core-tests",
+ srcs = glob(["**/*.java"]),
+ deps = scheduler_deps_files + [
+ "//heron/statemgrs/src/java:null-statemgr-java",
+ "//heron/schedulers/src/java:null-scheduler-java",
+ "//heron/packing/src/java:roundrobin-packing",
+ ],
)
java_tests(
- test_classes = [
- "com.twitter.heron.scheduler.LaunchRunnerTest",
- "com.twitter.heron.scheduler.RuntimeManagerRunnerTest",
- "com.twitter.heron.scheduler.RuntimeManagerMainTest",
- "com.twitter.heron.scheduler.SubmitterMainTest",
- "com.twitter.heron.scheduler.SchedulerMainTest",
- "com.twitter.heron.scheduler.UpdateTopologyManagerTest",
- "com.twitter.heron.scheduler.client.LibrarySchedulerClientTest",
- "com.twitter.heron.scheduler.client.HttpServiceSchedulerClientTest",
- "com.twitter.heron.scheduler.client.SchedulerClientFactoryTest",
- "com.twitter.heron.scheduler.dryrun.SubmitDryRunRenderTest",
- "com.twitter.heron.scheduler.dryrun.UpdateDryRunRenderTest",
- "com.twitter.heron.scheduler.server.SchedulerServerTest",
- "com.twitter.heron.scheduler.utils.LauncherUtilsTest",
- "com.twitter.heron.scheduler.utils.SchedulerUtilsTest",
- ],
- runtime_deps = [ ":scheduler-core-tests" ],
- resources = ["//heron/scheduler-core/tests/resources:scheduler-resources"],
- size = "small",
+ size = "small",
+ resources = ["//heron/scheduler-core/tests/resources:scheduler-resources"],
+ test_classes = [
+ "com.twitter.heron.scheduler.LaunchRunnerTest",
+ "com.twitter.heron.scheduler.RuntimeManagerRunnerTest",
+ "com.twitter.heron.scheduler.RuntimeManagerMainTest",
+ "com.twitter.heron.scheduler.SubmitterMainTest",
+ "com.twitter.heron.scheduler.SchedulerMainTest",
+ "com.twitter.heron.scheduler.UpdateTopologyManagerTest",
+ "com.twitter.heron.scheduler.client.LibrarySchedulerClientTest",
+ "com.twitter.heron.scheduler.client.HttpServiceSchedulerClientTest",
+ "com.twitter.heron.scheduler.client.SchedulerClientFactoryTest",
+ "com.twitter.heron.scheduler.dryrun.SubmitDryRunRenderTest",
+ "com.twitter.heron.scheduler.dryrun.UpdateDryRunRenderTest",
+ "com.twitter.heron.scheduler.dryrun.JsonFormatterUtilsTest",
+ "com.twitter.heron.scheduler.server.SchedulerServerTest",
+ "com.twitter.heron.scheduler.utils.LauncherUtilsTest",
+ "com.twitter.heron.scheduler.utils.SchedulerUtilsTest",
+ ],
+ runtime_deps = [":scheduler-core-tests"],
)
diff --git
a/heron/scheduler-core/tests/java/com/twitter/heron/scheduler/dryrun/JsonFormatterUtilsTest.java
b/heron/scheduler-core/tests/java/com/twitter/heron/scheduler/dryrun/JsonFormatterUtilsTest.java
new file mode 100644
index 0000000..492efbb
--- /dev/null
+++
b/heron/scheduler-core/tests/java/com/twitter/heron/scheduler/dryrun/JsonFormatterUtilsTest.java
@@ -0,0 +1,70 @@
+// Copyright 2018 Twitter. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.twitter.heron.scheduler.dryrun;
+
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.util.HashSet;
+import java.util.Set;
+
+import org.apache.commons.io.IOUtils;
+import org.junit.Before;
+import org.junit.Test;
+
+import com.twitter.heron.common.basics.Pair;
+import com.twitter.heron.spi.packing.PackingPlan;
+import com.twitter.heron.spi.utils.PackingTestUtils;
+
+import static com.twitter.heron.spi.packing.PackingPlan.ContainerPlan;
+import static junit.framework.TestCase.assertEquals;
+
+public class JsonFormatterUtilsTest {
+
+ private PackingPlan plan;
+
+ @Before
+ public void setUp() throws Exception {
+ final String COMPONENT_A = "exclaim1";
+ final String COMPONENT_B = "word";
+ ContainerPlan containerPlanA = PackingTestUtils.testContainerPlan(
+ 1, new Pair<>(COMPONENT_A, 1),
+ new Pair<>(COMPONENT_A, 3),
+ new Pair<>(COMPONENT_B, 5));
+ ContainerPlan containerPlanB = PackingTestUtils.testContainerPlan(
+ 2, new Pair<>(COMPONENT_A, 2),
+ new Pair<>(COMPONENT_A, 4),
+ new Pair<>(COMPONENT_B, 6));
+ Set<ContainerPlan> containerPlans = new HashSet<>();
+ containerPlans.add(containerPlanA);
+ containerPlans.add(containerPlanB);
+ plan = new PackingPlan("A", containerPlans);
+ }
+
+ @Test
+ public void testRenderPackingPlan() throws Exception {
+ JsonFormatterUtils utils = new JsonFormatterUtils();
+ String packingPlanJson = utils.renderPackingPlan("test-topology",
+ "com.twitter.heron.packing.roundrobin.RoundRobingPacking", plan);
+
+ String filename =
"/heron/scheduler-core/tests/resources/JsonFormatterUtilsExpectedJson.txt";
+ InputStream stream =
JsonFormatterUtils.class.getResourceAsStream(filename);
+ if (stream == null) {
+ throw new RuntimeException("Expected json file cannot be found or
opened.");
+ }
+ // Input might contain UTF-8 character, so we read stream with UTF-8
decoding
+ String expectedJson = IOUtils.toString(stream, StandardCharsets.UTF_8);
+
+ assertEquals(expectedJson, packingPlanJson);
+ }
+}
diff --git
a/heron/scheduler-core/tests/resources/JsonFormatterUtilsExpectedJson.txt
b/heron/scheduler-core/tests/resources/JsonFormatterUtilsExpectedJson.txt
new file mode 100644
index 0000000..db9b92c
--- /dev/null
+++ b/heron/scheduler-core/tests/resources/JsonFormatterUtilsExpectedJson.txt
@@ -0,0 +1 @@
+{"packingClass":"com.twitter.heron.packing.roundrobin.RoundRobingPacking","topology":"test-topology","containers":[{"id":1,"resources":{"cpu":6.0,"ram":7516192768,"disk":7516192768},"components":[{"component":"word","id":5,"index":1,"resources":{"cpu":1.5,"ram":2147483648,"disk":3}},{"component":"exclaim1","id":3,"index":1,"resources":{"cpu":1.5,"ram":2147483648,"disk":3}},{"component":"exclaim1","id":1,"index":1,"resources":{"cpu":1.5,"ram":2147483648,"disk":3}}]},{"id":2,"resources":{"
[...]
\ No newline at end of file
diff --git a/heron/tools/cli/src/python/args.py
b/heron/tools/cli/src/python/args.py
index d70622e..b170357 100644
--- a/heron/tools/cli/src/python/args.py
+++ b/heron/tools/cli/src/python/args.py
@@ -167,7 +167,7 @@ def add_dry_run(parser):
:return:
'''
default_format = 'table'
- resp_formats = ['raw', 'table', 'colored_table']
+ resp_formats = ['raw', 'table', 'colored_table', 'json']
available_options = ', '.join(['%s' % opt for opt in resp_formats])
def dry_run_resp_format(value):
--
To stop receiving notification emails like this one, please contact
yali...@apache.org.
