This is an automated email from the ASF dual-hosted git repository.
nwang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-heron.git
The following commit(s) were added to refs/heads/master by this push:
new 22a57f5 Modularize packing-related constraints and exceptions (#3186)
22a57f5 is described below
commit 22a57f552c96648f60db528be099d3f03423e4e1
Author: Xiaoyao Qian <[email protected]>
AuthorDate: Sun Feb 3 18:52:48 2019 -0800
Modularize packing-related constraints and exceptions (#3186)
* modular constraints and exceptions
* address comment
---
heron/packing/src/java/BUILD | 2 +-
.../binpacking/FirstFitDecreasingPacking.java | 2 +-
.../apache/heron/packing/builder/Container.java | 4 +-
.../heron/packing/builder/PackingPlanBuilder.java | 2 +-
.../InstanceConstraint.java} | 18 ++----
.../packing/constraints/MinRamConstraint.java | 38 +++++++++++++
.../PackingConstraint.java} | 20 +++----
.../packing/constraints/ResourceConstraint.java | 65 ++++++++++++++++++++++
.../ConstraintViolationException.java} | 26 ++++++---
.../MinResourceNotSatisfiedException.java} | 10 ++--
.../ResourceExceededException.java | 4 +-
.../roundrobin/ResourceCompliantRRPacking.java | 2 +-
.../apache/heron/packing/CommonPackingTests.java | 1 +
.../apache/heron/packing/PackingTestHelper.java | 1 +
.../packing/builder/PackingPlanBuilderTest.java | 2 +-
.../apache/heron/packing/builder/ScorerTest.java | 2 +-
16 files changed, 149 insertions(+), 50 deletions(-)
diff --git a/heron/packing/src/java/BUILD b/heron/packing/src/java/BUILD
index a20954e..6e6640d 100644
--- a/heron/packing/src/java/BUILD
+++ b/heron/packing/src/java/BUILD
@@ -41,7 +41,7 @@ java_library(
java_library(
name='builder',
- srcs =
glob(["**/packing/builder/*.java","**/packing/ResourceExceededException.java"]),
+ srcs =
glob(["**/packing/builder/*.java","**/packing/constraints/*.java","**/packing/exceptions/*.java"]),
deps = heron_java_proto_files() + [
"@com_google_guava_guava//jar",
"//heron/api/src/java:classification",
diff --git
a/heron/packing/src/java/org/apache/heron/packing/binpacking/FirstFitDecreasingPacking.java
b/heron/packing/src/java/org/apache/heron/packing/binpacking/FirstFitDecreasingPacking.java
index 36bd124..d889abe 100644
---
a/heron/packing/src/java/org/apache/heron/packing/binpacking/FirstFitDecreasingPacking.java
+++
b/heron/packing/src/java/org/apache/heron/packing/binpacking/FirstFitDecreasingPacking.java
@@ -30,13 +30,13 @@ import org.apache.heron.api.generated.TopologyAPI;
import org.apache.heron.api.utils.TopologyUtils;
import org.apache.heron.common.basics.ByteAmount;
import org.apache.heron.packing.RamRequirement;
-import org.apache.heron.packing.ResourceExceededException;
import org.apache.heron.packing.builder.Container;
import org.apache.heron.packing.builder.ContainerIdScorer;
import org.apache.heron.packing.builder.HomogeneityScorer;
import org.apache.heron.packing.builder.InstanceCountScorer;
import org.apache.heron.packing.builder.PackingPlanBuilder;
import org.apache.heron.packing.builder.Scorer;
+import org.apache.heron.packing.exceptions.ResourceExceededException;
import org.apache.heron.packing.utils.PackingUtils;
import org.apache.heron.spi.common.Config;
import org.apache.heron.spi.common.Context;
diff --git
a/heron/packing/src/java/org/apache/heron/packing/builder/Container.java
b/heron/packing/src/java/org/apache/heron/packing/builder/Container.java
index 6d2c6a5..4ee84cb 100644
--- a/heron/packing/src/java/org/apache/heron/packing/builder/Container.java
+++ b/heron/packing/src/java/org/apache/heron/packing/builder/Container.java
@@ -24,7 +24,7 @@ import java.util.HashSet;
import com.google.common.base.Optional;
import org.apache.heron.common.basics.ByteAmount;
-import org.apache.heron.packing.ResourceExceededException;
+import org.apache.heron.packing.exceptions.ResourceExceededException;
import org.apache.heron.packing.utils.PackingUtils;
import org.apache.heron.spi.packing.PackingException;
import org.apache.heron.spi.packing.PackingPlan;
@@ -185,7 +185,7 @@ public class Container {
*
* @return a Resource object that describes the used CPU, RAM and disk in
the container.
*/
- private Resource getTotalUsedResources() {
+ public Resource getTotalUsedResources() {
ByteAmount usedRam = ByteAmount.ZERO;
double usedCpuCores = 0;
ByteAmount usedDisk = ByteAmount.ZERO;
diff --git
a/heron/packing/src/java/org/apache/heron/packing/builder/PackingPlanBuilder.java
b/heron/packing/src/java/org/apache/heron/packing/builder/PackingPlanBuilder.java
index 07c4629..633a0f4 100644
---
a/heron/packing/src/java/org/apache/heron/packing/builder/PackingPlanBuilder.java
+++
b/heron/packing/src/java/org/apache/heron/packing/builder/PackingPlanBuilder.java
@@ -38,7 +38,7 @@ import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Optional;
import org.apache.heron.common.basics.ByteAmount;
-import org.apache.heron.packing.ResourceExceededException;
+import org.apache.heron.packing.exceptions.ResourceExceededException;
import org.apache.heron.packing.utils.PackingUtils;
import org.apache.heron.spi.packing.InstanceId;
import org.apache.heron.spi.packing.PackingException;
diff --git
a/heron/packing/src/java/org/apache/heron/packing/ResourceExceededException.java
b/heron/packing/src/java/org/apache/heron/packing/constraints/InstanceConstraint.java
similarity index 66%
copy from
heron/packing/src/java/org/apache/heron/packing/ResourceExceededException.java
copy to
heron/packing/src/java/org/apache/heron/packing/constraints/InstanceConstraint.java
index 95a7912..d7ddeed 100644
---
a/heron/packing/src/java/org/apache/heron/packing/ResourceExceededException.java
+++
b/heron/packing/src/java/org/apache/heron/packing/constraints/InstanceConstraint.java
@@ -17,19 +17,11 @@
* under the License.
*/
-package org.apache.heron.packing;
+package org.apache.heron.packing.constraints;
-/**
- * Thrown to indicate that the resources required are not available
- */
-public class ResourceExceededException extends Exception {
- private static final long serialVersionUID = -3662229190395580148L;
-
- public ResourceExceededException(String message) {
- super(message);
- }
+import org.apache.heron.packing.exceptions.ConstraintViolationException;
+import org.apache.heron.spi.packing.PackingPlan;
- public ResourceExceededException(String message, Throwable cause) {
- super(message, cause);
- }
+public interface InstanceConstraint {
+ void test(PackingPlan.InstancePlan instancePlan) throws
ConstraintViolationException;
}
diff --git
a/heron/packing/src/java/org/apache/heron/packing/constraints/MinRamConstraint.java
b/heron/packing/src/java/org/apache/heron/packing/constraints/MinRamConstraint.java
new file mode 100644
index 0000000..39d9d5c
--- /dev/null
+++
b/heron/packing/src/java/org/apache/heron/packing/constraints/MinRamConstraint.java
@@ -0,0 +1,38 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.heron.packing.constraints;
+
+import org.apache.heron.common.basics.ByteAmount;
+import org.apache.heron.packing.exceptions.MinResourceNotSatisfiedException;
+import org.apache.heron.spi.packing.PackingPlan;
+
+public class MinRamConstraint implements InstanceConstraint {
+ private static final ByteAmount MIN_RAM_PER_INSTANCE =
ByteAmount.fromMegabytes(192);
+
+ @Override
+ public void test(PackingPlan.InstancePlan instancePlan) throws
MinResourceNotSatisfiedException {
+ if (instancePlan.getResource().getRam().lessThan(MIN_RAM_PER_INSTANCE)) {
+ throw new MinResourceNotSatisfiedException(String.format("Instance %s is
"
+ + "configured %s RAM that is less than the minimum RAM per instance
%s",
+ instancePlan.getComponentName(),
instancePlan.getResource().getRam().toString(),
+ MIN_RAM_PER_INSTANCE.toString()));
+ }
+ }
+}
diff --git
a/heron/packing/src/java/org/apache/heron/packing/ResourceExceededException.java
b/heron/packing/src/java/org/apache/heron/packing/constraints/PackingConstraint.java
similarity index 66%
copy from
heron/packing/src/java/org/apache/heron/packing/ResourceExceededException.java
copy to
heron/packing/src/java/org/apache/heron/packing/constraints/PackingConstraint.java
index 95a7912..dd21f9d 100644
---
a/heron/packing/src/java/org/apache/heron/packing/ResourceExceededException.java
+++
b/heron/packing/src/java/org/apache/heron/packing/constraints/PackingConstraint.java
@@ -17,19 +17,13 @@
* under the License.
*/
-package org.apache.heron.packing;
+package org.apache.heron.packing.constraints;
-/**
- * Thrown to indicate that the resources required are not available
- */
-public class ResourceExceededException extends Exception {
- private static final long serialVersionUID = -3662229190395580148L;
-
- public ResourceExceededException(String message) {
- super(message);
- }
+import org.apache.heron.packing.builder.Container;
+import org.apache.heron.packing.exceptions.ConstraintViolationException;
+import org.apache.heron.spi.packing.PackingPlan;
- public ResourceExceededException(String message, Throwable cause) {
- super(message, cause);
- }
+public interface PackingConstraint {
+ void test(Container container, PackingPlan.InstancePlan instancePlan)
+ throws ConstraintViolationException;
}
diff --git
a/heron/packing/src/java/org/apache/heron/packing/constraints/ResourceConstraint.java
b/heron/packing/src/java/org/apache/heron/packing/constraints/ResourceConstraint.java
new file mode 100644
index 0000000..5b7700c
--- /dev/null
+++
b/heron/packing/src/java/org/apache/heron/packing/constraints/ResourceConstraint.java
@@ -0,0 +1,65 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.heron.packing.constraints;
+
+import org.apache.heron.packing.builder.Container;
+import org.apache.heron.packing.exceptions.ResourceExceededException;
+import org.apache.heron.spi.packing.PackingPlan;
+import org.apache.heron.spi.packing.Resource;
+
+public class ResourceConstraint implements PackingConstraint {
+ @Override
+ public void test(Container container, PackingPlan.InstancePlan instancePlan)
+ throws ResourceExceededException {
+ Resource usedResource = container.getTotalUsedResources();
+ Resource newUsedResource = usedResource.plus(instancePlan.getResource());
+ Resource capacity = container.getCapacity();
+ if (capacity.getCpu() < newUsedResource.getCpu()) {
+ throw new ResourceExceededException(String.format("Adding instance %s
with %.3f cores "
+ + "to container %d with existing %.3f cores "
+ + "would exceed its capacity of %.3f cores",
+ instancePlan.getComponentName(),
+ instancePlan.getResource().getCpu(),
+ container.getContainerId(),
+ usedResource.getCpu(),
+ capacity.getCpu()));
+ }
+ if (capacity.getRam().lessThan(newUsedResource.getRam())) {
+ throw new ResourceExceededException(String.format("Adding instance %s
with %s RAM "
+ + "to container %d with existing %s RAM "
+ + "would exceed its capacity of %s RAM",
+ instancePlan.getComponentName(),
+ instancePlan.getResource().getRam().toString(),
+ container.getContainerId(),
+ usedResource.getRam().toString(),
+ capacity.getRam().toString()));
+ }
+ if (capacity.getDisk().lessThan(newUsedResource.getDisk())) {
+ throw new ResourceExceededException(String.format("Adding instance %s
with %s disk "
+ + "to container %d with existing %s disk "
+ + "would exceed its capacity of %s disk",
+ instancePlan.getComponentName(),
+ instancePlan.getResource().getDisk().toString(),
+ container.getContainerId(),
+ usedResource.getDisk().toString(),
+ capacity.getDisk().toString()));
+ }
+ }
+}
diff --git
a/heron/packing/src/java/org/apache/heron/packing/ResourceExceededException.java
b/heron/packing/src/java/org/apache/heron/packing/exceptions/ConstraintViolationException.java
similarity index 55%
copy from
heron/packing/src/java/org/apache/heron/packing/ResourceExceededException.java
copy to
heron/packing/src/java/org/apache/heron/packing/exceptions/ConstraintViolationException.java
index 95a7912..3add924 100644
---
a/heron/packing/src/java/org/apache/heron/packing/ResourceExceededException.java
+++
b/heron/packing/src/java/org/apache/heron/packing/exceptions/ConstraintViolationException.java
@@ -17,19 +17,29 @@
* under the License.
*/
-package org.apache.heron.packing;
+package org.apache.heron.packing.exceptions;
-/**
- * Thrown to indicate that the resources required are not available
- */
-public class ResourceExceededException extends Exception {
- private static final long serialVersionUID = -3662229190395580148L;
+public abstract class ConstraintViolationException extends Exception {
+ public ConstraintViolationException() {
+ super();
+ }
- public ResourceExceededException(String message) {
+ public ConstraintViolationException(String message) {
super(message);
}
- public ResourceExceededException(String message, Throwable cause) {
+ public ConstraintViolationException(String message, Throwable cause) {
super(message, cause);
}
+
+ public ConstraintViolationException(Throwable cause) {
+ super(cause);
+ }
+
+ protected ConstraintViolationException(String message,
+ Throwable cause,
+ boolean enableSuppression,
+ boolean writableStackTrace) {
+ super(message, cause, enableSuppression, writableStackTrace);
+ }
}
diff --git
a/heron/packing/src/java/org/apache/heron/packing/ResourceExceededException.java
b/heron/packing/src/java/org/apache/heron/packing/exceptions/MinResourceNotSatisfiedException.java
similarity index 76%
copy from
heron/packing/src/java/org/apache/heron/packing/ResourceExceededException.java
copy to
heron/packing/src/java/org/apache/heron/packing/exceptions/MinResourceNotSatisfiedException.java
index 95a7912..c1bf27a 100644
---
a/heron/packing/src/java/org/apache/heron/packing/ResourceExceededException.java
+++
b/heron/packing/src/java/org/apache/heron/packing/exceptions/MinResourceNotSatisfiedException.java
@@ -17,19 +17,17 @@
* under the License.
*/
-package org.apache.heron.packing;
+package org.apache.heron.packing.exceptions;
/**
* Thrown to indicate that the resources required are not available
*/
-public class ResourceExceededException extends Exception {
- private static final long serialVersionUID = -3662229190395580148L;
-
- public ResourceExceededException(String message) {
+public class MinResourceNotSatisfiedException extends
ConstraintViolationException {
+ public MinResourceNotSatisfiedException(String message) {
super(message);
}
- public ResourceExceededException(String message, Throwable cause) {
+ public MinResourceNotSatisfiedException(String message, Throwable cause) {
super(message, cause);
}
}
diff --git
a/heron/packing/src/java/org/apache/heron/packing/ResourceExceededException.java
b/heron/packing/src/java/org/apache/heron/packing/exceptions/ResourceExceededException.java
similarity index 90%
rename from
heron/packing/src/java/org/apache/heron/packing/ResourceExceededException.java
rename to
heron/packing/src/java/org/apache/heron/packing/exceptions/ResourceExceededException.java
index 95a7912..b0149ae 100644
---
a/heron/packing/src/java/org/apache/heron/packing/ResourceExceededException.java
+++
b/heron/packing/src/java/org/apache/heron/packing/exceptions/ResourceExceededException.java
@@ -17,12 +17,12 @@
* under the License.
*/
-package org.apache.heron.packing;
+package org.apache.heron.packing.exceptions;
/**
* Thrown to indicate that the resources required are not available
*/
-public class ResourceExceededException extends Exception {
+public class ResourceExceededException extends ConstraintViolationException {
private static final long serialVersionUID = -3662229190395580148L;
public ResourceExceededException(String message) {
diff --git
a/heron/packing/src/java/org/apache/heron/packing/roundrobin/ResourceCompliantRRPacking.java
b/heron/packing/src/java/org/apache/heron/packing/roundrobin/ResourceCompliantRRPacking.java
index ed7f32c..1dcb900 100644
---
a/heron/packing/src/java/org/apache/heron/packing/roundrobin/ResourceCompliantRRPacking.java
+++
b/heron/packing/src/java/org/apache/heron/packing/roundrobin/ResourceCompliantRRPacking.java
@@ -27,13 +27,13 @@ import java.util.logging.Logger;
import org.apache.heron.api.generated.TopologyAPI;
import org.apache.heron.api.utils.TopologyUtils;
import org.apache.heron.common.basics.ByteAmount;
-import org.apache.heron.packing.ResourceExceededException;
import org.apache.heron.packing.builder.Container;
import org.apache.heron.packing.builder.ContainerIdScorer;
import org.apache.heron.packing.builder.HomogeneityScorer;
import org.apache.heron.packing.builder.InstanceCountScorer;
import org.apache.heron.packing.builder.PackingPlanBuilder;
import org.apache.heron.packing.builder.Scorer;
+import org.apache.heron.packing.exceptions.ResourceExceededException;
import org.apache.heron.packing.utils.PackingUtils;
import org.apache.heron.spi.common.Config;
import org.apache.heron.spi.common.Context;
diff --git
a/heron/packing/tests/java/org/apache/heron/packing/CommonPackingTests.java
b/heron/packing/tests/java/org/apache/heron/packing/CommonPackingTests.java
index c72ce42..b74d0ef 100644
--- a/heron/packing/tests/java/org/apache/heron/packing/CommonPackingTests.java
+++ b/heron/packing/tests/java/org/apache/heron/packing/CommonPackingTests.java
@@ -30,6 +30,7 @@ import org.apache.heron.api.generated.TopologyAPI;
import org.apache.heron.common.basics.ByteAmount;
import org.apache.heron.common.basics.Pair;
import org.apache.heron.common.utils.topology.TopologyTests;
+import org.apache.heron.packing.exceptions.ResourceExceededException;
import org.apache.heron.spi.common.Config;
import org.apache.heron.spi.common.Context;
import org.apache.heron.spi.packing.IPacking;
diff --git
a/heron/packing/tests/java/org/apache/heron/packing/PackingTestHelper.java
b/heron/packing/tests/java/org/apache/heron/packing/PackingTestHelper.java
index ad354d4..530b5d2 100644
--- a/heron/packing/tests/java/org/apache/heron/packing/PackingTestHelper.java
+++ b/heron/packing/tests/java/org/apache/heron/packing/PackingTestHelper.java
@@ -22,6 +22,7 @@ package org.apache.heron.packing;
import org.apache.heron.common.basics.ByteAmount;
import org.apache.heron.common.basics.Pair;
import org.apache.heron.packing.builder.PackingPlanBuilder;
+import org.apache.heron.packing.exceptions.ResourceExceededException;
import org.apache.heron.spi.packing.InstanceId;
import org.apache.heron.spi.packing.PackingPlan;
import org.apache.heron.spi.packing.Resource;
diff --git
a/heron/packing/tests/java/org/apache/heron/packing/builder/PackingPlanBuilderTest.java
b/heron/packing/tests/java/org/apache/heron/packing/builder/PackingPlanBuilderTest.java
index 84627f7..24123ce 100644
---
a/heron/packing/tests/java/org/apache/heron/packing/builder/PackingPlanBuilderTest.java
+++
b/heron/packing/tests/java/org/apache/heron/packing/builder/PackingPlanBuilderTest.java
@@ -35,7 +35,7 @@ import org.apache.heron.common.basics.ByteAmount;
import org.apache.heron.common.basics.Pair;
import org.apache.heron.packing.AssertPacking;
import org.apache.heron.packing.PackingTestHelper;
-import org.apache.heron.packing.ResourceExceededException;
+import org.apache.heron.packing.exceptions.ResourceExceededException;
import org.apache.heron.spi.packing.InstanceId;
import org.apache.heron.spi.packing.PackingException;
import org.apache.heron.spi.packing.PackingPlan;
diff --git
a/heron/packing/tests/java/org/apache/heron/packing/builder/ScorerTest.java
b/heron/packing/tests/java/org/apache/heron/packing/builder/ScorerTest.java
index 5e549c5..5845ba3 100644
--- a/heron/packing/tests/java/org/apache/heron/packing/builder/ScorerTest.java
+++ b/heron/packing/tests/java/org/apache/heron/packing/builder/ScorerTest.java
@@ -23,7 +23,7 @@ import org.junit.Before;
import org.junit.Test;
import org.apache.heron.common.basics.ByteAmount;
-import org.apache.heron.packing.ResourceExceededException;
+import org.apache.heron.packing.exceptions.ResourceExceededException;
import org.apache.heron.spi.packing.Resource;
import org.apache.heron.spi.utils.PackingTestUtils;
