This is an automated email from the ASF dual-hosted git repository. mblow pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit d46e8b69468d1f01ba50595d1304ee8e28aa7785 Author: Ali Alsuliman <[email protected]> AuthorDate: Wed Dec 20 19:13:49 2023 -0800 [NO ISSUE][FUN] Skip job capacity assignment for completed/active requests - user model changes: no - storage format changes: no - interface changes: no Change-Id: I87c2fd9c864b29e8ec905df9d22e1e5cfa99147c Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/18021 Integration-Tests: Jenkins <[email protected]> Tested-by: Jenkins <[email protected]> Reviewed-by: Ali Alsuliman <[email protected]> Reviewed-by: Wail Alkowaileet <[email protected]> --- .../asterix/optimizer/base/AsterixOptimizationContext.java | 13 +++++++++---- .../asterix/app/function/ActiveRequestsDatasource.java | 5 +++++ .../asterix/app/function/CompletedRequestsDatasource.java | 5 +++++ .../results/misc/active_requests/active_requests.4.regex | 2 +- .../misc/completed_requests/completed_requests.3.regex | 2 +- .../asterix/metadata/declared/FunctionDataSource.java | 4 ++++ 6 files changed, 25 insertions(+), 6 deletions(-) diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/base/AsterixOptimizationContext.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/base/AsterixOptimizationContext.java index 05e7f7b8a6..733da2dcef 100644 --- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/base/AsterixOptimizationContext.java +++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/base/AsterixOptimizationContext.java @@ -24,6 +24,7 @@ import java.util.Set; import java.util.function.Predicate; import org.apache.asterix.metadata.declared.DataSource; +import org.apache.asterix.metadata.declared.FunctionDataSource; import org.apache.asterix.metadata.utils.MetadataConstants; import org.apache.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint; import org.apache.hyracks.algebricks.core.algebra.expressions.IConflictingTypeResolver; @@ -83,10 +84,10 @@ public final class AsterixOptimizationContext extends AlgebricksOptimizationCont } for (Int2ObjectMap.Entry<Set<DataSource>> me : dataSourceMap.int2ObjectEntrySet()) { int dataSourceType = me.getIntKey(); - if (dataSourceType != DataSource.Type.INTERNAL_DATASET) { + if (dataSourceType != DataSource.Type.INTERNAL_DATASET && dataSourceType != DataSource.Type.FUNCTION) { return false; } - Predicate<DataSource> dataSourceTest = AsterixOptimizationContext::isMetadata; + Predicate<DataSource> dataSourceTest = AsterixOptimizationContext::skipJobCapacityAssignment; if (!me.getValue().stream().allMatch(dataSourceTest)) { return false; } @@ -94,7 +95,11 @@ public final class AsterixOptimizationContext extends AlgebricksOptimizationCont return true; } - private static boolean isMetadata(DataSource ds) { - return MetadataConstants.METADATA_DATAVERSE_NAME.equals(ds.getId().getDataverseName()); + private static boolean skipJobCapacityAssignment(DataSource ds) { + return MetadataConstants.METADATA_DATAVERSE_NAME.equals(ds.getId().getDataverseName()) || isSkipping(ds); + } + + private static boolean isSkipping(DataSource ds) { + return (ds instanceof FunctionDataSource) && ((FunctionDataSource) ds).skipJobCapacityAssignment(); } } diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/ActiveRequestsDatasource.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/ActiveRequestsDatasource.java index 24a127ae8b..5239d0146f 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/ActiveRequestsDatasource.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/ActiveRequestsDatasource.java @@ -43,4 +43,9 @@ public class ActiveRequestsDatasource extends FunctionDataSource { AlgebricksAbsolutePartitionConstraint.randomLocation(locations.getLocations()); return new ClientRequestsFunction(randomLocation, ClientRequestsRequest.RequestType.RUNNING); } + + @Override + public boolean skipJobCapacityAssignment() { + return true; + } } diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/CompletedRequestsDatasource.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/CompletedRequestsDatasource.java index 0bfbff1bf1..f02af21955 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/CompletedRequestsDatasource.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/CompletedRequestsDatasource.java @@ -43,4 +43,9 @@ public class CompletedRequestsDatasource extends FunctionDataSource { AlgebricksAbsolutePartitionConstraint.randomLocation(locations.getLocations()); return new ClientRequestsFunction(randomLocation, ClientRequestsRequest.RequestType.COMPLETED); } + + @Override + public boolean skipJobCapacityAssignment() { + return true; + } } diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/active_requests/active_requests.4.regex b/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/active_requests/active_requests.4.regex index e2d166b9ad..695472c47f 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/active_requests/active_requests.4.regex +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/active_requests/active_requests.4.regex @@ -1 +1 @@ -/memory\D+229376/ \ No newline at end of file +/memory\D+0/ \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/completed_requests/completed_requests.3.regex b/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/completed_requests/completed_requests.3.regex index e2d166b9ad..695472c47f 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/completed_requests/completed_requests.3.regex +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/completed_requests/completed_requests.3.regex @@ -1 +1 @@ -/memory\D+229376/ \ No newline at end of file +/memory\D+0/ \ No newline at end of file diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/FunctionDataSource.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/FunctionDataSource.java index 5c874d7feb..6b9b03bcaa 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/FunctionDataSource.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/FunctionDataSource.java @@ -108,6 +108,10 @@ public abstract class FunctionDataSource extends DataSource { tupleFilterFactory, outputLimit); } + public boolean skipJobCapacityAssignment() { + return false; + } + protected abstract IDatasourceFunction createFunction(MetadataProvider metadataProvider, AlgebricksAbsolutePartitionConstraint locations);
