This is an automated email from the ASF dual-hosted git repository.
dlych pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
The following commit(s) were added to refs/heads/master by this push:
new 1e90cac [NO ISSUE][COMP] Reduce memory requirements for monitoring
functions
1e90cac is described below
commit 1e90caca704c85ba49f88b810163ae6fab344cf1
Author: Dmitry Lychagin <[email protected]>
AuthorDate: Fri Sep 6 16:11:20 2019 -0700
[NO ISSUE][COMP] Reduce memory requirements for monitoring functions
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
- Queries that only invoke monitoring/metadata datasource functions
should run with minimal memory requirements
- The functions are: active_requests(), completed_requests(),
dataset_resources(), storage_components(), jobs(), and ping()
Change-Id: If7de53a5b476c8ce0d3fe485bf526a5d425e4c37
Reviewed-on: https://asterix-gerrit.ics.uci.edu/3548
Sonar-Qube: Jenkins <[email protected]>
Contrib: Jenkins <[email protected]>
Integration-Tests: Jenkins <[email protected]>
Tested-by: Jenkins <[email protected]>
Reviewed-by: Dmitry Lychagin <[email protected]>
Reviewed-by: Ali Alsuliman <[email protected]>
---
.../rules/SetAsterixMemoryRequirementsRule.java | 42 ++++++++++++++++++----
.../app/function/ActiveRequestsDatasource.java | 4 +--
.../app/function/CompletedRequestsDatasource.java | 3 +-
.../app/function/DatasetResourcesDatasource.java | 7 ++--
.../asterix/app/function/DumpIndexDatasource.java | 5 +--
.../app/function/JobSummariesDatasource.java | 4 +--
.../asterix/app/function/PingDatasource.java | 3 +-
.../app/function/StorageComponentsDatasource.java | 7 ++--
.../TPCDSAllTablesDataGeneratorDatasource.java | 3 +-
.../TPCDSSingleTableDataGeneratorDatasource.java | 3 +-
.../asterix/util/MetadataBuiltinFunctions.java | 15 ++++----
.../active_requests.4.plans.sqlpp} | 16 ++++++---
...uery.sqlpp => completed_requests.2.query.sqlpp} | 0
...uery.sqlpp => completed_requests.3.plans.sqlpp} | 16 ++++++---
.../dataset-resources.6.plans.sqlpp} | 17 ++++++---
.../dataset-resources.7.plans.sqlpp} | 17 ++++++---
.../jobs.2.plans.sqlpp} | 17 ++++++---
.../metadata_only_01.1.plans.sqlpp | 2 ++
.../ping.2.plans.sqlpp} | 17 ++++++---
.../misc/active_requests/active_requests.4.regex | 1 +
.../completed_requests/completed_requests.3.regex | 1 +
.../dataset-resources/dataset-resources.6.regex | 1 +
.../dataset-resources/dataset-resources.7.regex | 1 +
.../runtimets/results/misc/jobs/jobs.2.regex | 1 +
.../runtimets/results/misc/ping/ping.2.regex | 1 +
.../metadata/declared/FunctionDataSource.java | 12 +++++++
.../asterix/om/functions/BuiltinFunctions.java | 12 +++++--
27 files changed, 172 insertions(+), 56 deletions(-)
diff --git
a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/SetAsterixMemoryRequirementsRule.java
b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/SetAsterixMemoryRequirementsRule.java
index a850cf4..91dfe7f 100644
---
a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/SetAsterixMemoryRequirementsRule.java
+++
b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/SetAsterixMemoryRequirementsRule.java
@@ -20,12 +20,15 @@
package org.apache.asterix.optimizer.rules;
import java.util.Set;
+import java.util.function.Predicate;
import org.apache.asterix.metadata.declared.DataSource;
import org.apache.asterix.metadata.declared.DataSourceId;
import org.apache.asterix.metadata.utils.MetadataConstants;
+import org.apache.asterix.om.functions.BuiltinFunctions;
import org.apache.asterix.optimizer.base.AsterixOptimizationContext;
import org.apache.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
import
org.apache.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
import org.apache.hyracks.algebricks.rewriter.rules.SetMemoryRequirementsRule;
@@ -34,7 +37,9 @@ import it.unimi.dsi.fastutil.ints.Int2ObjectMap;
/**
* This rule extends {@link SetMemoryRequirementsRule} and modifies its
functionality as follows:
* <ul>
- * <li>It skips memory requirements configuration if the query operates only
on metadata datasets.
+ * <li>It skips memory requirements configuration if the query operates only
on metadata datasets and/or
+ * datasource functions annotated with
+ * {@link BuiltinFunctions.DataSourceFunctionProperty#MIN_MEMORY_BUDGET
MIN_MEMORY_BUDGET} property.
* In this case operators will retain their default (minimal) memory
requirements.
* </li>
* </ul>
@@ -49,11 +54,36 @@ public final class SetAsterixMemoryRequirementsRule extends
SetMemoryRequirement
private boolean forceMinMemoryBudget(AsterixOptimizationContext context) {
Int2ObjectMap<Set<DataSourceId>> dataSourceMap =
context.getDataSourceMap();
- if (dataSourceMap.size() == 1) {
- Set<DataSourceId> dataSources =
dataSourceMap.get(DataSource.Type.INTERNAL_DATASET);
- return dataSources != null && dataSources.stream()
- .allMatch(dsId ->
MetadataConstants.METADATA_DATAVERSE_NAME.equals(dsId.getDataverseName()));
+ if (dataSourceMap.isEmpty()) {
+ return false;
}
- return false;
+ for (Int2ObjectMap.Entry<Set<DataSourceId>> me :
dataSourceMap.int2ObjectEntrySet()) {
+ int dataSourceType = me.getIntKey();
+ Predicate<DataSourceId> dataSourceTest;
+ switch (dataSourceType) {
+ case DataSource.Type.INTERNAL_DATASET:
+ dataSourceTest =
SetAsterixMemoryRequirementsRule::isMinMemoryBudgetDataset;
+ break;
+ case DataSource.Type.FUNCTION:
+ dataSourceTest =
SetAsterixMemoryRequirementsRule::isMinMemoryBudgetFunction;
+ break;
+ default:
+ return false;
+ }
+ if (!me.getValue().stream().allMatch(dataSourceTest)) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ private static boolean isMinMemoryBudgetDataset(DataSourceId dsId) {
+ return
MetadataConstants.METADATA_DATAVERSE_NAME.equals(dsId.getDataverseName());
+ }
+
+ private static boolean isMinMemoryBudgetFunction(DataSourceId dsId) {
+ return BuiltinFunctions.builtinFunctionHasProperty(
+ new FunctionIdentifier(dsId.getDataverseName(),
dsId.getDatasourceName()),
+ BuiltinFunctions.DataSourceFunctionProperty.MIN_MEMORY_BUDGET);
}
}
\ No newline at end of file
diff --git
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/ActiveRequestsDatasource.java
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/ActiveRequestsDatasource.java
index eb5416b..2afd11c 100644
---
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/ActiveRequestsDatasource.java
+++
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/ActiveRequestsDatasource.java
@@ -29,8 +29,8 @@ import
org.apache.hyracks.algebricks.core.algebra.properties.INodeDomain;
public class ActiveRequestsDatasource extends FunctionDataSource {
- private static final DataSourceId ACTIVE_REQUESTS_DATASOURCE_ID = new
DataSourceId(
- ActiveRequestsRewriter.ACTIVE_REQUESTS.getNamespace(),
ActiveRequestsRewriter.ACTIVE_REQUESTS.getName());
+ private static final DataSourceId ACTIVE_REQUESTS_DATASOURCE_ID =
+ createDataSourceId(ActiveRequestsRewriter.ACTIVE_REQUESTS);
public ActiveRequestsDatasource(INodeDomain domain) throws
AlgebricksException {
super(ACTIVE_REQUESTS_DATASOURCE_ID, domain);
diff --git
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/CompletedRequestsDatasource.java
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/CompletedRequestsDatasource.java
index 4c3672b..1fe103e 100644
---
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/CompletedRequestsDatasource.java
+++
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/CompletedRequestsDatasource.java
@@ -30,8 +30,7 @@ import
org.apache.hyracks.algebricks.core.algebra.properties.INodeDomain;
public class CompletedRequestsDatasource extends FunctionDataSource {
private static final DataSourceId COMPLETED_REQUESTS_DATASOURCE_ID =
- new
DataSourceId(ActiveRequestsRewriter.ACTIVE_REQUESTS.getNamespace(),
- CompletedRequestsRewriter.COMPLETED_REQUESTS.getName());
+ createDataSourceId(CompletedRequestsRewriter.COMPLETED_REQUESTS);
public CompletedRequestsDatasource(INodeDomain domain) throws
AlgebricksException {
super(COMPLETED_REQUESTS_DATASOURCE_ID, domain);
diff --git
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/DatasetResourcesDatasource.java
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/DatasetResourcesDatasource.java
index e6b025a..ffd04ab 100644
---
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/DatasetResourcesDatasource.java
+++
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/DatasetResourcesDatasource.java
@@ -27,11 +27,14 @@ import
org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
import org.apache.hyracks.algebricks.core.algebra.properties.INodeDomain;
public class DatasetResourcesDatasource extends FunctionDataSource {
+
+ private static final DataSourceId DATASET_RESOURCES_DATASOURCE_ID =
+ createDataSourceId(DatasetResourcesRewriter.DATASET_RESOURCES);
+
private final int datasetId;
public DatasetResourcesDatasource(INodeDomain domain, int datasetId)
throws AlgebricksException {
- super(new
DataSourceId(DatasetResourcesRewriter.DATASET_RESOURCES.getNamespace(),
- DatasetResourcesRewriter.DATASET_RESOURCES.getName()), domain);
+ super(DATASET_RESOURCES_DATASOURCE_ID, domain);
this.datasetId = datasetId;
}
diff --git
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/DumpIndexDatasource.java
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/DumpIndexDatasource.java
index 5981787..04e3eea 100644
---
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/DumpIndexDatasource.java
+++
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/DumpIndexDatasource.java
@@ -31,14 +31,15 @@ import
org.apache.hyracks.storage.am.common.dataflow.IndexDataflowHelperFactory;
public class DumpIndexDatasource extends FunctionDataSource {
+ public static final DataSourceId DUMP_INDEX_DATASOURCE_ID =
createDataSourceId(DumpIndexRewriter.DUMP_INDEX);
+
private final IndexDataflowHelperFactory indexDataflowHelperFactory;
private final RecordDescriptor recDesc;
private final IBinaryComparatorFactory[] comparatorFactories;
public DumpIndexDatasource(INodeDomain domain, IndexDataflowHelperFactory
indexDataflowHelperFactory,
RecordDescriptor recDesc, IBinaryComparatorFactory[]
comparatorFactories) throws AlgebricksException {
- super(new DataSourceId(DumpIndexRewriter.DUMP_INDEX.getNamespace(),
DumpIndexRewriter.DUMP_INDEX.getName()),
- domain);
+ super(DUMP_INDEX_DATASOURCE_ID, domain);
this.indexDataflowHelperFactory = indexDataflowHelperFactory;
this.recDesc = recDesc;
this.comparatorFactories = comparatorFactories;
diff --git
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/JobSummariesDatasource.java
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/JobSummariesDatasource.java
index b92dc55..835a97d 100644
---
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/JobSummariesDatasource.java
+++
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/JobSummariesDatasource.java
@@ -28,8 +28,8 @@ import
org.apache.hyracks.algebricks.core.algebra.properties.INodeDomain;
public class JobSummariesDatasource extends FunctionDataSource {
- private static final DataSourceId JOB_SUMMARIES_DATASOURCE_ID = new
DataSourceId(
- JobSummariesRewriter.JOBSUMMARIES.getNamespace(),
JobSummariesRewriter.JOBSUMMARIES.getName());
+ private static final DataSourceId JOB_SUMMARIES_DATASOURCE_ID =
+ createDataSourceId(JobSummariesRewriter.JOBSUMMARIES);
public JobSummariesDatasource(INodeDomain domain) throws
AlgebricksException {
super(JOB_SUMMARIES_DATASOURCE_ID, domain);
diff --git
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/PingDatasource.java
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/PingDatasource.java
index 480f3c4..67d5e53 100644
---
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/PingDatasource.java
+++
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/PingDatasource.java
@@ -28,8 +28,7 @@ import
org.apache.hyracks.algebricks.core.algebra.properties.INodeDomain;
public class PingDatasource extends FunctionDataSource {
- private static final DataSourceId PING_DATASOURCE_ID =
- new DataSourceId(PingRewriter.PING.getNamespace(),
PingRewriter.PING.getName());
+ private static final DataSourceId PING_DATASOURCE_ID =
createDataSourceId(PingRewriter.PING);
public PingDatasource(INodeDomain domain) throws AlgebricksException {
super(PING_DATASOURCE_ID, domain);
diff --git
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/StorageComponentsDatasource.java
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/StorageComponentsDatasource.java
index 7245b88..b8a4167 100644
---
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/StorageComponentsDatasource.java
+++
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/StorageComponentsDatasource.java
@@ -27,11 +27,14 @@ import
org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
import org.apache.hyracks.algebricks.core.algebra.properties.INodeDomain;
public class StorageComponentsDatasource extends FunctionDataSource {
+
+ public static final DataSourceId STORAGE_COMPONENTS_DATASOURCE_ID =
+ createDataSourceId(StorageComponentsRewriter.STORAGE_COMPONENTS);
+
private final int datasetId;
public StorageComponentsDatasource(INodeDomain domain, int datasetId)
throws AlgebricksException {
- super(new
DataSourceId(StorageComponentsRewriter.STORAGE_COMPONENTS.getNamespace(),
- StorageComponentsRewriter.STORAGE_COMPONENTS.getName()),
domain);
+ super(STORAGE_COMPONENTS_DATASOURCE_ID, domain);
this.datasetId = datasetId;
}
diff --git
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/TPCDSAllTablesDataGeneratorDatasource.java
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/TPCDSAllTablesDataGeneratorDatasource.java
index 352fea8..c382fb4 100644
---
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/TPCDSAllTablesDataGeneratorDatasource.java
+++
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/TPCDSAllTablesDataGeneratorDatasource.java
@@ -54,8 +54,7 @@ public class TPCDSAllTablesDataGeneratorDatasource extends
FunctionDataSource {
* @return A DataSourceId that's based on the function details and its
parameters
*/
private static DataSourceId createDataSourceId(double scalingFactor) {
- return new
DataSourceId(TPCDSAllTablesDataGeneratorRewriter.TPCDS_ALL_TABLES_DATA_GENERATOR.getNamespace(),
-
TPCDSAllTablesDataGeneratorRewriter.TPCDS_ALL_TABLES_DATA_GENERATOR.getName(),
+ return
createDataSourceId(TPCDSAllTablesDataGeneratorRewriter.TPCDS_ALL_TABLES_DATA_GENERATOR,
Double.toString(scalingFactor));
}
diff --git
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/TPCDSSingleTableDataGeneratorDatasource.java
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/TPCDSSingleTableDataGeneratorDatasource.java
index 968e289..df3903f 100644
---
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/TPCDSSingleTableDataGeneratorDatasource.java
+++
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/TPCDSSingleTableDataGeneratorDatasource.java
@@ -58,8 +58,7 @@ public class TPCDSSingleTableDataGeneratorDatasource extends
FunctionDataSource
* @return A DataSourceId that's based on the function details and its
parameters
*/
private static DataSourceId createDataSourceId(String tableName, double
scalingFactor) {
- return new
DataSourceId(TPCDSSingleTableDataGeneratorRewriter.TPCDS_SINGLE_TABLE_DATA_GENERATOR.getNamespace(),
-
TPCDSSingleTableDataGeneratorRewriter.TPCDS_SINGLE_TABLE_DATA_GENERATOR.getName(),
tableName,
+ return
createDataSourceId(TPCDSSingleTableDataGeneratorRewriter.TPCDS_SINGLE_TABLE_DATA_GENERATOR,
tableName,
Double.toString(scalingFactor));
}
diff --git
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/util/MetadataBuiltinFunctions.java
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/util/MetadataBuiltinFunctions.java
index 26726f6..d6dc67c 100644
---
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/util/MetadataBuiltinFunctions.java
+++
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/util/MetadataBuiltinFunctions.java
@@ -48,18 +48,19 @@ public class MetadataBuiltinFunctions {
(expression, env, mp) -> RecordUtil.FULLY_OPEN_RECORD_TYPE,
true);
BuiltinFunctions.addUnnestFun(DatasetResourcesRewriter.DATASET_RESOURCES,
false);
BuiltinFunctions.addDatasourceFunction(DatasetResourcesRewriter.DATASET_RESOURCES,
- DatasetResourcesRewriter.INSTANCE);
+ DatasetResourcesRewriter.INSTANCE,
BuiltinFunctions.DataSourceFunctionProperty.MIN_MEMORY_BUDGET);
// Dataset components function
BuiltinFunctions.addPrivateFunction(StorageComponentsRewriter.STORAGE_COMPONENTS,
(expression, env, mp) -> RecordUtil.FULLY_OPEN_RECORD_TYPE,
true);
BuiltinFunctions.addUnnestFun(StorageComponentsRewriter.STORAGE_COMPONENTS,
false);
BuiltinFunctions.addDatasourceFunction(StorageComponentsRewriter.STORAGE_COMPONENTS,
- StorageComponentsRewriter.INSTANCE);
+ StorageComponentsRewriter.INSTANCE,
BuiltinFunctions.DataSourceFunctionProperty.MIN_MEMORY_BUDGET);
// Ping function
BuiltinFunctions.addPrivateFunction(PingRewriter.PING,
(expression, env, mp) -> RecordUtil.FULLY_OPEN_RECORD_TYPE,
true);
BuiltinFunctions.addUnnestFun(PingRewriter.PING, true);
- BuiltinFunctions.addDatasourceFunction(PingRewriter.PING,
PingRewriter.INSTANCE);
+ BuiltinFunctions.addDatasourceFunction(PingRewriter.PING,
PingRewriter.INSTANCE,
+ BuiltinFunctions.DataSourceFunctionProperty.MIN_MEMORY_BUDGET);
// TPC-DS data generation function
BuiltinFunctions.addPrivateFunction(TPCDSSingleTableDataGeneratorRewriter.TPCDS_SINGLE_TABLE_DATA_GENERATOR,
(expression, env, mp) -> RecordUtil.FULLY_OPEN_RECORD_TYPE,
true);
@@ -75,18 +76,20 @@ public class MetadataBuiltinFunctions {
BuiltinFunctions.addFunction(ActiveRequestsRewriter.ACTIVE_REQUESTS,
(expression, env, mp) -> RecordUtil.FULLY_OPEN_RECORD_TYPE,
true);
BuiltinFunctions.addUnnestFun(ActiveRequestsRewriter.ACTIVE_REQUESTS,
true);
-
BuiltinFunctions.addDatasourceFunction(ActiveRequestsRewriter.ACTIVE_REQUESTS,
ActiveRequestsRewriter.INSTANCE);
+
BuiltinFunctions.addDatasourceFunction(ActiveRequestsRewriter.ACTIVE_REQUESTS,
ActiveRequestsRewriter.INSTANCE,
+ BuiltinFunctions.DataSourceFunctionProperty.MIN_MEMORY_BUDGET);
// job-summaries function
BuiltinFunctions.addPrivateFunction(JobSummariesRewriter.JOBSUMMARIES,
(expression, env, mp) -> RecordUtil.FULLY_OPEN_RECORD_TYPE,
true);
BuiltinFunctions.addUnnestFun(JobSummariesRewriter.JOBSUMMARIES, true);
-
BuiltinFunctions.addDatasourceFunction(JobSummariesRewriter.JOBSUMMARIES,
JobSummariesRewriter.INSTANCE);
+
BuiltinFunctions.addDatasourceFunction(JobSummariesRewriter.JOBSUMMARIES,
JobSummariesRewriter.INSTANCE,
+ BuiltinFunctions.DataSourceFunctionProperty.MIN_MEMORY_BUDGET);
// completed requests function
BuiltinFunctions.addFunction(CompletedRequestsRewriter.COMPLETED_REQUESTS,
(expression, env, mp) -> RecordUtil.FULLY_OPEN_RECORD_TYPE,
true);
BuiltinFunctions.addUnnestFun(CompletedRequestsRewriter.COMPLETED_REQUESTS,
true);
BuiltinFunctions.addDatasourceFunction(CompletedRequestsRewriter.COMPLETED_REQUESTS,
- CompletedRequestsRewriter.INSTANCE);
+ CompletedRequestsRewriter.INSTANCE,
BuiltinFunctions.DataSourceFunctionProperty.MIN_MEMORY_BUDGET);
// Dump index function
BuiltinFunctions.addPrivateFunction(DumpIndexRewriter.DUMP_INDEX,
(expression, env, mp) -> RecordUtil.FULLY_OPEN_RECORD_TYPE,
true);
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/completed_requests/completed_requests.3.query.sqlpp
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/active_requests/active_requests.4.plans.sqlpp
similarity index 77%
copy from
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/completed_requests/completed_requests.3.query.sqlpp
copy to
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/active_requests/active_requests.4.plans.sqlpp
index 1bfb32f..fe4447f 100644
---
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/completed_requests/completed_requests.3.query.sqlpp
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/active_requests/active_requests.4.plans.sqlpp
@@ -16,8 +16,16 @@
* specific language governing permissions and limitations
* under the License.
*/
- -- param client_context_id=ensure_completed_query
-LET request_count = (SELECT VALUE COUNT(*) FROM completed_requests() r
-WHERE r.state="completed" AND r.clientContextID =
"completed_requests_query")[0]
-SELECT VALUE request_count > 0;
\ No newline at end of file
+/*
+ * Test that a query that only uses active_requests()
+ * gets assigned a minimal memory budget
+ */
+
+-- param job:string=true
+
+SET `compiler.parallelism` "1";
+
+from active_requests() t
+select value t
+order by t
\ No newline at end of file
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/completed_requests/completed_requests.3.query.sqlpp
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/completed_requests/completed_requests.2.query.sqlpp
similarity index 100%
copy from
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/completed_requests/completed_requests.3.query.sqlpp
copy to
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/completed_requests/completed_requests.2.query.sqlpp
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/completed_requests/completed_requests.3.query.sqlpp
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/completed_requests/completed_requests.3.plans.sqlpp
similarity index 77%
copy from
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/completed_requests/completed_requests.3.query.sqlpp
copy to
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/completed_requests/completed_requests.3.plans.sqlpp
index 1bfb32f..0e75d22 100644
---
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/completed_requests/completed_requests.3.query.sqlpp
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/completed_requests/completed_requests.3.plans.sqlpp
@@ -16,8 +16,16 @@
* specific language governing permissions and limitations
* under the License.
*/
- -- param client_context_id=ensure_completed_query
-LET request_count = (SELECT VALUE COUNT(*) FROM completed_requests() r
-WHERE r.state="completed" AND r.clientContextID =
"completed_requests_query")[0]
-SELECT VALUE request_count > 0;
\ No newline at end of file
+/*
+ * Test that a query that only uses completed_requests()
+ * gets assigned a minimal memory budget
+ */
+
+-- param job:string=true
+
+SET `compiler.parallelism` "1";
+
+from completed_requests() t
+select value t
+order by t
\ No newline at end of file
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/completed_requests/completed_requests.3.query.sqlpp
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/dataset-resources/dataset-resources.6.plans.sqlpp
similarity index 74%
copy from
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/completed_requests/completed_requests.3.query.sqlpp
copy to
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/dataset-resources/dataset-resources.6.plans.sqlpp
index 1bfb32f..65e7f5b 100644
---
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/completed_requests/completed_requests.3.query.sqlpp
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/dataset-resources/dataset-resources.6.plans.sqlpp
@@ -16,8 +16,17 @@
* specific language governing permissions and limitations
* under the License.
*/
- -- param client_context_id=ensure_completed_query
-LET request_count = (SELECT VALUE COUNT(*) FROM completed_requests() r
-WHERE r.state="completed" AND r.clientContextID =
"completed_requests_query")[0]
-SELECT VALUE request_count > 0;
\ No newline at end of file
+/*
+ * Test that a query that only uses dataset_resources()
+ * gets assigned a minimal memory budget
+ */
+
+-- param job:string=true
+
+SET `compiler.parallelism` "1";
+SET `import-private-functions` `true`;
+
+from dataset_resources('test','LineItem') t
+select value t
+order by t
\ No newline at end of file
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/completed_requests/completed_requests.3.query.sqlpp
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/dataset-resources/dataset-resources.7.plans.sqlpp
similarity index 74%
copy from
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/completed_requests/completed_requests.3.query.sqlpp
copy to
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/dataset-resources/dataset-resources.7.plans.sqlpp
index 1bfb32f..c1aebd1 100644
---
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/completed_requests/completed_requests.3.query.sqlpp
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/dataset-resources/dataset-resources.7.plans.sqlpp
@@ -16,8 +16,17 @@
* specific language governing permissions and limitations
* under the License.
*/
- -- param client_context_id=ensure_completed_query
-LET request_count = (SELECT VALUE COUNT(*) FROM completed_requests() r
-WHERE r.state="completed" AND r.clientContextID =
"completed_requests_query")[0]
-SELECT VALUE request_count > 0;
\ No newline at end of file
+/*
+ * Test that a query that only uses storage_components()
+ * gets assigned a minimal memory budget
+ */
+
+-- param job:string=true
+
+SET `compiler.parallelism` "1";
+SET `import-private-functions` `true`;
+
+from storage_components('test','LineItem') t
+select value t
+order by t
\ No newline at end of file
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/completed_requests/completed_requests.3.query.sqlpp
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/jobs/jobs.2.plans.sqlpp
similarity index 77%
copy from
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/completed_requests/completed_requests.3.query.sqlpp
copy to
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/jobs/jobs.2.plans.sqlpp
index 1bfb32f..9143a86 100644
---
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/completed_requests/completed_requests.3.query.sqlpp
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/jobs/jobs.2.plans.sqlpp
@@ -16,8 +16,17 @@
* specific language governing permissions and limitations
* under the License.
*/
- -- param client_context_id=ensure_completed_query
-LET request_count = (SELECT VALUE COUNT(*) FROM completed_requests() r
-WHERE r.state="completed" AND r.clientContextID =
"completed_requests_query")[0]
-SELECT VALUE request_count > 0;
\ No newline at end of file
+/*
+ * Test that a query that only uses jobs()
+ * gets assigned a minimal memory budget
+ */
+
+-- param job:string=true
+
+SET `compiler.parallelism` "1";
+SET `import-private-functions` "true";
+
+from jobs() t
+select value t
+order by t
\ No newline at end of file
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/metadata_only_01/metadata_only_01.1.plans.sqlpp
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/metadata_only_01/metadata_only_01.1.plans.sqlpp
index 6862b8f..c2e086f 100644
---
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/metadata_only_01/metadata_only_01.1.plans.sqlpp
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/metadata_only_01/metadata_only_01.1.plans.sqlpp
@@ -17,6 +17,8 @@
* under the License.
*/
+/* Test that a metadata-only query gets assigned a minimal memory budget */
+
-- param job:string=true
SET `compiler.parallelism` "1";
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/completed_requests/completed_requests.3.query.sqlpp
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/ping/ping.2.plans.sqlpp
similarity index 77%
rename from
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/completed_requests/completed_requests.3.query.sqlpp
rename to
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/ping/ping.2.plans.sqlpp
index 1bfb32f..6928d06 100644
---
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/completed_requests/completed_requests.3.query.sqlpp
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/ping/ping.2.plans.sqlpp
@@ -16,8 +16,17 @@
* specific language governing permissions and limitations
* under the License.
*/
- -- param client_context_id=ensure_completed_query
-LET request_count = (SELECT VALUE COUNT(*) FROM completed_requests() r
-WHERE r.state="completed" AND r.clientContextID =
"completed_requests_query")[0]
-SELECT VALUE request_count > 0;
\ No newline at end of file
+/*
+ * Test that a query that only uses ping()
+ * get assigned a minimal memory budget
+ */
+
+-- param job:string=true
+
+SET `compiler.parallelism` "1";
+SET `import-private-functions` "true";
+
+from ping() t
+select value t
+order by t
\ No newline at end of file
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/active_requests/active_requests.4.regex
b/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/active_requests/active_requests.4.regex
new file mode 100644
index 0000000..e2d166b
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/active_requests/active_requests.4.regex
@@ -0,0 +1 @@
+/memory\D+229376/
\ No newline at end of file
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/completed_requests/completed_requests.3.regex
b/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/completed_requests/completed_requests.3.regex
new file mode 100644
index 0000000..e2d166b
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/completed_requests/completed_requests.3.regex
@@ -0,0 +1 @@
+/memory\D+229376/
\ No newline at end of file
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/dataset-resources/dataset-resources.6.regex
b/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/dataset-resources/dataset-resources.6.regex
new file mode 100644
index 0000000..e2d166b
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/dataset-resources/dataset-resources.6.regex
@@ -0,0 +1 @@
+/memory\D+229376/
\ No newline at end of file
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/dataset-resources/dataset-resources.7.regex
b/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/dataset-resources/dataset-resources.7.regex
new file mode 100644
index 0000000..e2d166b
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/dataset-resources/dataset-resources.7.regex
@@ -0,0 +1 @@
+/memory\D+229376/
\ No newline at end of file
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/jobs/jobs.2.regex
b/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/jobs/jobs.2.regex
new file mode 100644
index 0000000..e2d166b
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/jobs/jobs.2.regex
@@ -0,0 +1 @@
+/memory\D+229376/
\ No newline at end of file
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/ping/ping.2.regex
b/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/ping/ping.2.regex
new file mode 100644
index 0000000..e2d166b
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/ping/ping.2.regex
@@ -0,0 +1 @@
+/memory\D+229376/
\ No newline at end of file
diff --git
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/FunctionDataSource.java
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/FunctionDataSource.java
index 1635507..8e79683 100644
---
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/FunctionDataSource.java
+++
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/FunctionDataSource.java
@@ -37,6 +37,7 @@ import
org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
import org.apache.hyracks.algebricks.common.utils.Pair;
import org.apache.hyracks.algebricks.core.algebra.base.LogicalVariable;
import
org.apache.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
+import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
import org.apache.hyracks.algebricks.core.algebra.metadata.IDataSource;
import
org.apache.hyracks.algebricks.core.algebra.metadata.IDataSourcePropertiesProvider;
import
org.apache.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
@@ -96,4 +97,15 @@ public abstract class FunctionDataSource extends DataSource {
Set<String> ncs = new HashSet<>(Arrays.asList(allPartitions));
return new AlgebricksAbsolutePartitionConstraint(ncs.toArray(new
String[ncs.size()]));
}
+
+ protected static DataSourceId createDataSourceId(FunctionIdentifier fid,
String... parameters) {
+ int paramCount = parameters != null ? parameters.length : 0;
+ String[] components = new String[paramCount + 2];
+ components[0] = fid.getNamespace();
+ components[1] = fid.getName();
+ if (paramCount > 0) {
+ System.arraycopy(parameters, 0, components, 2, paramCount);
+ }
+ return new DataSourceId(components);
+ }
}
diff --git
a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/BuiltinFunctions.java
b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/BuiltinFunctions.java
index e95dfd0..e96d455 100644
---
a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/BuiltinFunctions.java
+++
b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/BuiltinFunctions.java
@@ -2984,8 +2984,16 @@ public class BuiltinFunctions {
addUnnestFun(SUBSET_COLLECTION, false);
}
- public static void addDatasourceFunction(FunctionIdentifier fi,
IFunctionToDataSourceRewriter transformer) {
- datasourceFunctions.put(getAsterixFunctionInfo(fi), transformer);
+ public enum DataSourceFunctionProperty implements BuiltinFunctionProperty {
+ /** Force minimum memory budget if a query only uses this function */
+ MIN_MEMORY_BUDGET
+ }
+
+ public static void addDatasourceFunction(FunctionIdentifier fi,
IFunctionToDataSourceRewriter transformer,
+ DataSourceFunctionProperty... properties) {
+ IFunctionInfo finfo = getAsterixFunctionInfo(fi);
+ datasourceFunctions.put(finfo, transformer);
+ registerFunctionProperties(finfo, DataSourceFunctionProperty.class,
properties);
}
public static IFunctionToDataSourceRewriter
getDatasourceTransformer(FunctionIdentifier fi) {
