Yingyi Bu has uploaded a new change for review.
https://asterix-gerrit.ics.uci.edu/1458
Change subject: ASTERIXDB-1760: support per-query customizations for memory
parameters.
......................................................................
ASTERIXDB-1760: support per-query customizations for memory parameters.
Change-Id: I6e18f3f7706e574553a02e15a39daddda3c413b2
---
M
asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java
M
asterixdb/asterix-app/src/test/resources/runtimets/queries/big-object/big_object_load_20M/big_object_load_20M.2.update.aql
A
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/big-object/big_object_load_20M/big_object_load_20M.1.ddl.sqlpp
A
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/big-object/big_object_load_20M/big_object_load_20M.2.update.sqlpp
A
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/big-object/big_object_load_20M/big_object_load_20M.3.query.sqlpp
M asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
M
asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CompilerProperties.java
7 files changed, 119 insertions(+), 9 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb
refs/changes/58/1458/1
diff --git
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java
index 4140f9a..a06501c 100644
---
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java
+++
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java
@@ -35,7 +35,9 @@
import org.apache.asterix.app.result.ResultUtil;
import org.apache.asterix.common.config.CompilerProperties;
import org.apache.asterix.common.config.ExternalProperties;
+import org.apache.asterix.common.config.IPropertyInterpreter;
import org.apache.asterix.common.config.OptimizationConfUtil;
+import org.apache.asterix.common.config.PropertyInterpreters;
import org.apache.asterix.common.exceptions.ACIDException;
import org.apache.asterix.common.exceptions.CompilationException;
import org.apache.asterix.compiler.provider.ILangCompilationProvider;
@@ -59,9 +61,9 @@
import org.apache.asterix.runtime.job.listener.JobEventListenerFactory;
import org.apache.asterix.runtime.util.AppContextInfo;
import
org.apache.asterix.transaction.management.service.transaction.JobIdFactory;
+import org.apache.asterix.translator.SessionConfig;
import org.apache.asterix.translator.CompiledStatements.ICompiledDmlStatement;
import org.apache.asterix.translator.IStatementExecutor.Stats;
-import org.apache.asterix.translator.SessionConfig;
import
org.apache.hyracks.algebricks.common.constraints.AlgebricksAbsolutePartitionConstraint;
import
org.apache.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint;
import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
@@ -208,9 +210,13 @@
CompilerProperties compilerProperties =
AppContextInfo.INSTANCE.getCompilerProperties();
int frameSize = compilerProperties.getFrameSize();
- int sortFrameLimit = (int) (compilerProperties.getSortMemorySize() /
frameSize);
- int groupFrameLimit = (int) (compilerProperties.getGroupMemorySize() /
frameSize);
- int joinFrameLimit = (int) (compilerProperties.getJoinMemorySize() /
frameSize);
+ Map<String, String> querySpecificConfig = metadataProvider.getConfig();
+ int sortFrameLimit =
getFrameLimit(querySpecificConfig.get(CompilerProperties.COMPILER_SORTMEMORY_KEY),
+ compilerProperties.getSortMemorySize(), frameSize);
+ int groupFrameLimit =
getFrameLimit(querySpecificConfig.get(CompilerProperties.COMPILER_GROUPMEMORY_KEY),
+ compilerProperties.getGroupMemorySize(), frameSize);
+ int joinFrameLimit =
getFrameLimit(querySpecificConfig.get(CompilerProperties.COMPILER_JOINMEMORY_KEY),
+ compilerProperties.getJoinMemorySize(), frameSize);
OptimizationConfUtil.getPhysicalOptimizationConfig().setFrameSize(frameSize);
OptimizationConfUtil.getPhysicalOptimizationConfig().setMaxFramesExternalSort(sortFrameLimit);
OptimizationConfUtil.getPhysicalOptimizationConfig().setMaxFramesExternalGroupBy(groupFrameLimit);
@@ -404,4 +410,11 @@
throw new AlgebricksException(e);
}
}
+
+ private int getFrameLimit(String parameter, long memBudgetInConfiguration,
int frameSize) {
+ IPropertyInterpreter<Long> longBytePropertyInterpreter =
PropertyInterpreters.getLongBytePropertyInterpreter();
+ long memBudget = parameter == null ? memBudgetInConfiguration
+ : longBytePropertyInterpreter.interpret(parameter);
+ return (int) (memBudget / frameSize);
+ }
}
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries/big-object/big_object_load_20M/big_object_load_20M.2.update.aql
b/asterixdb/asterix-app/src/test/resources/runtimets/queries/big-object/big_object_load_20M/big_object_load_20M.2.update.aql
index f384e96..39842e4 100644
---
a/asterixdb/asterix-app/src/test/resources/runtimets/queries/big-object/big_object_load_20M/big_object_load_20M.2.update.aql
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries/big-object/big_object_load_20M/big_object_load_20M.2.update.aql
@@ -26,8 +26,8 @@
use dataverse testdv2;
-set hash_merge "true"
+set "compiler.sortmemory" "64MB"
load dataset testds
using localfs
-(("path"="asterix_nc1://target/data/big-object/big_object_20M.adm"),("format"="adm"))
pre-sorted;
+(("path"="asterix_nc1://target/data/big-object/big_object_20M.adm"),("format"="adm"));
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/big-object/big_object_load_20M/big_object_load_20M.1.ddl.sqlpp
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/big-object/big_object_load_20M/big_object_load_20M.1.ddl.sqlpp
new file mode 100644
index 0000000..e81fa19
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/big-object/big_object_load_20M/big_object_load_20M.1.ddl.sqlpp
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Test case Name : big_object_bulkload.aql
+ * Description : bulkload insert of large objects
+ * Expected Result : Success
+ * Date : 20th April 2016
+ */
+
+drop dataverse testdv2 if exists;
+create dataverse testdv2;
+use testdv2;
+
+create type testtype as closed {
+ id: int64,
+ name: string,
+ hobbies: {{string}}
+}
+
+create dataset testds(testtype) primary key id;
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/big-object/big_object_load_20M/big_object_load_20M.2.update.sqlpp
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/big-object/big_object_load_20M/big_object_load_20M.2.update.sqlpp
new file mode 100644
index 0000000..09f3a69
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/big-object/big_object_load_20M/big_object_load_20M.2.update.sqlpp
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/**
+ *
+ * Big object (20 MB) loading test
+ * Expected result: success
+ *
+ */
+
+use testdv2;
+
+set `compiler.sortmemory` "64MB"
+
+load dataset testds
+using localfs
+(("path"="asterix_nc1://target/data/big-object/big_object_20M.adm"),("format"="adm"));
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/big-object/big_object_load_20M/big_object_load_20M.3.query.sqlpp
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/big-object/big_object_load_20M/big_object_load_20M.3.query.sqlpp
new file mode 100644
index 0000000..4730ac8
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/big-object/big_object_load_20M/big_object_load_20M.3.query.sqlpp
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use testdv2;
+
+select value d
+from testds d
+where d.id=1;
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
index 0aaf8c3..8139e98 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
@@ -7824,6 +7824,11 @@
<output-dir compare="Text">big_object_join</output-dir>
</compilation-unit>
</test-case>
+ <test-case FilePath="big-object">
+ <compilation-unit name="big_object_load_20M">
+ <output-dir compare="Text">big_object_load_20M</output-dir>
+ </compilation-unit>
+ </test-case>
</test-group>
<test-group name="external-indexing">
<test-case FilePath="external-indexing">
diff --git
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CompilerProperties.java
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CompilerProperties.java
index 4e18bde..eb7a2a0 100644
---
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CompilerProperties.java
+++
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CompilerProperties.java
@@ -25,13 +25,13 @@
public class CompilerProperties extends AbstractProperties {
- private static final String COMPILER_SORTMEMORY_KEY =
"compiler.sortmemory";
+ public static final String COMPILER_SORTMEMORY_KEY = "compiler.sortmemory";
private static final long COMPILER_SORTMEMORY_DEFAULT =
StorageUtil.getSizeInBytes(32, MEGABYTE);
- private static final String COMPILER_GROUPMEMORY_KEY =
"compiler.groupmemory";
+ public static final String COMPILER_GROUPMEMORY_KEY =
"compiler.groupmemory";
private static final long COMPILER_GROUPMEMORY_DEFAULT =
StorageUtil.getSizeInBytes(32, MEGABYTE);
- private static final String COMPILER_JOINMEMORY_KEY =
"compiler.joinmemory";
+ public static final String COMPILER_JOINMEMORY_KEY = "compiler.joinmemory";
private static final long COMPILER_JOINMEMORY_DEFAULT =
StorageUtil.getSizeInBytes(32, MEGABYTE);
private static final String COMPILER_FRAMESIZE_KEY = "compiler.framesize";
--
To view, visit https://asterix-gerrit.ics.uci.edu/1458
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I6e18f3f7706e574553a02e15a39daddda3c413b2
Gerrit-PatchSet: 1
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Yingyi Bu <[email protected]>
