This is an automated email from the ASF dual-hosted git repository.
ggalvizo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
The following commit(s) were added to refs/heads/master by this push:
new 7fc85c3af1 [ASTERIXDB-3225][CONF] Fixing extension support in cc.conf
7fc85c3af1 is described below
commit 7fc85c3af14f226af0b853f19d66e5f7b7e5cbe8
Author: glennga <[email protected]>
AuthorDate: Sat Jul 15 21:48:17 2023 -0700
[ASTERIXDB-3225][CONF] Fixing extension support in cc.conf
- user model changes: no
- storage format changes: no
- interface changes: no
This patch adds back support for specifying custom key-value pairs for
statement executor extensions in the cc.conf file.
Change-Id: Icdee8bfd9eda3947b0dfe34d88d418803345bbb2
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17645
Integration-Tests: Jenkins <[email protected]>
Tested-by: Jenkins <[email protected]>
Reviewed-by: Ian Maxon <[email protected]>
---
.../org/apache/asterix/api/common/APIFramework.java | 4 ++--
.../asterix/common/config/OptimizationConfUtil.java | 19 ++++++++++++++++++-
.../rewriter/base/PhysicalOptimizationConfig.java | 18 ++++++++++++++++++
.../application/ConfigManagerApplicationConfig.java | 6 ++++--
.../hyracks/control/common/config/ConfigManager.java | 17 +++++++++++++++++
5 files changed, 59 insertions(+), 5 deletions(-)
diff --git
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java
index afdcee40dc..10f118c06f 100644
---
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java
+++
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java
@@ -214,8 +214,8 @@ public class APIFramework {
ICcApplicationContext ccAppContext =
metadataProvider.getApplicationContext();
CompilerProperties compilerProperties =
ccAppContext.getCompilerProperties();
Map<String, Object> querySpecificConfig =
validateConfig(metadataProvider.getConfig(), sourceLoc);
- final PhysicalOptimizationConfig physOptConf =
-
OptimizationConfUtil.createPhysicalOptimizationConf(compilerProperties,
querySpecificConfig, sourceLoc);
+ final PhysicalOptimizationConfig physOptConf =
OptimizationConfUtil.createPhysicalOptimizationConf(
+ compilerProperties, querySpecificConfig,
configurableParameterNames, sourceLoc);
boolean cboMode = physOptConf.getCBOMode() ||
physOptConf.getCBOTestMode();
HeuristicCompilerFactoryBuilder builder =
new
HeuristicCompilerFactoryBuilder(OptimizationContextFactory.INSTANCE);
diff --git
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/OptimizationConfUtil.java
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/OptimizationConfUtil.java
index c103b50413..ad312812c1 100644
---
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/OptimizationConfUtil.java
+++
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/OptimizationConfUtil.java
@@ -18,7 +18,11 @@
*/
package org.apache.asterix.common.config;
+import java.util.Arrays;
+import java.util.HashSet;
import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
import org.apache.asterix.common.exceptions.AsterixException;
import org.apache.asterix.common.exceptions.ErrorCode;
@@ -28,6 +32,7 @@ import
org.apache.hyracks.algebricks.core.algebra.operators.physical.AbstractJoi
import
org.apache.hyracks.algebricks.core.algebra.operators.physical.AbstractStableSortPOperator;
import
org.apache.hyracks.algebricks.core.algebra.operators.physical.WindowPOperator;
import
org.apache.hyracks.algebricks.core.rewriter.base.PhysicalOptimizationConfig;
+import org.apache.hyracks.api.config.IOption;
import org.apache.hyracks.api.config.IOptionType;
import org.apache.hyracks.api.exceptions.SourceLocation;
import org.apache.hyracks.control.common.config.OptionTypes;
@@ -44,7 +49,8 @@ public class OptimizationConfUtil {
}
public static PhysicalOptimizationConfig
createPhysicalOptimizationConf(CompilerProperties compilerProperties,
- Map<String, Object> querySpecificConfig, SourceLocation sourceLoc)
throws AlgebricksException {
+ Map<String, Object> querySpecificConfig, Set<String>
parameterNames, SourceLocation sourceLoc)
+ throws AlgebricksException {
int frameSize = compilerProperties.getFrameSize();
int sortFrameLimit = getSortNumFrames(compilerProperties,
querySpecificConfig, sourceLoc);
int groupFrameLimit =
getFrameLimit(CompilerProperties.COMPILER_GROUPMEMORY_KEY,
@@ -113,6 +119,17 @@ public class OptimizationConfUtil {
physOptConf.setForceJoinOrderMode(forceJoinOrder);
physOptConf.setQueryPlanShapeMode(queryPlanShape);
physOptConf.setColumnFilter(columnFilter);
+
+ // We should have already validated the parameter names at this
point...
+ Set<String> filteredParameterNames = new HashSet<>(parameterNames);
+ filteredParameterNames.removeAll(
+
Arrays.stream(CompilerProperties.Option.values()).map(IOption::ini).collect(Collectors.toSet()));
+ for (String parameterName : filteredParameterNames) {
+ Object parameterValue = querySpecificConfig.get(parameterName);
+ if (parameterValue != null) {
+ physOptConf.setExtensionProperty(parameterName,
parameterValue);
+ }
+ }
return physOptConf;
}
diff --git
a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/rewriter/base/PhysicalOptimizationConfig.java
b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/rewriter/base/PhysicalOptimizationConfig.java
index 3e51a805af..d16715388b 100644
---
a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/rewriter/base/PhysicalOptimizationConfig.java
+++
b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/rewriter/base/PhysicalOptimizationConfig.java
@@ -303,6 +303,24 @@ public class PhysicalOptimizationConfig {
return getBoolean(COLUMN_FILTER,
AlgebricksConfig.COLUMN_FILTER_DEFAULT);
}
+ public void setExtensionProperty(String property, Object value) {
+ if (value instanceof Integer) {
+ setInt(property, (Integer) value);
+ } else if (value instanceof Double) {
+ setDouble(property, (Double) value);
+ } else if (value instanceof Boolean) {
+ setBoolean(property, (Boolean) value);
+ } else if (value instanceof String) {
+ setString(property, (String) value);
+ } else {
+ throw new IllegalArgumentException();
+ }
+ }
+
+ public Object getExtensionProperty(String property) {
+ return properties.getProperty(property);
+ }
+
private void setInt(String property, int value) {
properties.setProperty(property, Integer.toString(value));
}
diff --git
a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/application/ConfigManagerApplicationConfig.java
b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/application/ConfigManagerApplicationConfig.java
index 44f57fa031..f5f508ac37 100644
---
a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/application/ConfigManagerApplicationConfig.java
+++
b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/application/ConfigManagerApplicationConfig.java
@@ -72,11 +72,13 @@ public class ConfigManagerApplicationConfig implements
IApplicationConfig, Seria
@Override
public Set<String> getKeys(String section) {
- return configManager.getOptionNames(section);
+ Set<String> keys = configManager.getOptionNames(section);
+ return keys.isEmpty() ? configManager.getExtensionKeys(section) : keys;
}
private Object get(String section, String key) {
- return get(configManager.lookupOption(section, key));
+ IOption option = configManager.lookupOption(section, key);
+ return option != null ? get(option) :
configManager.getExtensionValue(section, key);
}
@Override
diff --git
a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/config/ConfigManager.java
b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/config/ConfigManager.java
index 85661fe800..a2e43a2f26 100644
---
a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/config/ConfigManager.java
+++
b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/config/ConfigManager.java
@@ -453,6 +453,23 @@ public class ConfigManager implements IConfigManager,
Serializable {
return optionNames;
}
+ public Set<String> getExtensionKeys(String sectionName) {
+ Set<String> extensionKeys = new HashSet<>();
+ for (Map.Entry<String, String> entry :
extensionOptions.get(sectionName)) {
+ extensionKeys.add(entry.getKey());
+ }
+ return extensionKeys;
+ }
+
+ public Object getExtensionValue(String sectionName, String entryKey) {
+ for (Map.Entry<String, String> entry :
extensionOptions.get(sectionName)) {
+ if (entry.getKey().equals(entryKey)) {
+ return entry.getValue();
+ }
+ }
+ return null;
+ }
+
@Override
public Set<IOption> getOptions(Section section) {
return new HashSet<>(getSectionOptionMap(section).values());
