This is an automated email from the ASF dual-hosted git repository.
wyk pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
The following commit(s) were added to refs/heads/master by this push:
new b27f0db529 [NO ISSUE][COMP] Fixes and cleanup for external filters
b27f0db529 is described below
commit b27f0db529e287140b92e1b9213466aa7bb55df4
Author: Wail Alkowaileet <wael....@gmail.com>
AuthorDate: Tue Aug 8 11:03:09 2023 -0700
[NO ISSUE][COMP] Fixes and cleanup for external filters
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
- Enable filter pushdowns for external datasets
- Always register external dataset scans
- Make ExternalDataPrefix with KEY_PATH
Change-Id: I08b509fca0eb3ab9aad82e816dfe4849adf725eb
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17707
Integration-Tests: Jenkins <jenk...@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <jenk...@fulliautomatix.ics.uci.edu>
Reviewed-by: Hussain Towaileb <hussai...@gmail.com>
---
.../optimizer/rules/PushValueAccessAndFilterDownRule.java | 8 ++++----
.../optimizer/rules/pushdown/PushdownProcessorsExecutor.java | 3 ++-
.../rules/pushdown/visitor/PushdownOperatorVisitor.java | 9 ++-------
.../org/apache/asterix/external/util/ExternalDataPrefix.java | 7 ++++++-
.../main/java/org/apache/asterix/metadata/utils/DatasetUtil.java | 4 ++--
5 files changed, 16 insertions(+), 15 deletions(-)
diff --git
a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/PushValueAccessAndFilterDownRule.java
b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/PushValueAccessAndFilterDownRule.java
index 3b298d07a3..c74c2930e5 100644
---
a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/PushValueAccessAndFilterDownRule.java
+++
b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/PushValueAccessAndFilterDownRule.java
@@ -113,11 +113,11 @@ public class PushValueAccessAndFilterDownRule implements
IAlgebraicRewriteRule {
pushdownProcessorsExecutor.add(new
ColumnFilterPushdownProcessor(pushdownContext, context));
// Performs range-filter pushdowns
pushdownProcessorsExecutor.add(new
ColumnRangeFilterPushdownProcessor(pushdownContext, context));
- // Performs prefix pushdowns
- pushdownProcessorsExecutor.add(new
ExternalDatasetFilterPushdownProcessor(pushdownContext, context));
- // Inlines AND/OR expression (must be last to run)
- pushdownProcessorsExecutor.add(new
InlineFilterExpressionsProcessor(pushdownContext, context));
}
+ // Performs prefix pushdowns
+ pushdownProcessorsExecutor.add(new
ExternalDatasetFilterPushdownProcessor(pushdownContext, context));
+ // Inlines AND/OR expression (must be last to run)
+ pushdownProcessorsExecutor.add(new
InlineFilterExpressionsProcessor(pushdownContext, context));
}
/**
diff --git
a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/pushdown/PushdownProcessorsExecutor.java
b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/pushdown/PushdownProcessorsExecutor.java
index 1530f55a8c..ab6a26cd6a 100644
---
a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/pushdown/PushdownProcessorsExecutor.java
+++
b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/pushdown/PushdownProcessorsExecutor.java
@@ -67,7 +67,8 @@ public class PushdownProcessorsExecutor {
if (dataset.getDatasetFormatInfo().getFormat() ==
DatasetConfig.DatasetFormat.COLUMN) {
info = createInternalColumnarDatasetInfo(scanDefineDescriptor,
context);
} else if (dataset.getDatasetType() ==
DatasetConfig.DatasetType.EXTERNAL
- && DatasetUtil.isFieldAccessPushdownSupported(dataset)) {
+ && (DatasetUtil.isFieldAccessPushdownSupported(dataset)
+ ||
DatasetUtil.isFilterPushdownSupported(dataset))) {
info =
createExternalDatasetProjectionInfo(scanDefineDescriptor, context);
}
setInfoToDataScan(scanOp, info);
diff --git
a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/pushdown/visitor/PushdownOperatorVisitor.java
b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/pushdown/visitor/PushdownOperatorVisitor.java
index 65b131a5e6..dd9acd5ee8 100644
---
a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/pushdown/visitor/PushdownOperatorVisitor.java
+++
b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/pushdown/visitor/PushdownOperatorVisitor.java
@@ -26,13 +26,11 @@ import java.util.Set;
import org.apache.asterix.common.config.DatasetConfig;
import org.apache.asterix.common.config.DatasetConfig.DatasetFormat;
import org.apache.asterix.common.metadata.DataverseName;
-import org.apache.asterix.external.util.ExternalDataUtils;
import org.apache.asterix.metadata.declared.DataSource;
import org.apache.asterix.metadata.declared.DataSourceId;
import org.apache.asterix.metadata.declared.DatasetDataSource;
import org.apache.asterix.metadata.declared.MetadataProvider;
import org.apache.asterix.metadata.entities.Dataset;
-import org.apache.asterix.metadata.entities.ExternalDatasetDetails;
import org.apache.asterix.om.functions.BuiltinFunctions;
import org.apache.asterix.om.utils.ConstantExpressionUtil;
import org.apache.asterix.optimizer.rules.pushdown.PushdownContext;
@@ -202,11 +200,8 @@ public class PushdownOperatorVisitor implements
ILogicalOperatorVisitor<Void, Vo
Dataset dataset = getDataset(dataSource);
//Only external dataset can have pushed down expressions
- if (dataset.getDatasetType() == DatasetConfig.DatasetType.EXTERNAL
- && !ExternalDataUtils
- .supportsPushdown(((ExternalDatasetDetails)
dataset.getDatasetDetails()).getProperties())
- || dataset.getDatasetType() ==
DatasetConfig.DatasetType.INTERNAL
- && dataset.getDatasetFormatInfo().getFormat() ==
DatasetFormat.ROW) {
+ if (dataset.getDatasetType() == DatasetConfig.DatasetType.INTERNAL
+ && dataset.getDatasetFormatInfo().getFormat() ==
DatasetFormat.ROW) {
return null;
}
diff --git
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataPrefix.java
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataPrefix.java
index 899a179f63..ec56009ad8 100644
---
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataPrefix.java
+++
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataPrefix.java
@@ -21,6 +21,7 @@ package org.apache.asterix.external.util;
import static
org.apache.asterix.external.util.ExternalDataConstants.COMPUTED_FIELD_PATTERN;
import static
org.apache.asterix.external.util.ExternalDataConstants.DEFINITION_FIELD_NAME;
+import static org.apache.asterix.external.util.ExternalDataConstants.KEY_PATH;
import static
org.apache.asterix.external.util.ExternalDataConstants.PREFIX_DEFAULT_DELIMITER;
import java.util.ArrayList;
@@ -68,7 +69,7 @@ public class ExternalDataPrefix {
}
public ExternalDataPrefix(Map<String, String> configuration) throws
AlgebricksException {
- this(configuration.get(DEFINITION_FIELD_NAME));
+ this(getDefinitionOrPath(configuration));
}
public ExternalDataPrefix(String prefix) throws AlgebricksException {
@@ -245,4 +246,8 @@ public class ExternalDataPrefix {
return values;
}
+
+ private static String getDefinitionOrPath(Map<String, String>
configuration) {
+ return configuration.getOrDefault(DEFINITION_FIELD_NAME,
configuration.get(KEY_PATH));
+ }
}
diff --git
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DatasetUtil.java
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DatasetUtil.java
index 88eafcd254..44964f3949 100644
---
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DatasetUtil.java
+++
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DatasetUtil.java
@@ -699,8 +699,8 @@ public class DatasetUtil {
if (dataset.getDatasetType() == DatasetType.INTERNAL) {
return dataset.getDatasetFormatInfo().getFormat() ==
DatasetConfig.DatasetFormat.COLUMN;
}
- // TODO add external dataset with dynamic prefixes
- return false;
+ // External dataset support filter pushdown
+ return true;
}
public static boolean isRangeFilterPushdownSupported(Dataset dataset) {
