Yingyi Bu has uploaded a new change for review.
https://asterix-gerrit.ics.uci.edu/1008
Change subject: ASTERIXDB-1536: supports fully qualified dataset path in SQL++.
......................................................................
ASTERIXDB-1536: supports fully qualified dataset path in SQL++.
Change-Id: I93c7187b3a363a82dbfa225eb67ab526e04aa2dd
---
M
asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ResolveVariableRule.java
M
asterixdb/asterix-app/src/test/resources/parserts/queries_sqlpp/columnalias2.sqlpp
M
asterixdb/asterix-app/src/test/resources/parserts/results_parser_sqlpp/columnalias2.ast
A
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/custord/join_q_08/join_q_08.1.ddl.sqlpp
A
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/custord/join_q_08/join_q_08.2.update.sqlpp
A
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/custord/join_q_08/join_q_08.3.query.sqlpp
M asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
M
asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/visitor/VariableCheckAndRewriteVisitor.java
8 files changed, 193 insertions(+), 37 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb
refs/changes/08/1008/1
diff --git
a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ResolveVariableRule.java
b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ResolveVariableRule.java
index 4965197..295eb41 100644
---
a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ResolveVariableRule.java
+++
b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ResolveVariableRule.java
@@ -93,13 +93,15 @@
AbstractFunctionCallExpression funcExpr =
(AbstractFunctionCallExpression) expr;
for (Mutable<ILogicalExpression> funcArgRef : funcExpr.getArguments())
{
if (rewriteExpressionReference(op, funcArgRef, context)) {
- context.computeAndSetTypeEnvironmentForOperator(op);
changed = true;
}
}
// Cleans up extra scan-collections if there is.
- cleanupScanCollectionForDataset(funcExpr);
+ if (changed) {
+ cleanupScanCollectionForDataset(funcExpr);
+ context.computeAndSetTypeEnvironmentForOperator(op);
+ }
// Does the actual resolution.
return changed || resolve(op, context, exprRef);
@@ -146,8 +148,11 @@
throw new AlgebricksException(
"Undefined alias (variable) reference for identifier "
+ unresolvedVarName);
}
- // Rewrites the "resolve" function to a "dataset" function.
+ // Rewrites the "resolve" function to a "dataset" function and
only keep the dataset name argument.
func.setFunctionInfo(FunctionUtil.getFunctionInfo(AsterixBuiltinFunctions.DATASET));
+ Mutable<ILogicalExpression> datasetNameExpression =
func.getArguments().get(0);
+ func.getArguments().clear();
+ func.getArguments().add(datasetNameExpression);
} else {
// Rewrites to field-access-by-names.
Pair<LogicalVariable, List<String>> varAndPath =
varAccessCandidates.iterator().next();
diff --git
a/asterixdb/asterix-app/src/test/resources/parserts/queries_sqlpp/columnalias2.sqlpp
b/asterixdb/asterix-app/src/test/resources/parserts/queries_sqlpp/columnalias2.sqlpp
index 8eeea85..f74cf47 100644
---
a/asterixdb/asterix-app/src/test/resources/parserts/queries_sqlpp/columnalias2.sqlpp
+++
b/asterixdb/asterix-app/src/test/resources/parserts/queries_sqlpp/columnalias2.sqlpp
@@ -21,4 +21,4 @@
GROUP BY root.id
WITH u AS root.time
HAVING root.orders > 0
-ORDER BY u;
\ No newline at end of file
+ORDER BY u;
\ No newline at end of file
diff --git
a/asterixdb/asterix-app/src/test/resources/parserts/results_parser_sqlpp/columnalias2.ast
b/asterixdb/asterix-app/src/test/resources/parserts/results_parser_sqlpp/columnalias2.ast
index 0b7ccd2..89e3b4b 100644
---
a/asterixdb/asterix-app/src/test/resources/parserts/results_parser_sqlpp/columnalias2.ast
+++
b/asterixdb/asterix-app/src/test/resources/parserts/results_parser_sqlpp/columnalias2.ast
@@ -2,18 +2,12 @@
SELECT [
FunctionCall null.SQRT@1[
OperatorExpr [
- FieldAccessor [
- FunctionCall Metadata.dataset@1[
- LiteralExpr [STRING] [t]
- ]
- Field=a
+ FunctionCall Metadata.dataset@1[
+ LiteralExpr [STRING] [t.a]
]
*
- FieldAccessor [
- FunctionCall Metadata.dataset@1[
- LiteralExpr [STRING] [t]
- ]
- Field=b
+ FunctionCall Metadata.dataset@1[
+ LiteralExpr [STRING] [t.b]
]
]
]
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/custord/join_q_08/join_q_08.1.ddl.sqlpp
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/custord/join_q_08/join_q_08.1.ddl.sqlpp
new file mode 100644
index 0000000..193d6b4
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/custord/join_q_08/join_q_08.1.ddl.sqlpp
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+drop database test if exists;
+create database test;
+
+use test;
+
+
+create type test.AddressType as
+{
+ number : int64,
+ street : string,
+ city : string
+}
+
+create type test.CustomerType as
+ closed {
+ cid : int64,
+ name : string,
+ cashBack : int64,
+ age : int64?,
+ address : AddressType?,
+ lastorder : {
+ oid : int64,
+ total : float
+ }
+
+}
+
+create type test.OrderType as
+{
+ oid : int64,
+ cid : int64,
+ orderstatus : string,
+ orderpriority : string,
+ clerk : string,
+ total : float,
+ items : [int64]
+}
+
+create external table Customers(CustomerType) using
localfs((`path`=`asterix_nc1://data/nontagged/customerData.json`),(`format`=`adm`));
+
+create external table Orders(OrderType) using
localfs((`path`=`asterix_nc1://data/nontagged/orderData.json`),(`format`=`adm`));
+
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/custord/join_q_08/join_q_08.2.update.sqlpp
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/custord/join_q_08/join_q_08.2.update.sqlpp
new file mode 100644
index 0000000..6c98c1e
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/custord/join_q_08/join_q_08.2.update.sqlpp
@@ -0,0 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/custord/join_q_08/join_q_08.3.query.sqlpp
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/custord/join_q_08/join_q_08.3.query.sqlpp
new file mode 100644
index 0000000..c33377e
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/custord/join_q_08/join_q_08.3.query.sqlpp
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/** This query is to test fully qualified path for datasets. */
+
+SELECT c.name AS cust_name,
+ c.age AS cust_age,
+ o.total AS order_total,
+ [o.oid,o.cid] AS orderList
+FROM test.Customers c JOIN test.Orders o ON c.cid = o.cid
+ORDER BY c.name,o.total
+;
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
index e3d3102..6a19c93 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
@@ -1349,7 +1349,12 @@
<test-case FilePath="custord">
<compilation-unit name="join_q_07">
<output-dir compare="Text">join_q_06</output-dir>
- <expected-error>Undefined alias (variable) reference for identifier
c</expected-error>
+ <expected-error>Cannot find dataset c in dataverse test nor a variable
with name c</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="custord">
+ <compilation-unit name="join_q_08">
+ <output-dir compare="Text">join_q_01</output-dir>
</compilation-unit>
</test-case>
<test-case FilePath="custord">
@@ -2855,7 +2860,7 @@
<output-dir compare="Text">partition-by-nonexistent-field</output-dir>
<expected-error>java.lang.NullPointerException</expected-error>
<expected-error>Cannot find dataset</expected-error>
- <expected-error>Undefined alias (variable) reference for identifier
testds</expected-error>
+ <expected-error>Cannot find dataset testds in dataverse test nor a
variable with name testds</expected-error>
</compilation-unit>
</test-case>
<test-case FilePath="misc">
@@ -6666,7 +6671,7 @@
<test-case FilePath="user-defined-functions">
<compilation-unit name="udf30">
<output-dir compare="Text">udf30</output-dir>
- <expected-error>Undefined alias (variable) reference for identifier
y</expected-error>
+ <expected-error>Cannot find dataset y in dataverse null nor a variable
with name y!</expected-error>
</compilation-unit>
</test-case>
<test-case FilePath="user-defined-functions">
diff --git
a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/visitor/VariableCheckAndRewriteVisitor.java
b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/visitor/VariableCheckAndRewriteVisitor.java
index 7082c25..ba26ea4 100644
---
a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/visitor/VariableCheckAndRewriteVisitor.java
+++
b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/visitor/VariableCheckAndRewriteVisitor.java
@@ -26,8 +26,10 @@
import org.apache.asterix.common.exceptions.AsterixException;
import org.apache.asterix.common.functions.FunctionSignature;
import org.apache.asterix.lang.common.base.Expression;
+import org.apache.asterix.lang.common.base.Expression.Kind;
import org.apache.asterix.lang.common.base.ILangExpression;
import org.apache.asterix.lang.common.expression.CallExpr;
+import org.apache.asterix.lang.common.expression.FieldAccessor;
import org.apache.asterix.lang.common.expression.LiteralExpr;
import org.apache.asterix.lang.common.expression.VariableExpr;
import org.apache.asterix.lang.common.literal.StringLiteral;
@@ -41,8 +43,8 @@
public class VariableCheckAndRewriteVisitor extends
AbstractSqlppExpressionScopingVisitor {
- protected final FunctionSignature datasetFunction = new
FunctionSignature(MetadataConstants.METADATA_DATAVERSE_NAME,
- "dataset", 1);
+ protected final FunctionSignature datasetFunction =
+ new FunctionSignature(MetadataConstants.METADATA_DATAVERSE_NAME,
"dataset", 1);
protected final boolean overwrite;
protected final AqlMetadataProvider metadataProvider;
@@ -62,23 +64,65 @@
}
@Override
+ public Expression visit(FieldAccessor fa, ILangExpression arg) throws
AsterixException {
+ Expression leadingExpr = fa.getExpr();
+ if (leadingExpr.getKind() != Kind.VARIABLE_EXPRESSION) {
+ fa.setExpr(leadingExpr.accept(this, fa));
+ return fa;
+ } else {
+ VariableExpr varExpr = (VariableExpr) leadingExpr;
+ String lastIdentifier = fa.getIdent().getValue();
+ Expression resolvedExpr = resolve(varExpr,
+ /** Resolves within the dataverse that has the same name
as the variable name. */
+
SqlppVariableUtil.toUserDefinedVariableName(varExpr.getVar().getValue()).getValue(),
lastIdentifier,
+ arg);
+ if (resolvedExpr.getKind() == Kind.CALL_EXPRESSION) {
+ CallExpr callExpr = (CallExpr) resolvedExpr;
+ if (callExpr.getFunctionSignature().equals(datasetFunction)) {
+ // The field access is resolved to be a dataset access in
the form of "dataverse.dataset".
+ return resolvedExpr;
+ }
+ }
+ fa.setExpr(resolvedExpr);
+ return fa;
+ }
+ }
+
+ @Override
public Expression visit(VariableExpr varExpr, ILangExpression arg) throws
AsterixException {
+ return resolve(varExpr, null /** Resolves within the default
dataverse. */
+ ,
SqlppVariableUtil.toUserDefinedVariableName(varExpr.getVar().getValue()).getValue(),
arg);
+ }
+
+ // Resolve a variable expression with dataverse name and dataset name.
+ private Expression resolve(VariableExpr varExpr, String dataverseName,
String datasetName, ILangExpression arg)
+ throws AsterixException {
String varName = varExpr.getVar().getValue();
+ checkError(varName);
+ if (!rewriteNeeded(varExpr)) {
+ return varExpr;
+ }
+ Set<VariableExpr> liveVars =
SqlppVariableUtil.getLiveUserDefinedVariables(scopeChecker.getCurrentScope());
+ boolean resolveAsDataset = resolveDatasetFirst(arg) &&
datasetExists(dataverseName, datasetName);
+ if (resolveAsDataset) {
+ return wrapWithDatasetFunction(dataverseName, datasetName);
+ } else if (liveVars.isEmpty()) {
+ // If no available dataset nor in-scope variable to resolve to, we
throw an error.
+ throw new AsterixException("Cannot find dataset " + datasetName +
" in dataverse "
+ + (dataverseName == null ?
metadataProvider.getDefaultDataverseName() : dataverseName)
+ + " nor a variable with name " + datasetName + "!");
+ }
+ return wrapWithResolveFunction(varExpr, liveVars);
+ }
+
+ // Checks whether we need to error the variable reference, e.g., the
variable is referred
+ // in a LIMIT clause.
+ private void checkError(String varName) throws AsterixException {
if (scopeChecker.isInForbiddenScopes(varName)) {
throw new AsterixException(
"Inside limit clauses, it is disallowed to reference a
variable having the same name"
+ " as any variable bound in the same scope as the
limit clause.");
}
- if (!rewriteNeeded(varExpr)) {
- return varExpr;
- }
- boolean resolveAsDataset = resolveDatasetFirst(arg)
- &&
datasetExists(SqlppVariableUtil.toUserDefinedVariableName(varName).getValue());
- if (resolveAsDataset) {
- return wrapWithDatasetFunction(varExpr);
- }
- Set<VariableExpr> liveVars =
SqlppVariableUtil.getLiveUserDefinedVariables(scopeChecker.getCurrentScope());
- return wrapWithResolveFunction(varExpr, liveVars);
}
// For From/Join/UNNEST/NEST, we resolve the undefined identifier
reference as dataset reference first.
@@ -101,26 +145,25 @@
}
}
- private Expression wrapWithDatasetFunction(VariableExpr expr) throws
AsterixException {
+ private Expression wrapWithDatasetFunction(String dataverseName, String
datasetName) throws AsterixException {
+ String fullyQualifiedName = dataverseName == null ? datasetName :
dataverseName + "." + datasetName;
List<Expression> argList = new ArrayList<>();
- //Ignore the parser-generated prefix "$" for a dataset.
- String varName =
SqlppVariableUtil.toUserDefinedVariableName(expr.getVar()).getValue();
- argList.add(new LiteralExpr(new StringLiteral(varName)));
+ argList.add(new LiteralExpr(new StringLiteral(fullyQualifiedName)));
return new CallExpr(datasetFunction, argList);
}
- private boolean datasetExists(String name) throws AsterixException {
+ private boolean datasetExists(String dataverseName, String datasetName)
throws AsterixException {
try {
- if (metadataProvider.findDataset(null, name) != null) {
+ if (metadataProvider.findDataset(dataverseName, datasetName) !=
null) {
return true;
}
- return pathDatasetExists(name);
+ return fullyQualifiedDatasetNameExists(datasetName);
} catch (AlgebricksException e) {
throw new AsterixException(e);
}
}
- private boolean pathDatasetExists(String name) throws AlgebricksException {
+ private boolean fullyQualifiedDatasetNameExists(String name) throws
AlgebricksException {
if (!name.contains(".")) {
return false;
}
--
To view, visit https://asterix-gerrit.ics.uci.edu/1008
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I93c7187b3a363a82dbfa225eb67ab526e04aa2dd
Gerrit-PatchSet: 1
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Yingyi Bu <[email protected]>
