>From Hussain Towaileb <[email protected]>:
Hussain Towaileb has uploaded this change for review. (
https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/18146 )
Change subject: [ASTERIXDB-3347][COMP] Refactor COPY TO to support different
types of write destinations
......................................................................
[ASTERIXDB-3347][COMP] Refactor COPY TO to support different types of write
destinations
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
This change refactors the COPY TO KV code
to allow for easier extendability for supporting
other types of destinations to write to.
Change-Id: Id90a30c1e9f41ca82ef28f6edd0569a69002572c
---
M
asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/FormatPrintVisitor.java
M
asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/base/ILangVisitor.java
M
asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/CopyToStatement.java
M
asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/base/AbstractSqlppContainsExpressionVisitor.java
M
asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
M
asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java
M
asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/CheckSubqueryVisitor.java
M
asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/base/AbstractSqlppExpressionScopingVisitor.java
M
asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/visitor/SqlppCopyToRewriteVisitor.java
M
asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/base/AbstractSqlppSimpleExpressionVisitor.java
M
asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/FreeVariableVisitor.java
M
asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/AbstractInlineUdfsVisitor.java
M asterixdb/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj
M
asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/CompiledStatements.java
M
asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/GatherFunctionCallsVisitor.java
M
asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/CheckDatasetOnlyResolutionVisitor.java
A
asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/CopyToCloudStoreStatement.java
M
asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/base/AbstractQueryExpressionVisitor.java
18 files changed, 209 insertions(+), 130 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb
refs/changes/46/18146/1
diff --git
a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/CompiledStatements.java
b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/CompiledStatements.java
index 0ff0b72..2c21c4c 100644
---
a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/CompiledStatements.java
+++
b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/CompiledStatements.java
@@ -27,7 +27,7 @@
import org.apache.asterix.lang.common.base.Statement;
import org.apache.asterix.lang.common.clause.OrderbyClause;
import org.apache.asterix.lang.common.expression.VariableExpr;
-import org.apache.asterix.lang.common.statement.CopyToStatement;
+import org.apache.asterix.lang.common.statement.CopyToCloudStoreStatement;
import org.apache.asterix.lang.common.statement.ExternalDetailsDecl;
import org.apache.asterix.lang.common.statement.Query;
import org.apache.asterix.metadata.entities.Dataset;
@@ -591,7 +591,7 @@
}
}
- public static class CompiledCopyToStatement extends
AbstractCompiledStatement {
+ public static class CompileCopyToCloudStoreStatement extends
AbstractCompiledStatement {
private final Query query;
private final VariableExpr sourceVariable;
private final String adapter;
@@ -603,7 +603,7 @@
private final List<OrderbyClause.OrderModifier> orderByModifiers;
private final List<OrderbyClause.NullOrderModifier>
orderByNullModifierList;
- public CompiledCopyToStatement(CopyToStatement copyToStatement) {
+ public CompileCopyToCloudStoreStatement(CopyToCloudStoreStatement
copyToStatement) {
this.query = copyToStatement.getQuery();
this.sourceVariable = copyToStatement.getSourceVariable();
ExternalDetailsDecl eddDecl =
copyToStatement.getExternalDetailsDecl();
diff --git
a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java
b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java
index 7dd0217..8e846b0 100644
---
a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java
+++
b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java
@@ -104,6 +104,7 @@
import org.apache.asterix.om.types.BuiltinType;
import org.apache.asterix.om.types.IAType;
import org.apache.asterix.om.utils.ConstantExpressionUtil;
+import
org.apache.asterix.translator.CompiledStatements.CompileCopyToCloudStoreStatement;
import
org.apache.asterix.translator.CompiledStatements.CompiledCopyFromFileStatement;
import
org.apache.asterix.translator.CompiledStatements.CompiledInsertStatement;
import
org.apache.asterix.translator.CompiledStatements.CompiledLoadFromFileStatement;
@@ -363,7 +364,7 @@
private ILogicalPlan translateCopyTo(Query expr,
CompiledStatements.ICompiledStatement stmt,
IResultMetadata resultMetadata) throws AlgebricksException {
- CompiledStatements.CompiledCopyToStatement copyTo =
(CompiledStatements.CompiledCopyToStatement) stmt;
+ CompileCopyToCloudStoreStatement copyTo =
(CompileCopyToCloudStoreStatement) stmt;
MutableObject<ILogicalOperator> base = new MutableObject<>(new
EmptyTupleSourceOperator());
Pair<ILogicalOperator, LogicalVariable> p = expr.accept(this, base);
ArrayList<Mutable<ILogicalOperator>> globalPlanRoots = new
ArrayList<>();
diff --git
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
index 30c7127..ab90fea 100644
---
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
+++
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
@@ -125,6 +125,7 @@
import org.apache.asterix.lang.common.statement.CompactStatement;
import org.apache.asterix.lang.common.statement.ConnectFeedStatement;
import org.apache.asterix.lang.common.statement.CopyFromStatement;
+import org.apache.asterix.lang.common.statement.CopyToCloudStoreStatement;
import org.apache.asterix.lang.common.statement.CopyToStatement;
import org.apache.asterix.lang.common.statement.CreateAdapterStatement;
import org.apache.asterix.lang.common.statement.CreateDatabaseStatement;
@@ -225,7 +226,7 @@
import
org.apache.asterix.transaction.management.service.transaction.GlobalTxInfo;
import org.apache.asterix.translator.AbstractLangTranslator;
import org.apache.asterix.translator.ClientRequest;
-import org.apache.asterix.translator.CompiledStatements;
+import
org.apache.asterix.translator.CompiledStatements.CompileCopyToCloudStoreStatement;
import
org.apache.asterix.translator.CompiledStatements.CompiledCopyFromFileStatement;
import
org.apache.asterix.translator.CompiledStatements.CompiledDeleteStatement;
import
org.apache.asterix.translator.CompiledStatements.CompiledInsertStatement;
@@ -4026,7 +4027,14 @@
IHyracksClientConnection hcc, IResultSet resultSet, ResultDelivery
resultDelivery,
ResultMetadata outMetadata, IRequestParameters requestParameters,
Map<String, IAObject> stmtParams,
Stats stats) throws Exception {
- CopyToStatement copyTo = (CopyToStatement) stmt;
+ CopyToStatement copyTo;
+ if (stmt instanceof CopyToCloudStoreStatement) {
+ copyTo = (CopyToCloudStoreStatement) stmt;
+ } else {
+ // TODO(htowaileb)
+ throw new IllegalStateException("NYI");
+ }
+
final IRequestTracker requestTracker = appCtx.getRequestTracker();
final ClientRequest clientRequest =
(ClientRequest)
requestTracker.get(requestParameters.getRequestReference().getUuid());
@@ -4066,8 +4074,8 @@
Pair<IReturningStatement, Integer> rewrittenResult =
apiFramework.reWriteQuery(langRewritingContext,
copyTo, sessionOutput, true, true,
externalVars.keySet());
- CompiledStatements.CompiledCopyToStatement
compiledCopyToStatement =
- new CompiledStatements.CompiledCopyToStatement(copyTo);
+ CompileCopyToCloudStoreStatement compiledCopyToStatement =
+ new
CompileCopyToCloudStoreStatement((CopyToCloudStoreStatement) copyTo);
// Query Compilation (happens under the same ongoing metadata
transaction)
final JobSpecification jobSpec =
apiFramework.compileQuery(hcc, metadataProvider, copyTo.getQuery(),
diff --git
a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/CopyToCloudStoreStatement.java
b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/CopyToCloudStoreStatement.java
new file mode 100644
index 0000000..183a0d2
--- /dev/null
+++
b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/CopyToCloudStoreStatement.java
@@ -0,0 +1,130 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.lang.common.statement;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.asterix.common.exceptions.CompilationException;
+import org.apache.asterix.common.metadata.Namespace;
+import org.apache.asterix.lang.common.base.Expression;
+import org.apache.asterix.lang.common.base.IReturningStatement;
+import org.apache.asterix.lang.common.clause.OrderbyClause;
+import org.apache.asterix.lang.common.expression.LiteralExpr;
+import org.apache.asterix.lang.common.expression.VariableExpr;
+import org.apache.asterix.lang.common.literal.StringLiteral;
+import org.apache.asterix.lang.common.visitor.base.ILangVisitor;
+
+public class CopyToCloudStoreStatement extends CopyToStatement implements
IReturningStatement {
+ private final Map<Integer, VariableExpr> partitionsVariables;
+ private final List<OrderbyClause.OrderModifier> orderByModifiers;
+ private final List<OrderbyClause.NullOrderModifier>
orderByNullModifierList;
+
+ private List<Expression> pathExpressions;
+
+ private List<Expression> partitionExpressions;
+ private List<Expression> orderByList;
+
+ public CopyToCloudStoreStatement(Namespace namespace, String datasetName,
Query query, VariableExpr sourceVariable,
+ ExternalDetailsDecl externalDetailsDecl, List<Expression>
pathExpressions,
+ List<Expression> partitionExpressions, Map<Integer, VariableExpr>
partitionsVariables,
+ List<Expression> orderbyList, List<OrderbyClause.OrderModifier>
orderByModifiers,
+ List<OrderbyClause.NullOrderModifier> orderByNullModifierList, int
varCounter) {
+ super(namespace, datasetName, query, sourceVariable, varCounter,
externalDetailsDecl);
+ this.pathExpressions = pathExpressions;
+ this.partitionExpressions = partitionExpressions;
+ this.partitionsVariables = partitionsVariables;
+ this.orderByList = orderbyList;
+ this.orderByModifiers = orderByModifiers;
+ this.orderByNullModifierList = orderByNullModifierList;
+
+ if (pathExpressions.isEmpty()) {
+ // Ensure path expressions to have at least an empty string
+ pathExpressions.add(new LiteralExpr(new StringLiteral("")));
+ }
+ }
+
+ @Override
+ public <R, T> R accept(ILangVisitor<R, T> visitor, T arg) throws
CompilationException {
+ return visitor.visit(this, arg);
+ }
+
+ public List<Expression> getPathExpressions() {
+ return pathExpressions;
+ }
+
+ public void setPathExpressions(List<Expression> pathExpressions) {
+ if (pathExpressions.isEmpty()) {
+ pathExpressions.add(new LiteralExpr(new StringLiteral("")));
+ }
+ this.pathExpressions = pathExpressions;
+ }
+
+ public List<Expression> getPartitionExpressions() {
+ return partitionExpressions;
+ }
+
+ public void setPartitionExpressions(List<Expression> partitionExpressions)
{
+ this.partitionExpressions = partitionExpressions;
+ }
+
+ public Map<Integer, VariableExpr> getPartitionsVariables() {
+ return partitionsVariables;
+ }
+
+ public List<Expression> getOrderByList() {
+ return orderByList;
+ }
+
+ public void setOrderByList(List<Expression> orderbyList) {
+ this.orderByList = orderbyList;
+ }
+
+ public List<OrderbyClause.OrderModifier> getOrderByModifiers() {
+ return orderByModifiers;
+ }
+
+ public List<OrderbyClause.NullOrderModifier> getOrderByNullModifierList() {
+ return orderByNullModifierList;
+ }
+
+ public boolean hasOverClause() {
+ return hasPartitionClause() || hasOrderClause();
+ }
+
+ public boolean hasPartitionClause() {
+ return !partitionExpressions.isEmpty();
+ }
+
+ public boolean hasOrderClause() {
+ return !orderByList.isEmpty();
+ }
+
+ @Override
+ public List<Expression> getDirectlyEnclosedExpressions() {
+ List<Expression> topLevelExpressions = new ArrayList<>();
+ topLevelExpressions.add(query.getBody());
+ topLevelExpressions.addAll(pathExpressions);
+ topLevelExpressions.addAll(partitionExpressions);
+ topLevelExpressions.addAll(orderByList);
+ return topLevelExpressions;
+ }
+
+}
diff --git
a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/CopyToStatement.java
b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/CopyToStatement.java
index dbe7b35..f0ad3f4 100644
---
a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/CopyToStatement.java
+++
b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/CopyToStatement.java
@@ -22,62 +22,33 @@
import java.util.ArrayList;
import java.util.List;
-import java.util.Map;
-import org.apache.asterix.common.exceptions.CompilationException;
import org.apache.asterix.common.metadata.Namespace;
import org.apache.asterix.lang.common.base.AbstractStatement;
import org.apache.asterix.lang.common.base.Expression;
import org.apache.asterix.lang.common.base.IReturningStatement;
-import org.apache.asterix.lang.common.clause.OrderbyClause;
-import org.apache.asterix.lang.common.expression.LiteralExpr;
import org.apache.asterix.lang.common.expression.VariableExpr;
-import org.apache.asterix.lang.common.literal.StringLiteral;
-import org.apache.asterix.lang.common.visitor.base.ILangVisitor;
-public class CopyToStatement extends AbstractStatement implements
IReturningStatement {
- private final String datasetName;
- private final VariableExpr sourceVariable;
+public abstract class CopyToStatement extends AbstractStatement implements
IReturningStatement {
+ // TODO(htowaileb): consider replacing this to just a map
private final ExternalDetailsDecl externalDetailsDecl;
- private final Map<Integer, VariableExpr> partitionsVariables;
- private final List<OrderbyClause.OrderModifier> orderByModifiers;
- private final List<OrderbyClause.NullOrderModifier>
orderByNullModifierList;
- private Namespace namespace;
- private Query query;
- private List<Expression> pathExpressions;
+ final String datasetName;
+ final VariableExpr sourceVariable;
- private List<Expression> partitionExpressions;
- private List<Expression> orderByList;
- private int varCounter;
+ Namespace namespace;
+ Query query;
+
+ int varCounter;
public CopyToStatement(Namespace namespace, String datasetName, Query
query, VariableExpr sourceVariable,
- ExternalDetailsDecl externalDetailsDecl, List<Expression>
pathExpressions,
- List<Expression> partitionExpressions, Map<Integer, VariableExpr>
partitionsVariables,
- List<Expression> orderbyList, List<OrderbyClause.OrderModifier>
orderByModifiers,
- List<OrderbyClause.NullOrderModifier> orderByNullModifierList, int
varCounter) {
+ int varCounter, ExternalDetailsDecl externalDetailsDecl) {
this.namespace = namespace;
this.datasetName = datasetName;
this.query = query;
this.sourceVariable = sourceVariable;
- this.externalDetailsDecl = externalDetailsDecl;
- this.pathExpressions = pathExpressions;
- this.partitionExpressions = partitionExpressions;
- this.partitionsVariables = partitionsVariables;
- this.orderByList = orderbyList;
- this.orderByModifiers = orderByModifiers;
- this.orderByNullModifierList = orderByNullModifierList;
this.varCounter = varCounter;
-
- if (pathExpressions.isEmpty()) {
- // Ensure path expressions to have at least an empty string
- pathExpressions.add(new LiteralExpr(new StringLiteral("")));
- }
- }
-
- @Override
- public <R, T> R accept(ILangVisitor<R, T> visitor, T arg) throws
CompilationException {
- return visitor.visit(this, arg);
+ this.externalDetailsDecl = externalDetailsDecl;
}
@Override
@@ -114,61 +85,6 @@
return sourceVariable;
}
- public ExternalDetailsDecl getExternalDetailsDecl() {
- return externalDetailsDecl;
- }
-
- public List<Expression> getPathExpressions() {
- return pathExpressions;
- }
-
- public void setPathExpressions(List<Expression> pathExpressions) {
- if (pathExpressions.isEmpty()) {
- pathExpressions.add(new LiteralExpr(new StringLiteral("")));
- }
- this.pathExpressions = pathExpressions;
- }
-
- public List<Expression> getPartitionExpressions() {
- return partitionExpressions;
- }
-
- public void setPartitionExpressions(List<Expression> partitionExpressions)
{
- this.partitionExpressions = partitionExpressions;
- }
-
- public Map<Integer, VariableExpr> getPartitionsVariables() {
- return partitionsVariables;
- }
-
- public List<Expression> getOrderByList() {
- return orderByList;
- }
-
- public void setOrderByList(List<Expression> orderbyList) {
- this.orderByList = orderbyList;
- }
-
- public List<OrderbyClause.OrderModifier> getOrderByModifiers() {
- return orderByModifiers;
- }
-
- public List<OrderbyClause.NullOrderModifier> getOrderByNullModifierList() {
- return orderByNullModifierList;
- }
-
- public boolean hasOverClause() {
- return hasPartitionClause() || hasOrderClause();
- }
-
- public boolean hasPartitionClause() {
- return !partitionExpressions.isEmpty();
- }
-
- public boolean hasOrderClause() {
- return !orderByList.isEmpty();
- }
-
@Override
public int getVarCounter() {
return varCounter;
@@ -179,6 +95,10 @@
this.varCounter = varCounter;
}
+ public ExternalDetailsDecl getExternalDetailsDecl() {
+ return externalDetailsDecl;
+ }
+
@Override
public boolean isTopLevel() {
return true;
@@ -188,9 +108,6 @@
public List<Expression> getDirectlyEnclosedExpressions() {
List<Expression> topLevelExpressions = new ArrayList<>();
topLevelExpressions.add(query.getBody());
- topLevelExpressions.addAll(pathExpressions);
- topLevelExpressions.addAll(partitionExpressions);
- topLevelExpressions.addAll(orderByList);
return topLevelExpressions;
}
diff --git
a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/AbstractInlineUdfsVisitor.java
b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/AbstractInlineUdfsVisitor.java
index 8c7b915..034a823 100644
---
a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/AbstractInlineUdfsVisitor.java
+++
b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/AbstractInlineUdfsVisitor.java
@@ -52,7 +52,7 @@
import org.apache.asterix.lang.common.expression.VariableExpr;
import org.apache.asterix.lang.common.rewrites.LangRewritingContext;
import org.apache.asterix.lang.common.rewrites.VariableSubstitutionEnvironment;
-import org.apache.asterix.lang.common.statement.CopyToStatement;
+import org.apache.asterix.lang.common.statement.CopyToCloudStoreStatement;
import org.apache.asterix.lang.common.statement.FunctionDecl;
import org.apache.asterix.lang.common.statement.InsertStatement;
import org.apache.asterix.lang.common.statement.Query;
@@ -283,7 +283,7 @@
}
@Override
- public Boolean visit(CopyToStatement stmtCopy, Void arg) throws
CompilationException {
+ public Boolean visit(CopyToCloudStoreStatement stmtCopy, Void arg) throws
CompilationException {
boolean changed = false;
Pair<Boolean, Expression> queryBody =
inlineUdfsAndViewsInExpr(stmtCopy.getBody());
diff --git
a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/FormatPrintVisitor.java
b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/FormatPrintVisitor.java
index 3091b30..609d687 100644
---
a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/FormatPrintVisitor.java
+++
b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/FormatPrintVisitor.java
@@ -71,7 +71,7 @@
import org.apache.asterix.lang.common.statement.CompactStatement;
import org.apache.asterix.lang.common.statement.ConnectFeedStatement;
import org.apache.asterix.lang.common.statement.CopyFromStatement;
-import org.apache.asterix.lang.common.statement.CopyToStatement;
+import org.apache.asterix.lang.common.statement.CopyToCloudStoreStatement;
import org.apache.asterix.lang.common.statement.CreateAdapterStatement;
import org.apache.asterix.lang.common.statement.CreateDatabaseStatement;
import org.apache.asterix.lang.common.statement.CreateDataverseStatement;
@@ -560,7 +560,7 @@
}
@Override
- public Void visit(CopyToStatement cto, Integer step) throws
CompilationException {
+ public Void visit(CopyToCloudStoreStatement cto, Integer step) throws
CompilationException {
out.println(skip(step) + "copy ");
if (cto.getQuery() != null) {
out.print("(");
diff --git
a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/GatherFunctionCallsVisitor.java
b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/GatherFunctionCallsVisitor.java
index febc9d8..de77b52 100644
---
a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/GatherFunctionCallsVisitor.java
+++
b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/GatherFunctionCallsVisitor.java
@@ -48,7 +48,7 @@
import org.apache.asterix.lang.common.expression.TypeReferenceExpression;
import org.apache.asterix.lang.common.expression.UnaryExpr;
import org.apache.asterix.lang.common.expression.VariableExpr;
-import org.apache.asterix.lang.common.statement.CopyToStatement;
+import org.apache.asterix.lang.common.statement.CopyToCloudStoreStatement;
import org.apache.asterix.lang.common.statement.InsertStatement;
import org.apache.asterix.lang.common.statement.Query;
import org.apache.asterix.lang.common.struct.Identifier;
@@ -250,7 +250,7 @@
}
@Override
- public Void visit(CopyToStatement stmtCopy, Void arg) throws
CompilationException {
+ public Void visit(CopyToCloudStoreStatement stmtCopy, Void arg) throws
CompilationException {
stmtCopy.getQuery().accept(this, arg);
acceptList(stmtCopy.getPathExpressions(), arg);
acceptList(stmtCopy.getPartitionExpressions(), arg);
diff --git
a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/base/AbstractQueryExpressionVisitor.java
b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/base/AbstractQueryExpressionVisitor.java
index 0118f4c..9d71b3a 100644
---
a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/base/AbstractQueryExpressionVisitor.java
+++
b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/base/AbstractQueryExpressionVisitor.java
@@ -30,7 +30,7 @@
import org.apache.asterix.lang.common.statement.CompactStatement;
import org.apache.asterix.lang.common.statement.ConnectFeedStatement;
import org.apache.asterix.lang.common.statement.CopyFromStatement;
-import org.apache.asterix.lang.common.statement.CopyToStatement;
+import org.apache.asterix.lang.common.statement.CopyToCloudStoreStatement;
import org.apache.asterix.lang.common.statement.CreateAdapterStatement;
import org.apache.asterix.lang.common.statement.CreateDatabaseStatement;
import org.apache.asterix.lang.common.statement.CreateDataverseStatement;
@@ -126,7 +126,7 @@
}
@Override
- public R visit(CopyToStatement stmtCopy, T arg) throws
CompilationException {
+ public R visit(CopyToCloudStoreStatement stmtCopy, T arg) throws
CompilationException {
return null;
}
diff --git
a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/base/ILangVisitor.java
b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/base/ILangVisitor.java
index 1ed9b3c..1e1813d 100644
---
a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/base/ILangVisitor.java
+++
b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/base/ILangVisitor.java
@@ -48,7 +48,7 @@
import org.apache.asterix.lang.common.statement.CompactStatement;
import org.apache.asterix.lang.common.statement.ConnectFeedStatement;
import org.apache.asterix.lang.common.statement.CopyFromStatement;
-import org.apache.asterix.lang.common.statement.CopyToStatement;
+import org.apache.asterix.lang.common.statement.CopyToCloudStoreStatement;
import org.apache.asterix.lang.common.statement.CreateAdapterStatement;
import org.apache.asterix.lang.common.statement.CreateDatabaseStatement;
import org.apache.asterix.lang.common.statement.CreateDataverseStatement;
@@ -107,7 +107,7 @@
R visit(CopyFromStatement stmtCopy, T arg) throws CompilationException;
- R visit(CopyToStatement cto, T arg) throws CompilationException;
+ R visit(CopyToCloudStoreStatement cto, T arg) throws CompilationException;
R visit(DropDatasetStatement del, T arg) throws CompilationException;
diff --git
a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/visitor/SqlppCopyToRewriteVisitor.java
b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/visitor/SqlppCopyToRewriteVisitor.java
index d15eb2d..720753a 100644
---
a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/visitor/SqlppCopyToRewriteVisitor.java
+++
b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/visitor/SqlppCopyToRewriteVisitor.java
@@ -24,6 +24,7 @@
import org.apache.asterix.common.metadata.DataverseName;
import org.apache.asterix.lang.common.expression.CallExpr;
import org.apache.asterix.lang.common.expression.VariableExpr;
+import org.apache.asterix.lang.common.statement.CopyToCloudStoreStatement;
import org.apache.asterix.lang.common.statement.CopyToStatement;
import org.apache.asterix.lang.common.statement.Query;
import org.apache.asterix.lang.common.util.FunctionUtil;
@@ -42,7 +43,8 @@
public static final SqlppCopyToRewriteVisitor INSTANCE = new
SqlppCopyToRewriteVisitor();
@Override
- public Void visit(CopyToStatement stmtCopy, MetadataProvider
metadataProvider) throws CompilationException {
+ public Void visit(CopyToCloudStoreStatement stmtCopy, MetadataProvider
metadataProvider)
+ throws CompilationException {
if (stmtCopy.getNamespace() == null) {
stmtCopy.setNamespace(metadataProvider.getDefaultNamespace());
}
diff --git
a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/CheckDatasetOnlyResolutionVisitor.java
b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/CheckDatasetOnlyResolutionVisitor.java
index ad4385c..150f550 100644
---
a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/CheckDatasetOnlyResolutionVisitor.java
+++
b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/CheckDatasetOnlyResolutionVisitor.java
@@ -39,7 +39,7 @@
import org.apache.asterix.lang.common.expression.RecordConstructor;
import org.apache.asterix.lang.common.expression.UnaryExpr;
import org.apache.asterix.lang.common.expression.VariableExpr;
-import org.apache.asterix.lang.common.statement.CopyToStatement;
+import org.apache.asterix.lang.common.statement.CopyToCloudStoreStatement;
import org.apache.asterix.lang.common.statement.FunctionDecl;
import org.apache.asterix.lang.common.statement.InsertStatement;
import org.apache.asterix.lang.common.statement.Query;
@@ -107,7 +107,7 @@
}
@Override
- public Boolean visit(CopyToStatement stmtCopy, VariableExpr arg) throws
CompilationException {
+ public Boolean visit(CopyToCloudStoreStatement stmtCopy, VariableExpr arg)
throws CompilationException {
return contains(stmtCopy.getQuery(), arg);
}
diff --git
a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/CheckSubqueryVisitor.java
b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/CheckSubqueryVisitor.java
index 1863956..8c0be19 100644
---
a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/CheckSubqueryVisitor.java
+++
b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/CheckSubqueryVisitor.java
@@ -44,7 +44,7 @@
import org.apache.asterix.lang.common.expression.RecordConstructor;
import org.apache.asterix.lang.common.expression.UnaryExpr;
import org.apache.asterix.lang.common.expression.VariableExpr;
-import org.apache.asterix.lang.common.statement.CopyToStatement;
+import org.apache.asterix.lang.common.statement.CopyToCloudStoreStatement;
import org.apache.asterix.lang.common.statement.FunctionDecl;
import org.apache.asterix.lang.common.statement.Query;
import org.apache.asterix.lang.common.struct.Identifier;
@@ -314,7 +314,7 @@
}
@Override
- public Boolean visit(CopyToStatement stmtCopy, ILangExpression arg) throws
CompilationException {
+ public Boolean visit(CopyToCloudStoreStatement stmtCopy, ILangExpression
arg) throws CompilationException {
return stmtCopy.getQuery().accept(this, arg) ||
visitExprList(stmtCopy.getPathExpressions(), arg)
|| visitExprList(stmtCopy.getPartitionExpressions(), arg)
|| visitExprList(stmtCopy.getOrderByList(), arg);
diff --git
a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/FreeVariableVisitor.java
b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/FreeVariableVisitor.java
index bfa5c91..80a30f7 100644
---
a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/FreeVariableVisitor.java
+++
b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/FreeVariableVisitor.java
@@ -47,7 +47,7 @@
import org.apache.asterix.lang.common.expression.RecordConstructor;
import org.apache.asterix.lang.common.expression.UnaryExpr;
import org.apache.asterix.lang.common.expression.VariableExpr;
-import org.apache.asterix.lang.common.statement.CopyToStatement;
+import org.apache.asterix.lang.common.statement.CopyToCloudStoreStatement;
import org.apache.asterix.lang.common.statement.FunctionDecl;
import org.apache.asterix.lang.common.statement.Query;
import org.apache.asterix.lang.common.struct.Identifier;
@@ -519,7 +519,8 @@
}
@Override
- public Void visit(CopyToStatement stmtCopy, Collection<VariableExpr>
freeVars) throws CompilationException {
+ public Void visit(CopyToCloudStoreStatement stmtCopy,
Collection<VariableExpr> freeVars)
+ throws CompilationException {
stmtCopy.getBody().accept(this, freeVars);
visit(stmtCopy.getPathExpressions(), freeVars);
visit(stmtCopy.getPartitionExpressions(), freeVars);
diff --git
a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/base/AbstractSqlppContainsExpressionVisitor.java
b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/base/AbstractSqlppContainsExpressionVisitor.java
index e4117f0..90de180 100644
---
a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/base/AbstractSqlppContainsExpressionVisitor.java
+++
b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/base/AbstractSqlppContainsExpressionVisitor.java
@@ -43,7 +43,7 @@
import org.apache.asterix.lang.common.expression.RecordConstructor;
import org.apache.asterix.lang.common.expression.UnaryExpr;
import org.apache.asterix.lang.common.expression.VariableExpr;
-import org.apache.asterix.lang.common.statement.CopyToStatement;
+import org.apache.asterix.lang.common.statement.CopyToCloudStoreStatement;
import org.apache.asterix.lang.common.struct.Identifier;
import org.apache.asterix.lang.common.struct.QuantifiedPair;
import org.apache.asterix.lang.sqlpp.clause.AbstractBinaryCorrelateClause;
@@ -309,7 +309,7 @@
}
@Override
- public Boolean visit(CopyToStatement stmtCopy, T arg) throws
CompilationException {
+ public Boolean visit(CopyToCloudStoreStatement stmtCopy, T arg) throws
CompilationException {
return stmtCopy.accept(this, arg) ||
visitExprList(stmtCopy.getPathExpressions(), arg)
|| visitExprList(stmtCopy.getPartitionExpressions(), arg)
|| visitExprList(stmtCopy.getOrderByList(), arg);
diff --git
a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/base/AbstractSqlppExpressionScopingVisitor.java
b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/base/AbstractSqlppExpressionScopingVisitor.java
index e3e2484..fc038ed 100644
---
a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/base/AbstractSqlppExpressionScopingVisitor.java
+++
b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/base/AbstractSqlppExpressionScopingVisitor.java
@@ -42,7 +42,7 @@
import org.apache.asterix.lang.common.expression.VariableExpr;
import org.apache.asterix.lang.common.parser.ScopeChecker;
import org.apache.asterix.lang.common.rewrites.LangRewritingContext;
-import org.apache.asterix.lang.common.statement.CopyToStatement;
+import org.apache.asterix.lang.common.statement.CopyToCloudStoreStatement;
import org.apache.asterix.lang.common.statement.FunctionDecl;
import org.apache.asterix.lang.common.statement.InsertStatement;
import org.apache.asterix.lang.common.statement.Query;
@@ -414,7 +414,7 @@
}
@Override
- public Expression visit(CopyToStatement stmtCopy, ILangExpression arg)
throws CompilationException {
+ public Expression visit(CopyToCloudStoreStatement stmtCopy,
ILangExpression arg) throws CompilationException {
stmtCopy.setBody(stmtCopy.getBody().accept(this, stmtCopy));
// Scope that only contains the source variable
diff --git
a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/base/AbstractSqlppSimpleExpressionVisitor.java
b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/base/AbstractSqlppSimpleExpressionVisitor.java
index f3d2675..77932be 100644
---
a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/base/AbstractSqlppSimpleExpressionVisitor.java
+++
b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/base/AbstractSqlppSimpleExpressionVisitor.java
@@ -45,7 +45,7 @@
import org.apache.asterix.lang.common.expression.RecordConstructor;
import org.apache.asterix.lang.common.expression.UnaryExpr;
import org.apache.asterix.lang.common.expression.VariableExpr;
-import org.apache.asterix.lang.common.statement.CopyToStatement;
+import org.apache.asterix.lang.common.statement.CopyToCloudStoreStatement;
import org.apache.asterix.lang.common.statement.FunctionDecl;
import org.apache.asterix.lang.common.statement.InsertStatement;
import org.apache.asterix.lang.common.statement.Query;
@@ -359,7 +359,7 @@
}
@Override
- public Expression visit(CopyToStatement stmtCopy, ILangExpression arg)
throws CompilationException {
+ public Expression visit(CopyToCloudStoreStatement stmtCopy,
ILangExpression arg) throws CompilationException {
stmtCopy.setBody(stmtCopy.getBody().accept(this, arg));
stmtCopy.setPathExpressions(visit(stmtCopy.getPathExpressions(), arg));
stmtCopy.setPartitionExpressions(visit(stmtCopy.getPartitionExpressions(),
arg));
diff --git a/asterixdb/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj
b/asterixdb/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj
index a81807c..159a71b 100644
--- a/asterixdb/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj
+++ b/asterixdb/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj
@@ -167,6 +167,7 @@
import org.apache.asterix.lang.common.statement.InternalDetailsDecl;
import org.apache.asterix.lang.common.statement.LoadStatement;
import org.apache.asterix.lang.common.statement.CopyFromStatement;
+import org.apache.asterix.lang.common.statement.CopyToCloudStoreStatement;
import org.apache.asterix.lang.common.statement.CopyToStatement;
import org.apache.asterix.lang.common.statement.NodeGroupDropStatement;
import org.apache.asterix.lang.common.statement.NodegroupDecl;
@@ -2868,12 +2869,13 @@
VariableExpr alias = null;
}
{
+ // TODO check if the LOOKAHEAD below is needed
<COPY>
( LOOKAHEAD(1) <INTO> { startToken = token; }
nameComponents = QualifiedName()
((<AS>)? (typeExpr =
DatasetTypeSpecification(RecordTypeDefinition.RecordKind.OPEN)))?
stmt = CopyFromStatement(startToken, nameComponents, typeExpr)
- | LOOKAHEAD(1) <LEFTPAREN> { startToken = token; } query = Query()
<RIGHTPAREN> (<AS>)? alias = Variable() stmt = CopyToStatement(startToken,
nameComponents, query, alias)
+ | LOOKAHEAD(1) <LEFTPAREN> { startToken = token; } query = Query()
<RIGHTPAREN> (<AS>)? alias = Variable() stmt = CopyToStatement(startToken,
nameComponents, query, alias)
| { startToken = token; } nameComponents = QualifiedName()
(<AS>)? (typeExpr =
DatasetTypeSpecification(RecordTypeDefinition.RecordKind.OPEN) | alias =
Variable())?
(stmt = CopyFromStatement(startToken, nameComponents, typeExpr) | stmt =
CopyToStatement(startToken, nameComponents, query, alias))
@@ -2942,7 +2944,7 @@
usedAlias = new
VariableExpr(SqlppVariableUtil.toInternalVariableIdentifier(datasetName));
}
- CopyToStatement stmt = new CopyToStatement(namespace, datasetName,
query, usedAlias, edd, pathExprs,
+ CopyToStatement stmt = new CopyToCloudStoreStatement(namespace,
datasetName, query, usedAlias, edd, pathExprs,
partitionExprs, partitionVarExprs, orderbyList,
orderbyModifierList, orderbyNullModifierList, getVarCounter());
return addSourceLocation(stmt, startToken);
}
--
To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/18146
To unsubscribe, or for help writing mail filters, visit
https://asterix-gerrit.ics.uci.edu/settings
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Change-Id: Id90a30c1e9f41ca82ef28f6edd0569a69002572c
Gerrit-Change-Number: 18146
Gerrit-PatchSet: 1
Gerrit-Owner: Hussain Towaileb <[email protected]>
Gerrit-MessageType: newchange
