This is an automated email from the ASF dual-hosted git repository.
wyk pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
The following commit(s) were added to refs/heads/master by this push:
new ac81f12836 [ASTERIXDB-3287][SQL++] Support parsing COPY TO statement
ac81f12836 is described below
commit ac81f12836f97040c4150ef22115d12894f02883
Author: Wail Alkowaileet <[email protected]>
AuthorDate: Thu Oct 26 09:41:39 2023 -0700
[ASTERIXDB-3287][SQL++] Support parsing COPY TO statement
- user model changes: yes
- storage format changes: no
- interface changes: no
Details:
- Support parsing COPY TO
- Change the syntax for COPY FROM to be more aligned
with the COPY TO
Change-Id: I5e3f42c9f3a39b1308c54d3bc365256510746a0c
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17877
Integration-Tests: Jenkins <[email protected]>
Reviewed-by: Ali Alsuliman <[email protected]>
Tested-by: Wail Alkowaileet <[email protected]>
---
.../asterix/translator/CompiledStatements.java | 2 +-
.../translator/LangExpressionToPlanTranslator.java | 4 +-
.../apache/asterix/api/common/APIFramework.java | 2 +-
.../asterix/app/translator/QueryTranslator.java | 27 +--
.../copy/copy-1/copy-1.2.update.sqlpp | 9 +-
.../copy/copy-2/copy-2.2.update.sqlpp | 16 +-
.../create-dataset-3.02.update.sqlpp | 9 +-
.../asterix/external/util/ExternalDataUtils.java | 13 ++
.../apache/asterix/lang/common/base/Statement.java | 23 ++-
.../{CopyStatement.java => CopyFromStatement.java} | 22 ++-
.../lang/common/statement/CopyToStatement.java | 188 +++++++++++++++++++++
.../lang/common/util/ConfigurationUtil.java | 14 +-
.../asterix/lang/common/util/ExpressionUtils.java | 2 -
.../common/visitor/AbstractInlineUdfsVisitor.java | 30 +++-
.../lang/common/visitor/FormatPrintVisitor.java | 59 ++++++-
.../common/visitor/GatherFunctionCallsVisitor.java | 16 ++
.../base/AbstractQueryExpressionVisitor.java | 10 +-
.../lang/common/visitor/base/ILangVisitor.java | 7 +-
.../sqlpp/rewrites/SqlppStatementRewriter.java | 3 +
.../visitor/SqlppCopyToRewriteVisitor.java | 89 ++++++++++
.../visitor/CheckDatasetOnlyResolutionVisitor.java | 6 +
.../lang/sqlpp/visitor/CheckSubqueryVisitor.java | 8 +
.../lang/sqlpp/visitor/FreeVariableVisitor.java | 26 +--
.../AbstractSqlppContainsExpressionVisitor.java | 8 +
.../AbstractSqlppExpressionScopingVisitor.java | 36 +++-
.../base/AbstractSqlppSimpleExpressionVisitor.java | 10 ++
.../asterix-lang-sqlpp/src/main/javacc/SQLPP.jj | 145 ++++++++++++++--
27 files changed, 693 insertions(+), 91 deletions(-)
diff --git
a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/CompiledStatements.java
b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/CompiledStatements.java
index 012df018db..3a1d9acd4d 100644
---
a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/CompiledStatements.java
+++
b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/CompiledStatements.java
@@ -351,7 +351,7 @@ public class CompiledStatements {
@Override
public Statement.Kind getKind() {
- return Statement.Kind.COPY;
+ return Statement.Kind.COPY_FROM;
}
@Override
diff --git
a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java
b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java
index d156b252da..e48214439b 100644
---
a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java
+++
b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java
@@ -224,7 +224,7 @@ abstract class LangExpressionToPlanTranslator
lds = new LoadableDataSource(dataset, itemType, metaItemType,
((CompiledLoadFromFileStatement) stmt).getAdapter(),
((CompiledLoadFromFileStatement)
stmt).getProperties());
- } else if (stmt.getKind() == Statement.Kind.COPY) {
+ } else if (stmt.getKind() == Statement.Kind.COPY_FROM) {
CompiledCopyFromFileStatement copyStmt =
(CompiledCopyFromFileStatement) stmt;
lds = new LoadableDataSource(dataset,
copyStmt.getItemType().getDatatype(), metaItemType,
copyStmt.getAdapter(), copyStmt.getProperties());
@@ -316,7 +316,7 @@ abstract class LangExpressionToPlanTranslator
leafOperator.getInputs().add(new MutableObject<>(insertOp));
leafOperator.setSourceLocation(sourceLoc);
return new ALogicalPlanImpl(new MutableObject<>(leafOperator));
- } else if (stmt.getKind() == Statement.Kind.COPY) {
+ } else if (stmt.getKind() == Statement.Kind.COPY_FROM) {
InsertDeleteUpsertOperator upsertOp = new
InsertDeleteUpsertOperator(targetDatasource, payloadRef,
varRefsForLoading, InsertDeleteUpsertOperator.Kind.UPSERT,
false);
upsertOp.setAdditionalFilteringExpressions(additionalFilteringExpressions);
diff --git
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java
index 9d9432744e..24db0c04ed 100644
---
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java
+++
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java
@@ -210,7 +210,7 @@ public class APIFramework {
// establish facts
final boolean isQuery = query != null;
final boolean isLoad = statement != null && statement.getKind() ==
Statement.Kind.LOAD;
- final boolean isCopy = statement != null && statement.getKind() ==
Statement.Kind.COPY;
+ final boolean isCopy = statement != null && statement.getKind() ==
Statement.Kind.COPY_FROM;
final SourceLocation sourceLoc =
query != null ? query.getSourceLocation() : statement != null
? statement.getSourceLocation() : null;
final boolean isExplainOnly = isQuery && query.isExplain();
diff --git
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
index 75ff63b318..6dd6cc9f63 100644
---
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
+++
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
@@ -120,7 +120,7 @@ import
org.apache.asterix.lang.common.statement.AnalyzeDropStatement;
import org.apache.asterix.lang.common.statement.AnalyzeStatement;
import org.apache.asterix.lang.common.statement.CompactStatement;
import org.apache.asterix.lang.common.statement.ConnectFeedStatement;
-import org.apache.asterix.lang.common.statement.CopyStatement;
+import org.apache.asterix.lang.common.statement.CopyFromStatement;
import org.apache.asterix.lang.common.statement.CreateAdapterStatement;
import org.apache.asterix.lang.common.statement.CreateDatabaseStatement;
import org.apache.asterix.lang.common.statement.CreateDataverseStatement;
@@ -459,11 +459,11 @@ public class QueryTranslator extends
AbstractLangTranslator implements IStatemen
}
handleLoadStatement(metadataProvider, stmt, hcc);
break;
- case COPY:
+ case COPY_FROM:
if (stats.getProfileType() == Stats.ProfileType.FULL) {
this.jobFlags.add(JobFlag.PROFILE_RUNTIME);
}
- handleCopyStatement(metadataProvider, stmt, hcc);
+ handleCopyFromStatement(metadataProvider, stmt, hcc);
break;
case INSERT:
case UPSERT:
@@ -3842,15 +3842,20 @@ public class QueryTranslator extends
AbstractLangTranslator implements IStatemen
}
}
- protected Map<String, String>
createExternalDataPropertiesForCopyStmt(String databaseName,
- DataverseName dataverseName, CopyStatement copyStatement, Datatype
itemType,
+ protected Map<String, String>
createExternalDataPropertiesForCopyFromStmt(String databaseName,
+ DataverseName dataverseName, CopyFromStatement copyFromStatement,
Datatype itemType,
MetadataTransactionContext mdTxnCtx) throws AlgebricksException {
- return copyStatement.getExternalDetails().getProperties();
+ ExternalDetailsDecl edd = copyFromStatement.getExternalDetails();
+ Map<String, String> properties =
copyFromStatement.getExternalDetails().getProperties();
+ String path = copyFromStatement.getPath();
+ String pathKey = ExternalDataUtils.getPathKey(edd.getAdapter());
+ properties.put(pathKey, path);
+ return properties;
}
- protected void handleCopyStatement(MetadataProvider metadataProvider,
Statement stmt, IHyracksClientConnection hcc)
- throws Exception {
- CopyStatement copyStmt = (CopyStatement) stmt;
+ protected void handleCopyFromStatement(MetadataProvider metadataProvider,
Statement stmt,
+ IHyracksClientConnection hcc) throws Exception {
+ CopyFromStatement copyStmt = (CopyFromStatement) stmt;
String datasetName = copyStmt.getDatasetName();
metadataProvider.validateDatabaseObjectName(copyStmt.getNamespace(),
datasetName, copyStmt.getSourceLocation());
Namespace stmtActiveNamespace =
getActiveNamespace(copyStmt.getNamespace());
@@ -3887,8 +3892,8 @@ public class QueryTranslator extends
AbstractLangTranslator implements IStatemen
new Datatype(itemTypeDatabaseName,
itemTypeDataverseName, itemTypeName, itemTypeEntity, true);
}
ExternalDetailsDecl externalDetails =
copyStmt.getExternalDetails();
- Map<String, String> properties =
- createExternalDataPropertiesForCopyStmt(databaseName,
dataverseName, copyStmt, itemType, mdTxnCtx);
+ Map<String, String> properties =
createExternalDataPropertiesForCopyFromStmt(databaseName, dataverseName,
+ copyStmt, itemType, mdTxnCtx);
ExternalDataUtils.normalize(properties);
ExternalDataUtils.validate(properties);
validateExternalDatasetProperties(externalDetails, properties,
copyStmt.getSourceLocation(), mdTxnCtx,
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy/copy-1/copy-1.2.update.sqlpp
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy/copy-1/copy-1.2.update.sqlpp
index 002d43d950..86b749f0d8 100644
---
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy/copy-1/copy-1.2.update.sqlpp
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy/copy-1/copy-1.2.update.sqlpp
@@ -33,9 +33,12 @@ upsert into Customers([
}
]);
-copy Customers
-using localfs
-(("path"="asterix_nc1://data/nontagged/customerData.json"),("format"="adm"));
+COPY Customers
+FROM localfs
+PATH ("asterix_nc1://data/nontagged/customerData.json")
+WITH {
+ "format": "adm"
+};
upsert into Customers([
{
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy/copy-2/copy-2.2.update.sqlpp
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy/copy-2/copy-2.2.update.sqlpp
index baae560c27..60ad511b27 100644
---
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy/copy-2/copy-2.2.update.sqlpp
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy/copy-2/copy-2.2.update.sqlpp
@@ -28,13 +28,15 @@ upsert into test1([
}
]);
-copy test1 USING S3 (
-("region"="us-west-2"),
-("serviceEndpoint"="http://127.0.0.1:8001";),
-("container"="playground"),
-("definition"="data_dir"),
-("format"="json")
-);
+COPY test1
+FROM S3
+PATH ("data_dir")
+WITH {
+ "format": "json",
+ "container": "playground",
+ "serviceEndpoint": "http://127.0.0.1:8001";,
+ "region": "us-west-2"
+};
upsert into test1([
{
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/ddl/create-dataset-3/create-dataset-3.02.update.sqlpp
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/ddl/create-dataset-3/create-dataset-3.02.update.sqlpp
index ab308047e9..30e4526e07 100644
---
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/ddl/create-dataset-3/create-dataset-3.02.update.sqlpp
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/ddl/create-dataset-3/create-dataset-3.02.update.sqlpp
@@ -18,6 +18,9 @@
*/
use test;
-copy orders
-using localfs
-(("path"="asterix_nc1://data/nontagged/orderData.json"),("format"="adm"));
+COPY orders
+FROM localfs
+PATH("asterix_nc1://data/nontagged/orderData.json")
+WITH {
+ "format": "adm"
+};
diff --git
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
index 62dc4669dd..439e8529d8 100644
---
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
+++
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
@@ -1057,4 +1057,17 @@ public class ExternalDataUtils {
+ ExternalDataConstants.KEY_EMBED_FILTER_VALUES + "' is
enabled");
}
}
+
+ public static String getPathKey(String adapter) {
+ String normalizedAdapter = adapter.toUpperCase();
+ switch (normalizedAdapter) {
+ case ExternalDataConstants.KEY_ADAPTER_NAME_AWS_S3:
+ case ExternalDataConstants.KEY_ADAPTER_NAME_AZURE_BLOB:
+ case ExternalDataConstants.KEY_ADAPTER_NAME_AZURE_DATA_LAKE:
+ case ExternalDataConstants.KEY_ADAPTER_NAME_GCS:
+ return ExternalDataConstants.DEFINITION_FIELD_NAME;
+ default:
+ return ExternalDataConstants.KEY_PATH;
+ }
+ }
}
diff --git
a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/base/Statement.java
b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/base/Statement.java
index 404360cc50..42c7eb6f7e 100644
---
a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/base/Statement.java
+++
b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/base/Statement.java
@@ -27,21 +27,29 @@ public interface Statement extends ILangExpression {
Kind getKind();
/**
- * get a byte representing the statement category.
- * Each category describes the type of modifications this statement does.
+ * get a byte representing the statement category.
+ * Each category describes the type of modifications this statement does.
*
* @return kind byte
*/
byte getCategory();
class Category {
- /** no modifications */
+ /**
+ * no modifications
+ */
public static final byte QUERY = 0x01;
- /** modify data */
+ /**
+ * modify data
+ */
public static final byte UPDATE = 0x02;
- /** modify metadata */
+ /**
+ * modify metadata
+ */
public static final byte DDL = 0x04;
- /** modify anything */
+ /**
+ * modify anything
+ */
public static final byte PROCEDURE = 0x08;
private Category() {
@@ -113,6 +121,7 @@ public interface Statement extends ILangExpression {
COMPACT,
SUBSCRIBE_FEED,
EXTENSION,
- COPY
+ COPY_FROM,
+ COPY_TO,
}
}
diff --git
a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/CopyStatement.java
b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/CopyFromStatement.java
similarity index 76%
rename from
asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/CopyStatement.java
rename to
asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/CopyFromStatement.java
index a07371d1a2..51c9803b83 100644
---
a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/CopyStatement.java
+++
b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/CopyFromStatement.java
@@ -28,18 +28,30 @@ import org.apache.asterix.lang.common.util.ExpressionUtils;
import org.apache.asterix.lang.common.visitor.base.ILangVisitor;
import org.apache.asterix.object.base.AdmObjectNode;
-public class CopyStatement extends AbstractStatement {
+public class CopyFromStatement extends AbstractStatement {
private final Namespace namespace;
private final String datasetName;
+ private final String path;
private final TypeExpression typeExpr;
private final ExternalDetailsDecl externalDetails;
private final AdmObjectNode withObjectNode;
- public CopyStatement(Namespace namespace, String datasetName,
TypeExpression typeExpr,
+ public CopyFromStatement(Namespace namespace, String datasetName,
TypeExpression typeExpr,
+ ExternalDetailsDecl externalDetails, RecordConstructor withRecord)
throws CompilationException {
+ this(namespace, datasetName, null, typeExpr, externalDetails,
withRecord);
+ }
+
+ public CopyFromStatement(Namespace namespace, String datasetName, String
path, TypeExpression typeExpr,
+ ExternalDetailsDecl externalDetails) throws CompilationException {
+ this(namespace, datasetName, path, typeExpr, externalDetails, null);
+ }
+
+ private CopyFromStatement(Namespace namespace, String datasetName, String
path, TypeExpression typeExpr,
ExternalDetailsDecl externalDetails, RecordConstructor withRecord)
throws CompilationException {
this.namespace = namespace;
this.datasetName = datasetName;
+ this.path = path;
this.typeExpr = typeExpr;
this.externalDetails = externalDetails;
this.withObjectNode = withRecord == null ? new AdmObjectNode() :
ExpressionUtils.toNode(withRecord);
@@ -55,7 +67,7 @@ public class CopyStatement extends AbstractStatement {
@Override
public Kind getKind() {
- return Kind.COPY;
+ return Kind.COPY_FROM;
}
public String getDatasetName() {
@@ -75,6 +87,10 @@ public class CopyStatement extends AbstractStatement {
return withObjectNode;
}
+ public String getPath() {
+ return path;
+ }
+
@Override
public byte getCategory() {
return Category.UPDATE;
diff --git
a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/CopyToStatement.java
b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/CopyToStatement.java
new file mode 100644
index 0000000000..52e357014d
--- /dev/null
+++
b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/CopyToStatement.java
@@ -0,0 +1,188 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.lang.common.statement;
+
+import static org.apache.asterix.lang.common.base.Statement.Category.QUERY;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.asterix.common.exceptions.CompilationException;
+import org.apache.asterix.common.metadata.Namespace;
+import org.apache.asterix.lang.common.base.AbstractStatement;
+import org.apache.asterix.lang.common.base.Expression;
+import org.apache.asterix.lang.common.base.IReturningStatement;
+import org.apache.asterix.lang.common.clause.OrderbyClause;
+import org.apache.asterix.lang.common.expression.VariableExpr;
+import org.apache.asterix.lang.common.visitor.base.ILangVisitor;
+
+public class CopyToStatement extends AbstractStatement implements
IReturningStatement {
+ private final Namespace namespace;
+ private final String datasetName;
+ private final VariableExpr sourceVariable;
+ private final ExternalDetailsDecl externalDetailsDecl;
+ private final Map<Integer, VariableExpr> partitionsVariables;
+ private final List<OrderbyClause.OrderModifier> orderbyModifiers;
+ private final List<OrderbyClause.NullOrderModifier>
orderbyNullModifierList;
+
+ private Query query;
+ private Expression pathExpression;
+
+ private List<Expression> partitionExpressions;
+ private List<Expression> orderbyList;
+ private int varCounter;
+
+ public CopyToStatement(Namespace namespace, String datasetName, Query
query, VariableExpr sourceVariable,
+ ExternalDetailsDecl externalDetailsDecl, Expression
pathExpression, List<Expression> partitionExpressions,
+ Map<Integer, VariableExpr> partitionsVariables, List<Expression>
orderbyList,
+ List<OrderbyClause.OrderModifier> orderbyModifiers,
+ List<OrderbyClause.NullOrderModifier> orderbyNullModifierList, int
varCounter) {
+ this.namespace = namespace;
+ this.datasetName = datasetName;
+ this.query = query;
+ this.sourceVariable = sourceVariable;
+ this.externalDetailsDecl = externalDetailsDecl;
+ this.pathExpression = pathExpression;
+ this.partitionExpressions = partitionExpressions;
+ this.partitionsVariables = partitionsVariables;
+ this.orderbyList = orderbyList;
+ this.orderbyModifiers = orderbyModifiers;
+ this.orderbyNullModifierList = orderbyNullModifierList;
+ this.varCounter = varCounter;
+ }
+
+ @Override
+ public <R, T> R accept(ILangVisitor<R, T> visitor, T arg) throws
CompilationException {
+ return visitor.visit(this, arg);
+ }
+
+ @Override
+ public Kind getKind() {
+ return Kind.COPY_TO;
+ }
+
+ @Override
+ public byte getCategory() {
+ return QUERY;
+ }
+
+ public Namespace getNamespace() {
+ return namespace;
+ }
+
+ public String getDatasetName() {
+ return datasetName;
+ }
+
+ public void setQuery(Query query) {
+ this.query = query;
+ }
+
+ public Query getQuery() {
+ return query;
+ }
+
+ public VariableExpr getSourceVariable() {
+ return sourceVariable;
+ }
+
+ public ExternalDetailsDecl getExternalDetailsDecl() {
+ return externalDetailsDecl;
+ }
+
+ public Expression getPathExpression() {
+ return pathExpression;
+ }
+
+ public void setPathExpression(Expression pathExpression) {
+ this.pathExpression = pathExpression;
+ }
+
+ public List<Expression> getPartitionExpressions() {
+ return partitionExpressions;
+ }
+
+ public void setPartitionExpressions(List<Expression> partitionExpressions)
{
+ this.partitionExpressions = partitionExpressions;
+ }
+
+ public Map<Integer, VariableExpr> getPartitionsVariables() {
+ return partitionsVariables;
+ }
+
+ public List<Expression> getOrderbyList() {
+ return orderbyList;
+ }
+
+ public void setOrderbyList(List<Expression> orderbyList) {
+ this.orderbyList = orderbyList;
+ }
+
+ public List<OrderbyClause.OrderModifier> getOrderbyModifiers() {
+ return orderbyModifiers;
+ }
+
+ public List<OrderbyClause.NullOrderModifier> getOrderbyNullModifierList() {
+ return orderbyNullModifierList;
+ }
+
+ public boolean hasOverClause() {
+ return !partitionExpressions.isEmpty();
+ }
+
+ public boolean hasOrderClause() {
+ return !orderbyList.isEmpty();
+ }
+
+ @Override
+ public int getVarCounter() {
+ return varCounter;
+ }
+
+ @Override
+ public void setVarCounter(int varCounter) {
+ this.varCounter = varCounter;
+ }
+
+ @Override
+ public boolean isTopLevel() {
+ return true;
+ }
+
+ @Override
+ public List<Expression> getDirectlyEnclosedExpressions() {
+ List<Expression> topLevelExpressions = new ArrayList<>();
+ topLevelExpressions.add(query.getBody());
+ topLevelExpressions.add(pathExpression);
+ topLevelExpressions.addAll(partitionExpressions);
+ topLevelExpressions.addAll(orderbyList);
+ return topLevelExpressions;
+ }
+
+ @Override
+ public Expression getBody() {
+ return query.getBody();
+ }
+
+ @Override
+ public void setBody(Expression expr) {
+ query.setBody(expr);
+ }
+}
diff --git
a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/ConfigurationUtil.java
b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/ConfigurationUtil.java
index 4bb799b3e1..422ffe3043 100644
---
a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/ConfigurationUtil.java
+++
b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/ConfigurationUtil.java
@@ -24,6 +24,7 @@ import java.util.Map.Entry;
import org.apache.asterix.common.exceptions.CompilationException;
import org.apache.asterix.common.exceptions.ErrorCode;
+import org.apache.asterix.lang.common.expression.RecordConstructor;
import org.apache.asterix.object.base.AdmObjectNode;
import org.apache.asterix.object.base.AdmStringNode;
import org.apache.asterix.object.base.IAdmNode;
@@ -33,12 +34,15 @@ public class ConfigurationUtil {
private ConfigurationUtil() {
}
+ public static Map<String, String> toProperties(RecordConstructor
recordConstructor) throws CompilationException {
+ return toProperties(ExpressionUtils.toNode(recordConstructor));
+ }
+
/**
* Convert the parameters object to a Map<String,String>
* This method should go away once we store the with object as it is in
storage
*
- * @param parameters
- * the parameters passed for the merge policy in the with clause
+ * @param parameters the parameters passed for the merge policy in the
with clause
* @return the parameters as a map
*/
public static Map<String, String> toProperties(AdmObjectNode parameters)
throws CompilationException {
@@ -53,10 +57,8 @@ public class ConfigurationUtil {
/**
* Get string value of {@link IAdmNode}
*
- * @param value
- * IAdmNode value should be of type integer or string
- * @return
- * string value of <code>value</code>
+ * @param value IAdmNode value should be of type integer or string
+ * @return string value of <code>value</code>
* @throws CompilationException
*/
public static String getStringValue(IAdmNode value) throws
CompilationException {
diff --git
a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/ExpressionUtils.java
b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/ExpressionUtils.java
index 925fada4de..3e91e22b45 100644
---
a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/ExpressionUtils.java
+++
b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/ExpressionUtils.java
@@ -18,8 +18,6 @@
*/
package org.apache.asterix.lang.common.util;
-import static org.apache.asterix.lang.common.base.Literal.Type.DOUBLE;
-
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
diff --git
a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/AbstractInlineUdfsVisitor.java
b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/AbstractInlineUdfsVisitor.java
index 077748d9cc..d91873cf52 100644
---
a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/AbstractInlineUdfsVisitor.java
+++
b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/AbstractInlineUdfsVisitor.java
@@ -52,6 +52,7 @@ import org.apache.asterix.lang.common.expression.UnaryExpr;
import org.apache.asterix.lang.common.expression.VariableExpr;
import org.apache.asterix.lang.common.rewrites.LangRewritingContext;
import org.apache.asterix.lang.common.rewrites.VariableSubstitutionEnvironment;
+import org.apache.asterix.lang.common.statement.CopyToStatement;
import org.apache.asterix.lang.common.statement.FunctionDecl;
import org.apache.asterix.lang.common.statement.InsertStatement;
import org.apache.asterix.lang.common.statement.Query;
@@ -86,10 +87,8 @@ public abstract class AbstractInlineUdfsVisitor extends
AbstractQueryExpressionV
}
/**
- * @param letClauses
- * , a list of let-binding clauses
- * @param returnExpr
- * , a return expression
+ * @param letClauses , a list of let-binding clauses
+ * @param returnExpr , a return expression
* @return a query expression which is upto a specific langauge, e.g.,
FLWOGR in AQL and expression query in SQL++.
*/
protected abstract Expression generateQueryExpression(List<LetClause>
letClauses, Expression returnExpr)
@@ -283,6 +282,29 @@ public abstract class AbstractInlineUdfsVisitor extends
AbstractQueryExpressionV
return changed || rewrittenBodyExpression.first;
}
+ @Override
+ public Boolean visit(CopyToStatement stmtCopy, Void arg) throws
CompilationException {
+ boolean changed = false;
+
+ Pair<Boolean, Expression> queryBody =
inlineUdfsAndViewsInExpr(stmtCopy.getBody());
+ changed |= queryBody.first;
+ stmtCopy.setBody(queryBody.second);
+
+ Pair<Boolean, Expression> path =
inlineUdfsAndViewsInExpr(stmtCopy.getPathExpression());
+ changed |= path.first;
+ stmtCopy.setPathExpression(path.second);
+
+ Pair<Boolean, List<Expression>> part =
inlineUdfsInExprList(stmtCopy.getPartitionExpressions());
+ changed |= part.first;
+ stmtCopy.setPartitionExpressions(part.second);
+
+ Pair<Boolean, List<Expression>> order =
inlineUdfsInExprList(stmtCopy.getOrderbyList());
+ changed |= order.first;
+ stmtCopy.setOrderbyList(order.second);
+
+ return changed;
+ }
+
protected Pair<Boolean, Expression> inlineUdfsAndViewsInExpr(Expression
expr) throws CompilationException {
if (expr.getKind() != Kind.CALL_EXPRESSION) {
boolean r = expr.accept(this, null);
diff --git
a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/FormatPrintVisitor.java
b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/FormatPrintVisitor.java
index 8edff2872d..7ac996717d 100644
---
a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/FormatPrintVisitor.java
+++
b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/FormatPrintVisitor.java
@@ -70,7 +70,8 @@ import
org.apache.asterix.lang.common.statement.AnalyzeDropStatement;
import org.apache.asterix.lang.common.statement.AnalyzeStatement;
import org.apache.asterix.lang.common.statement.CompactStatement;
import org.apache.asterix.lang.common.statement.ConnectFeedStatement;
-import org.apache.asterix.lang.common.statement.CopyStatement;
+import org.apache.asterix.lang.common.statement.CopyFromStatement;
+import org.apache.asterix.lang.common.statement.CopyToStatement;
import org.apache.asterix.lang.common.statement.CreateAdapterStatement;
import org.apache.asterix.lang.common.statement.CreateDatabaseStatement;
import org.apache.asterix.lang.common.statement.CreateDataverseStatement;
@@ -549,15 +550,65 @@ public abstract class FormatPrintVisitor implements
ILangVisitor<Void, Integer>
}
@Override
- public Void visit(CopyStatement stmtCopy, Integer step) throws
CompilationException {
+ public Void visit(CopyFromStatement stmtCopy, Integer step) throws
CompilationException {
out.print(skip(step) + "copy " + datasetSymbol
- + generateFullName(stmtCopy.getDataverseName(),
stmtCopy.getDatasetName()) + " using "
- +
revertStringToQuoted(stmtCopy.getExternalDetails().getAdapter()) + " ");
+ + generateFullName(stmtCopy.getDataverseName(),
stmtCopy.getDatasetName()) + " from "
+ +
revertStringToQuoted(stmtCopy.getExternalDetails().getAdapter()) + " with ");
printConfiguration(stmtCopy.getExternalDetails().getProperties());
out.println();
return null;
}
+ @Override
+ public Void visit(CopyToStatement cto, Integer step) throws
CompilationException {
+ out.println(skip(step) + "copy ");
+ if (cto.getQuery() != null) {
+ out.print("(");
+ cto.getQuery().accept(this, step + 2);
+ out.print(")");
+ } else {
+ out.print(datasetSymbol +
generateFullName(cto.getNamespace().getDataverseName(), cto.getDatasetName()));
+ }
+ out.print(" as ");
+ cto.getSourceVariable().accept(this, step);
+ out.println();
+
+ out.print("path (");
+ cto.getPathExpression().accept(this, step + 1);
+ out.print(")");
+
+ out.println();
+
+ if (cto.hasOverClause()) {
+ out.print("over (");
+ List<Expression> partitionExprs = cto.getPartitionExpressions();
+ Map<Integer, VariableExpr> partitionVars =
cto.getPartitionsVariables();
+ out.print(skip(step + 1) + "partition ");
+ for (int i = 0; i < partitionExprs.size(); i++) {
+ if (i > 0) {
+ out.print(COMMA + " ");
+ }
+ partitionExprs.get(i).accept(this, step + 2);
+ VariableExpr partVar = partitionVars.get(i);
+ if (partVar != null) {
+ out.print(" as ");
+ partVar.accept(this, step);
+ }
+ }
+ out.println();
+ if (cto.hasOrderClause()) {
+ out.print(skip(step + 1) + "order ");
+ printDelimitedObyExpressions(cto.getOrderbyList(),
cto.getOrderbyModifiers(),
+ cto.getOrderbyNullModifierList(), step + 1);
+ out.println();
+ }
+ }
+
+ out.println("with ");
+ printConfiguration(cto.getExternalDetailsDecl().getProperties());
+ return null;
+ }
+
@Override
public Void visit(DropDatasetStatement del, Integer step) throws
CompilationException {
out.println(
diff --git
a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/GatherFunctionCallsVisitor.java
b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/GatherFunctionCallsVisitor.java
index 6dcfb837c7..4ea7789df9 100644
---
a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/GatherFunctionCallsVisitor.java
+++
b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/GatherFunctionCallsVisitor.java
@@ -48,6 +48,7 @@ import
org.apache.asterix.lang.common.expression.RecordConstructor;
import org.apache.asterix.lang.common.expression.TypeReferenceExpression;
import org.apache.asterix.lang.common.expression.UnaryExpr;
import org.apache.asterix.lang.common.expression.VariableExpr;
+import org.apache.asterix.lang.common.statement.CopyToStatement;
import org.apache.asterix.lang.common.statement.InsertStatement;
import org.apache.asterix.lang.common.statement.Query;
import org.apache.asterix.lang.common.struct.Identifier;
@@ -247,4 +248,19 @@ public abstract class GatherFunctionCallsVisitor extends
AbstractQueryExpression
}
return null;
}
+
+ @Override
+ public Void visit(CopyToStatement stmtCopy, Void arg) throws
CompilationException {
+ stmtCopy.getQuery().accept(this, arg);
+ stmtCopy.getPathExpression().accept(this, arg);
+ acceptList(stmtCopy.getPartitionExpressions(), arg);
+ acceptList(stmtCopy.getOrderbyList(), arg);
+ return null;
+ }
+
+ private void acceptList(List<Expression> expressions, Void arg) throws
CompilationException {
+ for (Expression expression : expressions) {
+ expression.accept(this, arg);
+ }
+ }
}
diff --git
a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/base/AbstractQueryExpressionVisitor.java
b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/base/AbstractQueryExpressionVisitor.java
index 6c0eea9604..0118f4cd72 100644
---
a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/base/AbstractQueryExpressionVisitor.java
+++
b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/base/AbstractQueryExpressionVisitor.java
@@ -29,7 +29,8 @@ import
org.apache.asterix.lang.common.statement.AnalyzeDropStatement;
import org.apache.asterix.lang.common.statement.AnalyzeStatement;
import org.apache.asterix.lang.common.statement.CompactStatement;
import org.apache.asterix.lang.common.statement.ConnectFeedStatement;
-import org.apache.asterix.lang.common.statement.CopyStatement;
+import org.apache.asterix.lang.common.statement.CopyFromStatement;
+import org.apache.asterix.lang.common.statement.CopyToStatement;
import org.apache.asterix.lang.common.statement.CreateAdapterStatement;
import org.apache.asterix.lang.common.statement.CreateDatabaseStatement;
import org.apache.asterix.lang.common.statement.CreateDataverseStatement;
@@ -120,7 +121,12 @@ public abstract class AbstractQueryExpressionVisitor<R, T>
implements ILangVisit
}
@Override
- public R visit(CopyStatement stmtCopy, T arg) throws CompilationException {
+ public R visit(CopyFromStatement stmtCopy, T arg) throws
CompilationException {
+ return null;
+ }
+
+ @Override
+ public R visit(CopyToStatement stmtCopy, T arg) throws
CompilationException {
return null;
}
diff --git
a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/base/ILangVisitor.java
b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/base/ILangVisitor.java
index 4f70cb00d4..1ed9b3cab0 100644
---
a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/base/ILangVisitor.java
+++
b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/base/ILangVisitor.java
@@ -47,7 +47,8 @@ import
org.apache.asterix.lang.common.statement.AnalyzeDropStatement;
import org.apache.asterix.lang.common.statement.AnalyzeStatement;
import org.apache.asterix.lang.common.statement.CompactStatement;
import org.apache.asterix.lang.common.statement.ConnectFeedStatement;
-import org.apache.asterix.lang.common.statement.CopyStatement;
+import org.apache.asterix.lang.common.statement.CopyFromStatement;
+import org.apache.asterix.lang.common.statement.CopyToStatement;
import org.apache.asterix.lang.common.statement.CreateAdapterStatement;
import org.apache.asterix.lang.common.statement.CreateDatabaseStatement;
import org.apache.asterix.lang.common.statement.CreateDataverseStatement;
@@ -104,7 +105,9 @@ public interface ILangVisitor<R, T> {
R visit(LoadStatement stmtLoad, T arg) throws CompilationException;
- R visit(CopyStatement stmtCopy, T arg) throws CompilationException;
+ R visit(CopyFromStatement stmtCopy, T arg) throws CompilationException;
+
+ R visit(CopyToStatement cto, T arg) throws CompilationException;
R visit(DropDatasetStatement del, T arg) throws CompilationException;
diff --git
a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/SqlppStatementRewriter.java
b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/SqlppStatementRewriter.java
index f9b0e7a63b..30b20bbcd5 100644
---
a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/SqlppStatementRewriter.java
+++
b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/SqlppStatementRewriter.java
@@ -21,6 +21,7 @@ package org.apache.asterix.lang.sqlpp.rewrites;
import org.apache.asterix.common.exceptions.CompilationException;
import org.apache.asterix.lang.common.base.IStatementRewriter;
import org.apache.asterix.lang.common.base.Statement;
+import
org.apache.asterix.lang.sqlpp.rewrites.visitor.SqlppCopyToRewriteVisitor;
import org.apache.asterix.lang.sqlpp.visitor.SqlppDeleteRewriteVisitor;
import org.apache.asterix.lang.sqlpp.visitor.SqlppSynonymRewriteVisitor;
import org.apache.asterix.metadata.declared.MetadataProvider;
@@ -34,6 +35,7 @@ class SqlppStatementRewriter implements IStatementRewriter {
case INSERT:
case UPSERT:
case DELETE:
+ case COPY_TO:
return true;
default:
return false;
@@ -45,6 +47,7 @@ class SqlppStatementRewriter implements IStatementRewriter {
if (stmt != null) {
stmt.accept(SqlppSynonymRewriteVisitor.INSTANCE, metadataProvider);
stmt.accept(SqlppDeleteRewriteVisitor.INSTANCE, metadataProvider);
+ stmt.accept(SqlppCopyToRewriteVisitor.INSTANCE, metadataProvider);
}
}
}
diff --git
a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/visitor/SqlppCopyToRewriteVisitor.java
b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/visitor/SqlppCopyToRewriteVisitor.java
new file mode 100644
index 0000000000..68a2023315
--- /dev/null
+++
b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/visitor/SqlppCopyToRewriteVisitor.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.lang.sqlpp.rewrites.visitor;
+
+import java.util.Collections;
+
+import org.apache.asterix.common.exceptions.CompilationException;
+import org.apache.asterix.common.metadata.DataverseName;
+import org.apache.asterix.lang.common.expression.CallExpr;
+import org.apache.asterix.lang.common.expression.VariableExpr;
+import org.apache.asterix.lang.common.statement.CopyToStatement;
+import org.apache.asterix.lang.common.statement.Query;
+import org.apache.asterix.lang.common.util.FunctionUtil;
+import org.apache.asterix.lang.sqlpp.clause.FromClause;
+import org.apache.asterix.lang.sqlpp.clause.FromTerm;
+import org.apache.asterix.lang.sqlpp.clause.SelectBlock;
+import org.apache.asterix.lang.sqlpp.clause.SelectClause;
+import org.apache.asterix.lang.sqlpp.clause.SelectElement;
+import org.apache.asterix.lang.sqlpp.clause.SelectSetOperation;
+import org.apache.asterix.lang.sqlpp.expression.SelectExpression;
+import org.apache.asterix.lang.sqlpp.struct.SetOperationInput;
+import org.apache.asterix.lang.sqlpp.visitor.base.AbstractSqlppAstVisitor;
+import org.apache.asterix.metadata.declared.MetadataProvider;
+
+public class SqlppCopyToRewriteVisitor extends AbstractSqlppAstVisitor<Void,
MetadataProvider> {
+ public static final SqlppCopyToRewriteVisitor INSTANCE = new
SqlppCopyToRewriteVisitor();
+
+ @Override
+ public Void visit(CopyToStatement stmtCopy, MetadataProvider
metadataProvider) throws CompilationException {
+ if (stmtCopy.getQuery() == null) {
+ setQuery(stmtCopy);
+ }
+ return null;
+ }
+
+ private void setQuery(CopyToStatement stmtCopy) {
+ DataverseName dataverseName =
stmtCopy.getNamespace().getDataverseName();
+ String datasetName = stmtCopy.getDatasetName();
+ CallExpr callExpression =
+
FunctionUtil.makeDatasetCallExpr(stmtCopy.getNamespace().getDatabaseName(),
dataverseName, datasetName);
+ callExpression.setSourceLocation(stmtCopy.getSourceLocation());
+
+ // From clause.
+ VariableExpr var = stmtCopy.getSourceVariable();
+ FromTerm fromTerm = new FromTerm(callExpression, var, null, null);
+ fromTerm.setSourceLocation(var.getSourceLocation());
+ FromClause fromClause = new
FromClause(Collections.singletonList(fromTerm));
+ fromClause.setSourceLocation(var.getSourceLocation());
+
+ // Select clause.
+ VariableExpr returnExpr = new VariableExpr(var.getVar());
+ returnExpr.setIsNewVar(false);
+ returnExpr.setSourceLocation(var.getSourceLocation());
+ SelectElement selectElement = new SelectElement(returnExpr);
+ selectElement.setSourceLocation(stmtCopy.getSourceLocation());
+ SelectClause selectClause = new SelectClause(selectElement, null,
false);
+ selectClause.setSourceLocation(stmtCopy.getSourceLocation());
+
+ // Construct the select expression.
+ SelectBlock selectBlock = new SelectBlock(selectClause, fromClause,
null, null, null);
+ selectBlock.setSourceLocation(var.getSourceLocation());
+ SelectSetOperation selectSetOperation = new SelectSetOperation(new
SetOperationInput(selectBlock, null), null);
+ selectSetOperation.setSourceLocation(var.getSourceLocation());
+ SelectExpression selectExpression = new SelectExpression(null,
selectSetOperation, null, null, false);
+ selectExpression.setSourceLocation(var.getSourceLocation());
+ Query query = new Query(false, false, selectExpression, 0);
+ query.setBody(selectExpression);
+ query.setSourceLocation(stmtCopy.getSourceLocation());
+
+ // return the copy statement.
+ stmtCopy.setQuery(query);
+ }
+}
diff --git
a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/CheckDatasetOnlyResolutionVisitor.java
b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/CheckDatasetOnlyResolutionVisitor.java
index f700d212f9..ad4385cc88 100644
---
a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/CheckDatasetOnlyResolutionVisitor.java
+++
b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/CheckDatasetOnlyResolutionVisitor.java
@@ -39,6 +39,7 @@ import
org.apache.asterix.lang.common.expression.QuantifiedExpression;
import org.apache.asterix.lang.common.expression.RecordConstructor;
import org.apache.asterix.lang.common.expression.UnaryExpr;
import org.apache.asterix.lang.common.expression.VariableExpr;
+import org.apache.asterix.lang.common.statement.CopyToStatement;
import org.apache.asterix.lang.common.statement.FunctionDecl;
import org.apache.asterix.lang.common.statement.InsertStatement;
import org.apache.asterix.lang.common.statement.Query;
@@ -105,6 +106,11 @@ public final class CheckDatasetOnlyResolutionVisitor
return false;
}
+ @Override
+ public Boolean visit(CopyToStatement stmtCopy, VariableExpr arg) throws
CompilationException {
+ return contains(stmtCopy.getQuery(), arg);
+ }
+
@Override
public Boolean visit(Query q, VariableExpr arg) throws
CompilationException {
return contains(q, arg);
diff --git
a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/CheckSubqueryVisitor.java
b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/CheckSubqueryVisitor.java
index 399e463eab..387b7f7dbe 100644
---
a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/CheckSubqueryVisitor.java
+++
b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/CheckSubqueryVisitor.java
@@ -44,6 +44,7 @@ import
org.apache.asterix.lang.common.expression.QuantifiedExpression;
import org.apache.asterix.lang.common.expression.RecordConstructor;
import org.apache.asterix.lang.common.expression.UnaryExpr;
import org.apache.asterix.lang.common.expression.VariableExpr;
+import org.apache.asterix.lang.common.statement.CopyToStatement;
import org.apache.asterix.lang.common.statement.FunctionDecl;
import org.apache.asterix.lang.common.statement.Query;
import org.apache.asterix.lang.common.struct.Identifier;
@@ -312,6 +313,13 @@ public class CheckSubqueryVisitor extends
AbstractSqlppQueryExpressionVisitor<Bo
|| visitExprList(winExpr.getExprList(), arg);
}
+ @Override
+ public Boolean visit(CopyToStatement stmtCopy, ILangExpression arg) throws
CompilationException {
+ return stmtCopy.getQuery().accept(this, arg) ||
stmtCopy.getPathExpression().accept(this, arg)
+ || visitExprList(stmtCopy.getPartitionExpressions(), arg)
+ || visitExprList(stmtCopy.getOrderbyList(), arg);
+ }
+
@Override
public Boolean visit(CallExpr callExpr, ILangExpression arg) throws
CompilationException {
return visitExprList(callExpr.getExprList(), arg)
diff --git
a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/FreeVariableVisitor.java
b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/FreeVariableVisitor.java
index 8ab87e71bb..c1c124a273 100644
---
a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/FreeVariableVisitor.java
+++
b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/FreeVariableVisitor.java
@@ -47,6 +47,7 @@ import
org.apache.asterix.lang.common.expression.QuantifiedExpression;
import org.apache.asterix.lang.common.expression.RecordConstructor;
import org.apache.asterix.lang.common.expression.UnaryExpr;
import org.apache.asterix.lang.common.expression.VariableExpr;
+import org.apache.asterix.lang.common.statement.CopyToStatement;
import org.apache.asterix.lang.common.statement.FunctionDecl;
import org.apache.asterix.lang.common.statement.Query;
import org.apache.asterix.lang.common.struct.Identifier;
@@ -512,6 +513,15 @@ public class FreeVariableVisitor extends
AbstractSqlppQueryExpressionVisitor<Voi
return null;
}
+ @Override
+ public Void visit(CopyToStatement stmtCopy, Collection<VariableExpr>
freeVars) throws CompilationException {
+ stmtCopy.getBody().accept(this, freeVars);
+ stmtCopy.getPathExpression().accept(this, freeVars);
+ visit(stmtCopy.getPartitionExpressions(), freeVars);
+ visit(stmtCopy.getOrderbyList(), freeVars);
+ return null;
+ }
+
private void visitLetWhereClauses(List<? extends AbstractClause>
clauseList, Collection<VariableExpr> freeVars)
throws CompilationException {
if (clauseList == null || clauseList.isEmpty()) {
@@ -541,16 +551,12 @@ public class FreeVariableVisitor extends
AbstractSqlppQueryExpressionVisitor<Voi
/**
* Removes all binding variables defined in the select block for a free
variable collection.
- * @param selectFreeVars,
- * free variables.
- * @param fromBindingVars,
- * binding variables defined in the from clause of a select
block.
- * @param letsBindingVars,
- * binding variables defined in the let clauses of the select
block.
- * @param gbyBindingVars
- * binding variables defined in the groupby clauses of the
select block
- * @param gbyLetsBindingVars
- * binding variables defined in the let clauses after groupby
of the select block.
+ *
+ * @param selectFreeVars, free variables.
+ * @param fromBindingVars, binding variables defined in the from clause
of a select block.
+ * @param letsBindingVars, binding variables defined in the let clauses
of the select block.
+ * @param gbyBindingVars binding variables defined in the groupby
clauses of the select block
+ * @param gbyLetsBindingVars binding variables defined in the let clauses
after groupby of the select block.
*/
private void removeAllBindingVarsInSelectBlock(Collection<VariableExpr>
selectFreeVars,
Collection<VariableExpr> fromBindingVars, Collection<VariableExpr>
letsBindingVars,
diff --git
a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/base/AbstractSqlppContainsExpressionVisitor.java
b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/base/AbstractSqlppContainsExpressionVisitor.java
index 8e8e9590fa..368cf1d02c 100644
---
a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/base/AbstractSqlppContainsExpressionVisitor.java
+++
b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/base/AbstractSqlppContainsExpressionVisitor.java
@@ -43,6 +43,7 @@ import
org.apache.asterix.lang.common.expression.QuantifiedExpression;
import org.apache.asterix.lang.common.expression.RecordConstructor;
import org.apache.asterix.lang.common.expression.UnaryExpr;
import org.apache.asterix.lang.common.expression.VariableExpr;
+import org.apache.asterix.lang.common.statement.CopyToStatement;
import org.apache.asterix.lang.common.struct.Identifier;
import org.apache.asterix.lang.common.struct.QuantifiedPair;
import org.apache.asterix.lang.sqlpp.clause.AbstractBinaryCorrelateClause;
@@ -307,6 +308,13 @@ public abstract class
AbstractSqlppContainsExpressionVisitor<T>
|| (callExpr.hasAggregateFilterExpr() &&
visit(callExpr.getAggregateFilterExpr(), arg));
}
+ @Override
+ public Boolean visit(CopyToStatement stmtCopy, T arg) throws
CompilationException {
+ return stmtCopy.getQuery().accept(this, arg) ||
stmtCopy.getPathExpression().accept(this, arg)
+ || visitExprList(stmtCopy.getPartitionExpressions(), arg)
+ || visitExprList(stmtCopy.getOrderbyList(), arg);
+ }
+
private boolean visit(ILangExpression expr, T arg) throws
CompilationException {
return expr != null && expr.accept(this, arg);
}
diff --git
a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/base/AbstractSqlppExpressionScopingVisitor.java
b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/base/AbstractSqlppExpressionScopingVisitor.java
index dd48680267..f2519f1822 100644
---
a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/base/AbstractSqlppExpressionScopingVisitor.java
+++
b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/base/AbstractSqlppExpressionScopingVisitor.java
@@ -42,6 +42,7 @@ import
org.apache.asterix.lang.common.expression.QuantifiedExpression;
import org.apache.asterix.lang.common.expression.VariableExpr;
import org.apache.asterix.lang.common.parser.ScopeChecker;
import org.apache.asterix.lang.common.rewrites.LangRewritingContext;
+import org.apache.asterix.lang.common.statement.CopyToStatement;
import org.apache.asterix.lang.common.statement.FunctionDecl;
import org.apache.asterix.lang.common.statement.InsertStatement;
import org.apache.asterix.lang.common.statement.Query;
@@ -75,10 +76,8 @@ public class AbstractSqlppExpressionScopingVisitor extends
AbstractSqlppSimpleEx
}
/**
- * @param context,
- * manages ids of variables and guarantees uniqueness of
variables.
- * @param externalVars
- * pre-defined (external) variables that must be added to the
initial scope
+ * @param context, manages ids of variables and guarantees uniqueness
of variables.
+ * @param externalVars pre-defined (external) variables that must be added
to the initial scope
*/
public AbstractSqlppExpressionScopingVisitor(LangRewritingContext context,
Collection<VarIdentifier> externalVars) {
this.context = context;
@@ -414,6 +413,33 @@ public class AbstractSqlppExpressionScopingVisitor extends
AbstractSqlppSimpleEx
return winExpr;
}
+ @Override
+ public Expression visit(CopyToStatement stmtCopy, ILangExpression arg)
throws CompilationException {
+ stmtCopy.setBody(stmtCopy.getBody().accept(this, stmtCopy));
+
+ // Scope that only contains the source variable
+ Scope currentNewScope = scopeChecker.createNewScope();
+ VariableExpr sourceVarExpr = stmtCopy.getSourceVariable();
+ addNewVarSymbolToScope(currentNewScope, sourceVarExpr.getVar(),
stmtCopy.getSourceLocation());
+
+ // Visit partition exprs
+
stmtCopy.setPartitionExpressions(visit(stmtCopy.getPartitionExpressions(),
stmtCopy));
+
+ // Add partition variables to the current scope
+ Collection<VariableExpr> partitionVars =
stmtCopy.getPartitionsVariables().values();
+ for (VariableExpr partVarExpr : partitionVars) {
+ addNewVarSymbolToScope(currentNewScope, partVarExpr.getVar(),
partVarExpr.getSourceLocation());
+ }
+
+ // Visit order by
+ stmtCopy.setOrderbyList(visit(stmtCopy.getOrderbyList(), stmtCopy));
+
+ // Visit path expr
+ stmtCopy.setPathExpression(stmtCopy.getPathExpression().accept(this,
stmtCopy));
+
+ return null;
+ }
+
@Override
public Expression visit(IVisitorExtension ve, ILangExpression arg) throws
CompilationException {
return ve.variableScopeDispatch(this, arg, scopeChecker);
@@ -455,7 +481,7 @@ public class AbstractSqlppExpressionScopingVisitor extends
AbstractSqlppSimpleEx
* A single name identifier is first attempted to be resolved as a
variable reference. If that fails
* because there's no variable with such name then (second stage) it's
resolved as a field access on a context
* variable (if there's only one context variable defined in the local
scope).
- *
+ * <p>
* See {@link
org.apache.asterix.lang.sqlpp.rewrites.visitor.VariableCheckAndRewriteVisitor}
*/
CONTEXT_VARIABLE
diff --git
a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/base/AbstractSqlppSimpleExpressionVisitor.java
b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/base/AbstractSqlppSimpleExpressionVisitor.java
index 6bbb7406b0..16fe67a8dd 100644
---
a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/base/AbstractSqlppSimpleExpressionVisitor.java
+++
b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/base/AbstractSqlppSimpleExpressionVisitor.java
@@ -45,6 +45,7 @@ import
org.apache.asterix.lang.common.expression.QuantifiedExpression;
import org.apache.asterix.lang.common.expression.RecordConstructor;
import org.apache.asterix.lang.common.expression.UnaryExpr;
import org.apache.asterix.lang.common.expression.VariableExpr;
+import org.apache.asterix.lang.common.statement.CopyToStatement;
import org.apache.asterix.lang.common.statement.FunctionDecl;
import org.apache.asterix.lang.common.statement.InsertStatement;
import org.apache.asterix.lang.common.statement.Query;
@@ -357,6 +358,15 @@ public class AbstractSqlppSimpleExpressionVisitor
}
}
+ @Override
+ public Expression visit(CopyToStatement stmtCopy, ILangExpression arg)
throws CompilationException {
+ stmtCopy.setBody(stmtCopy.getBody().accept(this, arg));
+ stmtCopy.setPathExpression(stmtCopy.getPathExpression().accept(this,
arg));
+
stmtCopy.setPartitionExpressions(visit(stmtCopy.getPartitionExpressions(),
arg));
+ stmtCopy.setOrderbyList(visit(stmtCopy.getOrderbyList(), arg));
+ return null;
+ }
+
@Override
public Expression visit(FieldAccessor fa, ILangExpression arg) throws
CompilationException {
fa.setExpr(visit(fa.getExpr(), arg));
diff --git a/asterixdb/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj
b/asterixdb/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj
index 3675876778..f2dc18edc1 100644
--- a/asterixdb/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj
+++ b/asterixdb/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj
@@ -166,7 +166,8 @@ import
org.apache.asterix.lang.common.statement.FullTextConfigDropStatement;
import org.apache.asterix.lang.common.statement.InsertStatement;
import org.apache.asterix.lang.common.statement.InternalDetailsDecl;
import org.apache.asterix.lang.common.statement.LoadStatement;
-import org.apache.asterix.lang.common.statement.CopyStatement;
+import org.apache.asterix.lang.common.statement.CopyFromStatement;
+import org.apache.asterix.lang.common.statement.CopyToStatement;
import org.apache.asterix.lang.common.statement.NodeGroupDropStatement;
import org.apache.asterix.lang.common.statement.NodegroupDecl;
import org.apache.asterix.lang.common.statement.Query;
@@ -182,6 +183,7 @@ import org.apache.asterix.lang.common.struct.Identifier;
import org.apache.asterix.lang.common.struct.OperatorType;
import org.apache.asterix.lang.common.struct.QuantifiedPair;
import org.apache.asterix.lang.common.struct.VarIdentifier;
+import org.apache.asterix.lang.common.util.ConfigurationUtil;
import org.apache.asterix.lang.common.util.DatasetDeclParametersUtil;
import org.apache.asterix.lang.common.util.ExpressionUtils;
import org.apache.asterix.lang.common.util.RangeMapBuilder;
@@ -928,6 +930,14 @@ class SQLPPParser extends ScopeChecker implements IParser {
throw new SqlppParseException(getSourceLocation(startToken), "Invalid
primary key specification.");
}
}
+
+ private Map<String, String> getConfiguration(RecordConstructor withRecord)
throws SqlppParseException {
+ try {
+ return ConfigurationUtil.toProperties(withRecord);
+ } catch(AlgebricksException e) {
+ throw new SqlppParseException(withRecord.getSourceLocation(),
e.getMessage());
+ }
+ }
}
PARSER_END(SQLPPParser)
@@ -2848,31 +2858,47 @@ Statement SetStatement() throws ParseException:
}
}
-CopyStatement CopyStatement() throws ParseException:
+Statement CopyStatement() throws ParseException:
{
- Token startToken = null;
- Namespace namespace = null;
- Identifier datasetName = null;
- TypeExpression typeExpr = null;
- boolean alreadySorted = false;
+ Token startToken = null;
+ Pair<Namespace,Identifier> nameComponents = null;
+ Query query = null;
+ Statement stmt = null;
+ TypeExpression typeExpr = null;
+ VariableExpr alias = null;
+}
+{
+ <COPY>
+ ( LOOKAHEAD(1) <INTO> { startToken = token; } nameComponents =
QualifiedName() stmt = CopyFromStatement(startToken, nameComponents, typeExpr)
+ | LOOKAHEAD(1) <LEFTPAREN> { startToken = token; } query = Query()
<RIGHTPAREN> (<AS>)? alias = Variable() stmt = CopyToStatement(startToken,
nameComponents, query, alias)
+ | { startToken = token; } nameComponents = QualifiedName()
+ (<AS>)? (typeExpr =
DatasetTypeSpecification(RecordTypeDefinition.RecordKind.OPEN) | alias =
Variable())?
+ (stmt = CopyFromStatement(startToken, nameComponents, typeExpr) | stmt
= CopyToStatement(startToken, nameComponents, query, alias))
+ )
+ {
+ return stmt;
+ }
+}
+
+CopyFromStatement CopyFromStatement(Token startToken, Pair<Namespace,
Identifier> nameComponents, TypeExpression typeExpr) throws ParseException:
+{
+ Namespace namespace = nameComponents.first;
+ Identifier datasetName = nameComponents.second;
String adapterName;
- Map<String,String> properties;
- Pair<Namespace,Identifier> nameComponents = null;
+ RecordConstructor withRecord;
+ String sourcePath;
}
{
- <COPY> (<INTO>)? { startToken = token; } nameComponents = QualifiedName()
- {
- namespace = nameComponents.first;
- datasetName = nameComponents.second;
- }
- (<AS> typeExpr =
DatasetTypeSpecification(RecordTypeDefinition.RecordKind.OPEN))?
- <USING> adapterName = AdapterName() properties = Configuration()
+ <FROM> adapterName = AdapterName()
+ <PATH> <LEFTPAREN> sourcePath = ConcatenatedStrings() <RIGHTPAREN>
+ <WITH> withRecord = RecordConstructor()
{
ExternalDetailsDecl edd = new ExternalDetailsDecl();
edd.setAdapter(adapterName);
- edd.setProperties(properties);
+ edd.setProperties(getConfiguration(withRecord));
+
try {
- CopyStatement stmt = new CopyStatement(namespace,
datasetName.getValue(), typeExpr, edd, null);
+ CopyFromStatement stmt = new CopyFromStatement(namespace,
datasetName.getValue(), sourcePath, typeExpr, edd);
return addSourceLocation(stmt, startToken);
} catch (CompilationException e){
throw new SqlppParseException(getSourceLocation(startToken),
e.getMessage());
@@ -2880,6 +2906,76 @@ CopyStatement CopyStatement() throws ParseException:
}
}
+CopyToStatement CopyToStatement(Token startToken, Pair<Namespace, Identifier>
nameComponents, Query query, VariableExpr alias) throws ParseException:
+{
+ VariableExpr usedAlias = alias;
+ String adapterName;
+ RecordConstructor withRecord;
+ Namespace namespace = nameComponents == null ? null : nameComponents.first;
+ String datasetName = nameComponents == null ? null :
nameComponents.second.getValue();
+ Expression pathExpr = null;
+
+ List<Expression> partitionExprs = new ArrayList<Expression>();
+ Map<Integer, VariableExpr> partitionVarExprs = new HashMap<Integer,
VariableExpr>();
+ List<Expression> orderbyList = new ArrayList<Expression>();
+ List<OrderbyClause.OrderModifier> orderbyModifierList = new
ArrayList<OrderbyClause.OrderModifier>();
+ List<OrderbyClause.NullOrderModifier> orderbyNullModifierList = new
ArrayList<OrderbyClause.NullOrderModifier>();
+}
+{
+ <TO> adapterName = AdapterName()
+ <PATH> <LEFTPAREN> pathExpr = Expression() <RIGHTPAREN>
+ (CopyToOverClause(partitionExprs, partitionVarExprs, orderbyList,
orderbyModifierList, orderbyNullModifierList))?
+ <WITH> withRecord = RecordConstructor()
+ {
+ ExternalDetailsDecl edd = new ExternalDetailsDecl();
+ edd.setAdapter(adapterName);
+ edd.setProperties(getConfiguration(withRecord));
+
+ if(namespace == null) {
+ namespace = defaultNamespace;
+ }
+
+ if(usedAlias == null) {
+ usedAlias = new
VariableExpr(SqlppVariableUtil.toInternalVariableIdentifier(datasetName));
+ }
+
+ CopyToStatement stmt = new CopyToStatement(namespace, datasetName,
query, usedAlias, edd, pathExpr,
+ partitionExprs, partitionVarExprs, orderbyList,
orderbyModifierList, orderbyNullModifierList, getVarCounter());
+ return addSourceLocation(stmt, startToken);
+ }
+}
+
+ void CopyToOverClause(List<Expression> partitionExprs,
+ Map<Integer, VariableExpr> partitionVarExprs,
+ List<Expression> orderbyList,
+ List<OrderbyClause.OrderModifier> orderbyModifierList,
+ List<OrderbyClause.NullOrderModifier>
orderbyNullModifierList) throws ParseException:
+{
+ Expression partitionExpr = null;
+ VariableExpr partitionVar = null;
+ OrderbyClause orderByClause = null;
+}
+{
+ <OVER> <LEFTPAREN> <IDENTIFIER> {expectToken(PARTITION);} <BY>
+ partitionExpr = Expression() { partitionExprs.add(partitionExpr);}
+ ((<AS>)? partitionVar = Variable() { partitionVarExprs.put(0,
partitionVar); })?
+ (
+ <COMMA> partitionExpr = Expression() { partitionExprs.add(partitionExpr);
}
+ ((<AS>)? partitionVar = Variable() {
partitionVarExprs.put(partitionExprs.size() - 1, partitionVar); })?
+ )*
+ (
+ orderByClause = OrderbyClause()
+ {
+ orderbyList.addAll(orderByClause.getOrderbyList());
+ orderbyModifierList.addAll(orderByClause.getModifierList());
+ orderbyNullModifierList.addAll(orderByClause.getNullModifierList());
+ }
+ )?
+ <RIGHTPAREN>
+}
+
+
+
LoadStatement LoadStatement() throws ParseException:
{
Token startToken = null;
@@ -3120,6 +3216,19 @@ Pair<String, String> KeyValuePair() throws
ParseException:
}
}
+String ConcatenatedStrings() throws ParseException:
+{
+ String stringVal;
+ StringBuilder stringBuilder = new StringBuilder();
+}
+{
+ stringVal = StringLiteral() { stringBuilder.append(stringVal); }
+ (<COMMA> stringVal = StringLiteral() {
stringBuilder.append(',').append(stringVal); })*
+ {
+ return stringBuilder.toString();
+ }
+}
+
Map<String,String> Properties() throws ParseException:
{
Map<String,String> properties = new HashMap<String,String>();
