This is an automated email from the ASF dual-hosted git repository.
wyk pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
The following commit(s) were added to refs/heads/master by this push:
new 53d1488508 [ASTERIXDB-3347][COMP] Refactor COPY TO to support
different types of write destinations
53d1488508 is described below
commit 53d14885081b591384a5690bfcdaabeb94b213e9
Author: Hussain Towaileb <[email protected]>
AuthorDate: Tue Jan 30 06:21:39 2024 +0300
[ASTERIXDB-3347][COMP] Refactor COPY TO to support different types of write
destinations
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
This change refactors the COPY TO code
to allow for easier extendability for supporting
other types of destinations to write to.
Change-Id: Id90a30c1e9f41ca82ef28f6edd0569a69002572c
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/18146
Integration-Tests: Jenkins <[email protected]>
Tested-by: Jenkins <[email protected]>
Reviewed-by: Hussain Towaileb <[email protected]>
Reviewed-by: Wail Alkowaileet <[email protected]>
---
.../asterix/translator/CompiledStatements.java | 12 +++++++
.../lang/common/statement/CopyToStatement.java | 41 +++++++++++++++++++++-
.../lang/common/visitor/FormatPrintVisitor.java | 26 ++++++++++++--
.../asterix-lang-sqlpp/src/main/javacc/SQLPP.jj | 6 ++--
4 files changed, 78 insertions(+), 7 deletions(-)
diff --git
a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/CompiledStatements.java
b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/CompiledStatements.java
index 0ff0b728be..80252026fb 100644
---
a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/CompiledStatements.java
+++
b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/CompiledStatements.java
@@ -602,6 +602,8 @@ public class CompiledStatements {
private final List<Expression> orderbyList;
private final List<OrderbyClause.OrderModifier> orderByModifiers;
private final List<OrderbyClause.NullOrderModifier>
orderByNullModifierList;
+ private final List<Expression> keyExpressions;
+ private final boolean autogenerated;
public CompiledCopyToStatement(CopyToStatement copyToStatement) {
this.query = copyToStatement.getQuery();
@@ -615,6 +617,8 @@ public class CompiledStatements {
this.orderbyList = copyToStatement.getOrderByList();
this.orderByModifiers = copyToStatement.getOrderByModifiers();
this.orderByNullModifierList =
copyToStatement.getOrderByNullModifierList();
+ this.keyExpressions = copyToStatement.getKeyExpressions();
+ this.autogenerated = copyToStatement.isAutogenerated();
}
@Override
@@ -669,6 +673,14 @@ public class CompiledStatements {
public List<OrderbyClause.NullOrderModifier> getOrderByNullModifiers()
{
return orderByNullModifierList;
}
+
+ public List<Expression> getKeyExpressions() {
+ return keyExpressions;
+ }
+
+ public boolean isAutogenerated() {
+ return autogenerated;
+ }
}
}
diff --git
a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/CopyToStatement.java
b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/CopyToStatement.java
index dbe7b35b2b..2520755dbb 100644
---
a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/CopyToStatement.java
+++
b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/CopyToStatement.java
@@ -21,6 +21,7 @@ package org.apache.asterix.lang.common.statement;
import static org.apache.asterix.lang.common.base.Statement.Category.QUERY;
import java.util.ArrayList;
+import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -42,7 +43,8 @@ public class CopyToStatement extends AbstractStatement
implements IReturningStat
private final Map<Integer, VariableExpr> partitionsVariables;
private final List<OrderbyClause.OrderModifier> orderByModifiers;
private final List<OrderbyClause.NullOrderModifier>
orderByNullModifierList;
-
+ private final List<Expression> keyExpressions;
+ private final boolean autogenerated;
private Namespace namespace;
private Query query;
private List<Expression> pathExpressions;
@@ -51,11 +53,30 @@ public class CopyToStatement extends AbstractStatement
implements IReturningStat
private List<Expression> orderByList;
private int varCounter;
+ public CopyToStatement(Namespace namespace, String datasetName, Query
query, VariableExpr sourceVariable,
+ ExternalDetailsDecl externalDetailsDecl, int varCounter,
List<Expression> keyExpressions,
+ boolean autogenerated) {
+ this(namespace, datasetName, query, sourceVariable,
externalDetailsDecl, new ArrayList<>(), new ArrayList<>(),
+ new HashMap<>(), new ArrayList<>(), new ArrayList<>(), new
ArrayList<>(), varCounter, keyExpressions,
+ autogenerated);
+ }
+
public CopyToStatement(Namespace namespace, String datasetName, Query
query, VariableExpr sourceVariable,
ExternalDetailsDecl externalDetailsDecl, List<Expression>
pathExpressions,
List<Expression> partitionExpressions, Map<Integer, VariableExpr>
partitionsVariables,
List<Expression> orderbyList, List<OrderbyClause.OrderModifier>
orderByModifiers,
List<OrderbyClause.NullOrderModifier> orderByNullModifierList, int
varCounter) {
+ this(namespace, datasetName, query, sourceVariable,
externalDetailsDecl, pathExpressions, partitionExpressions,
+ partitionsVariables, orderbyList, orderByModifiers,
orderByNullModifierList, varCounter,
+ new ArrayList<>(), false);
+ }
+
+ private CopyToStatement(Namespace namespace, String datasetName, Query
query, VariableExpr sourceVariable,
+ ExternalDetailsDecl externalDetailsDecl, List<Expression>
pathExpressions,
+ List<Expression> partitionExpressions, Map<Integer, VariableExpr>
partitionsVariables,
+ List<Expression> orderbyList, List<OrderbyClause.OrderModifier>
orderByModifiers,
+ List<OrderbyClause.NullOrderModifier> orderByNullModifierList, int
varCounter,
+ List<Expression> keyExpressions, boolean autogenerated) {
this.namespace = namespace;
this.datasetName = datasetName;
this.query = query;
@@ -68,6 +89,8 @@ public class CopyToStatement extends AbstractStatement
implements IReturningStat
this.orderByModifiers = orderByModifiers;
this.orderByNullModifierList = orderByNullModifierList;
this.varCounter = varCounter;
+ this.keyExpressions = keyExpressions;
+ this.autogenerated = autogenerated;
if (pathExpressions.isEmpty()) {
// Ensure path expressions to have at least an empty string
@@ -203,4 +226,20 @@ public class CopyToStatement extends AbstractStatement
implements IReturningStat
public void setBody(Expression expr) {
query.setBody(expr);
}
+
+ public List<Expression> getKeyExpressions() {
+ return keyExpressions;
+ }
+
+ public boolean isAutogenerated() {
+ return autogenerated;
+ }
+
+ public boolean isSinkFileStore() {
+ return keyExpressions.isEmpty() && !autogenerated;
+ }
+
+ public boolean isSinkDatabaseWithKey() {
+ return !keyExpressions.isEmpty() || autogenerated;
+ }
}
diff --git
a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/FormatPrintVisitor.java
b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/FormatPrintVisitor.java
index 3091b3087e..52e267851b 100644
---
a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/FormatPrintVisitor.java
+++
b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/FormatPrintVisitor.java
@@ -573,6 +573,20 @@ public abstract class FormatPrintVisitor implements
ILangVisitor<Void, Integer>
cto.getSourceVariable().accept(this, step);
out.println();
+ if (cto.isSinkFileStore()) {
+ formatPrintCopyToFileStore(cto, step);
+ } else if (cto.isSinkDatabaseWithKey()) {
+ formatPrintCopyToDatabaseWithKey(cto, step);
+ } else {
+ throw new IllegalStateException("NYI: This should never happen");
+ }
+
+ out.println("with ");
+ printConfiguration(cto.getExternalDetailsDecl().getProperties());
+ return null;
+ }
+
+ private void formatPrintCopyToFileStore(CopyToStatement cto, Integer step)
throws CompilationException {
out.print("path (");
printDelimitedExpressions(cto.getPathExpressions(), COMMA, step + 1);
out.print(")");
@@ -606,10 +620,16 @@ public abstract class FormatPrintVisitor implements
ILangVisitor<Void, Integer>
}
out.println(')');
}
+ }
- out.println("with ");
- printConfiguration(cto.getExternalDetailsDecl().getProperties());
- return null;
+ private void formatPrintCopyToDatabaseWithKey(CopyToStatement cto, Integer
step) throws CompilationException {
+ out.print("key ");
+ if (!cto.getKeyExpressions().isEmpty()) {
+ printDelimitedExpressions(cto.getKeyExpressions(), COMMA, step +
1);
+ } else {
+ out.print("autogenerated");
+ }
+ out.println();
}
@Override
diff --git a/asterixdb/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj
b/asterixdb/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj
index a81807ce7d..a1b7daf3cc 100644
--- a/asterixdb/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj
+++ b/asterixdb/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj
@@ -2869,12 +2869,12 @@ Statement CopyStatement() throws ParseException:
}
{
<COPY>
- ( LOOKAHEAD(1) <INTO> { startToken = token; }
+ ( <INTO> { startToken = token; }
nameComponents = QualifiedName()
((<AS>)? (typeExpr =
DatasetTypeSpecification(RecordTypeDefinition.RecordKind.OPEN)))?
stmt = CopyFromStatement(startToken, nameComponents, typeExpr)
- | LOOKAHEAD(1) <LEFTPAREN> { startToken = token; } query = Query()
<RIGHTPAREN> (<AS>)? alias = Variable() stmt = CopyToStatement(startToken,
nameComponents, query, alias)
- | { startToken = token; } nameComponents = QualifiedName()
+ | <LEFTPAREN> { startToken = token; } query = Query() <RIGHTPAREN> (<AS>)?
alias = Variable() stmt = CopyToStatement(startToken, nameComponents, query,
alias)
+ | { startToken = token; } nameComponents = QualifiedName()
(<AS>)? (typeExpr =
DatasetTypeSpecification(RecordTypeDefinition.RecordKind.OPEN) | alias =
Variable())?
(stmt = CopyFromStatement(startToken, nameComponents, typeExpr) | stmt =
CopyToStatement(startToken, nameComponents, query, alias))
)
