[jira] [Work logged] (BEAM-4044) Take advantage of Calcite DDL
[ https://issues.apache.org/jira/browse/BEAM-4044?focusedWorklogId=100398&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-100398 ] ASF GitHub Bot logged work on BEAM-4044: Author: ASF GitHub Bot Created on: 10/May/18 04:00 Start Date: 10/May/18 04:00 Worklog Time Spent: 10m Work Description: kennknowles commented on issue #5325: [BEAM-4044] [SQL] Just One Parser URL: https://github.com/apache/beam/pull/5325#issuecomment-387945265 This looks good and you reached out on the dev list long ago with a good description. I'm going to go ahead and merge it. This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org Issue Time Tracking --- Worklog Id: (was: 100398) Time Spent: 16h 10m (was: 16h) > Take advantage of Calcite DDL > - > > Key: BEAM-4044 > URL: https://issues.apache.org/jira/browse/BEAM-4044 > Project: Beam > Issue Type: New Feature > Components: dsl-sql >Reporter: Andrew Pilloud >Assignee: Andrew Pilloud >Priority: Major > Time Spent: 16h 10m > Remaining Estimate: 0h > > In Calcite 1.15 support for abstract DDL moved into calcite core. We should > take advantage of that. -- This message was sent by Atlassian JIRA (v7.6.3#76005)
[jira] [Work logged] (BEAM-4044) Take advantage of Calcite DDL
[
https://issues.apache.org/jira/browse/BEAM-4044?focusedWorklogId=100399&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-100399
]
ASF GitHub Bot logged work on BEAM-4044:
Author: ASF GitHub Bot
Created on: 10/May/18 04:00
Start Date: 10/May/18 04:00
Worklog Time Spent: 10m
Work Description: kennknowles closed pull request #5325: [BEAM-4044]
[SQL] Just One Parser
URL: https://github.com/apache/beam/pull/5325
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git
a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/BeamSqlCli.java
b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/BeamSqlCli.java
index ce592cfbabb..d7ec10d3e43 100644
---
a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/BeamSqlCli.java
+++
b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/BeamSqlCli.java
@@ -20,7 +20,6 @@
import org.apache.beam.sdk.Pipeline;
import org.apache.beam.sdk.annotations.Experimental;
import org.apache.beam.sdk.extensions.sql.impl.BeamSqlEnv;
-import org.apache.beam.sdk.extensions.sql.impl.parser.BeamSqlParser;
import org.apache.beam.sdk.extensions.sql.impl.rel.BeamRelNode;
import org.apache.beam.sdk.extensions.sql.meta.store.MetaStore;
import org.apache.beam.sdk.options.PipelineOptions;
@@ -28,6 +27,9 @@
import org.apache.calcite.plan.RelOptUtil;
import org.apache.calcite.sql.SqlExecutableStatement;
import org.apache.calcite.sql.SqlNode;
+import org.apache.calcite.sql.parser.SqlParseException;
+import org.apache.calcite.tools.RelConversionException;
+import org.apache.calcite.tools.ValidationException;
/**
* {@link BeamSqlCli} provides methods to execute Beam SQL with an interactive
client.
@@ -54,7 +56,8 @@ public MetaStore getMetaStore() {
/**
* Returns a human readable representation of the query execution plan.
*/
- public String explainQuery(String sqlString) throws Exception {
+ public String explainQuery(String sqlString)
+ throws ValidationException, RelConversionException, SqlParseException {
BeamRelNode exeTree = env.getPlanner().convertToBeamRel(sqlString);
String beamPlan = RelOptUtil.toString(exeTree);
return beamPlan;
@@ -63,9 +66,9 @@ public String explainQuery(String sqlString) throws Exception
{
/**
* Executes the given sql.
*/
- public void execute(String sqlString) throws Exception {
-BeamSqlParser parser = new BeamSqlParser(sqlString);
-SqlNode sqlNode = parser.impl().parseSqlStmtEof();
+ public void execute(String sqlString)
+ throws ValidationException, RelConversionException, SqlParseException {
+SqlNode sqlNode = env.getPlanner().parse(sqlString);
// DDL nodes are SqlExecutableStatement
if (sqlNode instanceof SqlExecutableStatement) {
diff --git
a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/QueryTransform.java
b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/QueryTransform.java
index 45b649e466e..2757f172886 100644
---
a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/QueryTransform.java
+++
b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/QueryTransform.java
@@ -18,8 +18,6 @@
package org.apache.beam.sdk.extensions.sql;
-
-import static
org.apache.beam.sdk.extensions.sql.QueryValidationHelper.validateQuery;
import static org.apache.beam.sdk.extensions.sql.SchemaHelper.toRows;
import com.google.auto.value.AutoValue;
@@ -35,11 +33,13 @@
import org.apache.beam.sdk.transforms.PTransform;
import org.apache.beam.sdk.transforms.SerializableFunction;
import org.apache.beam.sdk.values.PCollection;
-import org.apache.beam.sdk.values.PCollectionTuple;
import org.apache.beam.sdk.values.PInput;
import org.apache.beam.sdk.values.PValue;
import org.apache.beam.sdk.values.Row;
import org.apache.beam.sdk.values.TupleTag;
+import org.apache.calcite.sql.parser.SqlParseException;
+import org.apache.calcite.tools.RelConversionException;
+import org.apache.calcite.tools.ValidationException;
/**
* A {@link PTransform} representing an execution plan for a SQL query.
@@ -59,20 +59,13 @@
public PCollection expand(PInput input) {
BeamSqlEnv sqlEnv = new BeamSqlEnv(toTableProvider(input));
-if (input instanceof PCollection) {
- validateQuery(sqlEnv, queryString());
-}
-
registerFunctions(sqlEnv);
try {
- return
- PCollectionTuple.empty(input.getPipeline()).apply(
- sqlEnv
- .getPlanner()
- .convertToBeamRel(queryString())
- .toPTransform
[jira] [Work logged] (BEAM-4044) Take advantage of Calcite DDL
[
https://issues.apache.org/jira/browse/BEAM-4044?focusedWorklogId=100397&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-100397
]
ASF GitHub Bot logged work on BEAM-4044:
Author: ASF GitHub Bot
Created on: 10/May/18 04:00
Start Date: 10/May/18 04:00
Worklog Time Spent: 10m
Work Description: kennknowles commented on a change in pull request
#5325: [BEAM-4044] [SQL] Just One Parser
URL: https://github.com/apache/beam/pull/5325#discussion_r187234601
##
File path:
sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/BeamSqlDslJoinTest.java
##
@@ -230,7 +224,9 @@ public void
testRejectsUnboundedWithinWindowsWithEndOfWindowTrigger() throws Exc
.accumulatingFiredPanes());
PCollectionTuple inputs = tuple("ORDER_DETAILS1", orders,
"ORDER_DETAILS2", orders);
-thrown.expectCause(expectedSingleFireTrigger());
+thrown.expect(UnsupportedOperationException.class);
+thrown.expectMessage(
+stringContainsInOrder(Arrays.asList("once per window", "default
trigger")));
Review comment:
I'm 50/50 on these. I do hold to the discipline that a test should not be
factored or engineered, but should be readable in isolation as pretty much
straight line. At the same time, this one seemed pretty benign. If it were
written as a matcher it would probably pass muster.
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
Issue Time Tracking
---
Worklog Id: (was: 100397)
Time Spent: 16h (was: 15h 50m)
> Take advantage of Calcite DDL
> -
>
> Key: BEAM-4044
> URL: https://issues.apache.org/jira/browse/BEAM-4044
> Project: Beam
> Issue Type: New Feature
> Components: dsl-sql
>Reporter: Andrew Pilloud
>Assignee: Andrew Pilloud
>Priority: Major
> Time Spent: 16h
> Remaining Estimate: 0h
>
> In Calcite 1.15 support for abstract DDL moved into calcite core. We should
> take advantage of that.
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
[jira] [Work logged] (BEAM-4044) Take advantage of Calcite DDL
[ https://issues.apache.org/jira/browse/BEAM-4044?focusedWorklogId=100318&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-100318 ] ASF GitHub Bot logged work on BEAM-4044: Author: ASF GitHub Bot Created on: 09/May/18 23:02 Start Date: 09/May/18 23:02 Worklog Time Spent: 10m Work Description: apilloud commented on issue #5325: [BEAM-4044] [SQL] Just One Parser URL: https://github.com/apache/beam/pull/5325#issuecomment-387900646 Failed `:beam-runners-flink_2.11:test` This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org Issue Time Tracking --- Worklog Id: (was: 100318) Time Spent: 15h 40m (was: 15.5h) > Take advantage of Calcite DDL > - > > Key: BEAM-4044 > URL: https://issues.apache.org/jira/browse/BEAM-4044 > Project: Beam > Issue Type: New Feature > Components: dsl-sql >Reporter: Andrew Pilloud >Assignee: Andrew Pilloud >Priority: Major > Time Spent: 15h 40m > Remaining Estimate: 0h > > In Calcite 1.15 support for abstract DDL moved into calcite core. We should > take advantage of that. -- This message was sent by Atlassian JIRA (v7.6.3#76005)
[jira] [Work logged] (BEAM-4044) Take advantage of Calcite DDL
[ https://issues.apache.org/jira/browse/BEAM-4044?focusedWorklogId=100319&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-100319 ] ASF GitHub Bot logged work on BEAM-4044: Author: ASF GitHub Bot Created on: 09/May/18 23:02 Start Date: 09/May/18 23:02 Worklog Time Spent: 10m Work Description: apilloud commented on issue #5325: [BEAM-4044] [SQL] Just One Parser URL: https://github.com/apache/beam/pull/5325#issuecomment-387900665 run java precommit This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org Issue Time Tracking --- Worklog Id: (was: 100319) Time Spent: 15h 50m (was: 15h 40m) > Take advantage of Calcite DDL > - > > Key: BEAM-4044 > URL: https://issues.apache.org/jira/browse/BEAM-4044 > Project: Beam > Issue Type: New Feature > Components: dsl-sql >Reporter: Andrew Pilloud >Assignee: Andrew Pilloud >Priority: Major > Time Spent: 15h 50m > Remaining Estimate: 0h > > In Calcite 1.15 support for abstract DDL moved into calcite core. We should > take advantage of that. -- This message was sent by Atlassian JIRA (v7.6.3#76005)
[jira] [Work logged] (BEAM-4044) Take advantage of Calcite DDL
[ https://issues.apache.org/jira/browse/BEAM-4044?focusedWorklogId=100294&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-100294 ] ASF GitHub Bot logged work on BEAM-4044: Author: ASF GitHub Bot Created on: 09/May/18 22:16 Start Date: 09/May/18 22:16 Worklog Time Spent: 10m Work Description: apilloud commented on issue #5325: [BEAM-4044] [SQL] Just One Parser URL: https://github.com/apache/beam/pull/5325#issuecomment-387892183 This is the parser consolidation I proposed on dev. R: @kennknowles cc: @akedin @XuMingmin @xumingming This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org Issue Time Tracking --- Worklog Id: (was: 100294) Time Spent: 15.5h (was: 15h 20m) > Take advantage of Calcite DDL > - > > Key: BEAM-4044 > URL: https://issues.apache.org/jira/browse/BEAM-4044 > Project: Beam > Issue Type: New Feature > Components: dsl-sql >Reporter: Andrew Pilloud >Assignee: Andrew Pilloud >Priority: Major > Time Spent: 15.5h > Remaining Estimate: 0h > > In Calcite 1.15 support for abstract DDL moved into calcite core. We should > take advantage of that. -- This message was sent by Atlassian JIRA (v7.6.3#76005)
[jira] [Work logged] (BEAM-4044) Take advantage of Calcite DDL
[ https://issues.apache.org/jira/browse/BEAM-4044?focusedWorklogId=100292&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-100292 ] ASF GitHub Bot logged work on BEAM-4044: Author: ASF GitHub Bot Created on: 09/May/18 22:10 Start Date: 09/May/18 22:10 Worklog Time Spent: 10m Work Description: apilloud opened a new pull request #5325: [BEAM-4044] [SQL] Just One Parser URL: https://github.com/apache/beam/pull/5325 Consolidate on one parser and drop `toLowerCase`. Follow this checklist to help us incorporate your contribution quickly and easily: - [X] Make sure there is a [JIRA issue](https://issues.apache.org/jira/projects/BEAM/issues/) filed for the change (usually before you start working on it). Trivial changes like typos do not require a JIRA issue. Your pull request should address just this issue, without pulling in other changes. - [X] Format the pull request title like `[BEAM-XXX] Fixes bug in ApproximateQuantiles`, where you replace `BEAM-XXX` with the appropriate JIRA issue. - [X] Write a pull request description that is detailed enough to understand: - [X] What the pull request does - [X] Why it does it - [X] How it does it - [X] Why this approach - [X] Each commit in the pull request should have a meaningful subject line and body. - [X] Run `./gradlew build` to make sure basic checks pass. A more thorough check will be performed on your pull request automatically. - [ ] If this contribution is large, please file an Apache [Individual Contributor License Agreement](https://www.apache.org/licenses/icla.pdf). This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org Issue Time Tracking --- Worklog Id: (was: 100292) Time Spent: 15h 20m (was: 15h 10m) > Take advantage of Calcite DDL > - > > Key: BEAM-4044 > URL: https://issues.apache.org/jira/browse/BEAM-4044 > Project: Beam > Issue Type: New Feature > Components: dsl-sql >Reporter: Andrew Pilloud >Assignee: Andrew Pilloud >Priority: Major > Time Spent: 15h 20m > Remaining Estimate: 0h > > In Calcite 1.15 support for abstract DDL moved into calcite core. We should > take advantage of that. -- This message was sent by Atlassian JIRA (v7.6.3#76005)
[jira] [Work logged] (BEAM-4044) Take advantage of Calcite DDL
[
https://issues.apache.org/jira/browse/BEAM-4044?focusedWorklogId=99477&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-99477
]
ASF GitHub Bot logged work on BEAM-4044:
Author: ASF GitHub Bot
Created on: 08/May/18 12:00
Start Date: 08/May/18 12:00
Worklog Time Spent: 10m
Work Description: kennknowles closed pull request #5224: [BEAM-4044]
[SQL] Add tables via TableStore in Schema, execute DDL in Calcite model
URL: https://github.com/apache/beam/pull/5224
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git
a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/BeamSqlCli.java
b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/BeamSqlCli.java
index 8cf689084d7..ce592cfbabb 100644
---
a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/BeamSqlCli.java
+++
b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/BeamSqlCli.java
@@ -17,21 +17,16 @@
*/
package org.apache.beam.sdk.extensions.sql;
-import java.util.List;
import org.apache.beam.sdk.Pipeline;
import org.apache.beam.sdk.annotations.Experimental;
import org.apache.beam.sdk.extensions.sql.impl.BeamSqlEnv;
import org.apache.beam.sdk.extensions.sql.impl.parser.BeamSqlParser;
-import org.apache.beam.sdk.extensions.sql.impl.parser.SqlCreateTable;
-import org.apache.beam.sdk.extensions.sql.impl.parser.SqlDropTable;
import org.apache.beam.sdk.extensions.sql.impl.rel.BeamRelNode;
-import org.apache.beam.sdk.extensions.sql.meta.Table;
import org.apache.beam.sdk.extensions.sql.meta.store.MetaStore;
import org.apache.beam.sdk.options.PipelineOptions;
import org.apache.beam.sdk.options.PipelineOptionsFactory;
-import org.apache.beam.sdk.values.PCollection;
-import org.apache.beam.sdk.values.Row;
import org.apache.calcite.plan.RelOptUtil;
+import org.apache.calcite.sql.SqlExecutableStatement;
import org.apache.calcite.sql.SqlNode;
/**
@@ -47,13 +42,7 @@
public BeamSqlCli metaStore(MetaStore metaStore) {
this.metaStore = metaStore;
-this.env = new BeamSqlEnv();
-
-// dump tables in metaStore into schema
-List tables = this.metaStore.listTables();
-for (Table table : tables) {
- env.registerTable(table.getName(),
metaStore.buildBeamSqlTable(table.getName()));
-}
+this.env = new BeamSqlEnv(metaStore);
return this;
}
@@ -78,45 +67,16 @@ public void execute(String sqlString) throws Exception {
BeamSqlParser parser = new BeamSqlParser(sqlString);
SqlNode sqlNode = parser.impl().parseSqlStmtEof();
-if (sqlNode instanceof SqlCreateTable) {
- handleCreateTable((SqlCreateTable) sqlNode, metaStore);
-} else if (sqlNode instanceof SqlDropTable) {
- handleDropTable((SqlDropTable) sqlNode);
+// DDL nodes are SqlExecutableStatement
+if (sqlNode instanceof SqlExecutableStatement) {
+ ((SqlExecutableStatement) sqlNode).execute(env.getContext());
} else {
PipelineOptions options = PipelineOptionsFactory.fromArgs(new String[]
{}).withValidation()
.as(PipelineOptions.class);
options.setJobName("BeamPlanCreator");
Pipeline pipeline = Pipeline.create(options);
- compilePipeline(sqlString, pipeline, env);
+ env.getPlanner().compileBeamPipeline(sqlString, pipeline);
pipeline.run();
}
}
-
- private void handleCreateTable(SqlCreateTable stmt, MetaStore store) {
-Table table = stmt.toTable();
-if (table.getType() == null) {
- throw new IllegalStateException("Table type is not specified and
BeamSqlCli#defaultTableType"
- + "is not configured!");
-}
-
-store.createTable(table);
-
-// register the new table into the schema
-env.registerTable(table.getName(),
metaStore.buildBeamSqlTable(table.getName()));
- }
-
- private void handleDropTable(SqlDropTable stmt) {
-metaStore.dropTable(stmt.getNameSimple());
-env.deregisterTable(stmt.getNameSimple());
- }
-
- /**
- * compile SQL, and return a {@link Pipeline}.
- */
- private static PCollection compilePipeline(String sqlStatement,
Pipeline basePipeline,
- BeamSqlEnv sqlEnv) throws
Exception {
-PCollection resultStream =
-sqlEnv.getPlanner().compileBeamPipeline(sqlStatement, basePipeline,
sqlEnv);
-return resultStream;
- }
}
diff --git
a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/QueryTransform.java
b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/QueryTransform.java
index 274c14a26e7..45b649e466e 100644
---
a/sdks/java/extensions/sql/sr
[jira] [Work logged] (BEAM-4044) Take advantage of Calcite DDL
[ https://issues.apache.org/jira/browse/BEAM-4044?focusedWorklogId=99242&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-99242 ] ASF GitHub Bot logged work on BEAM-4044: Author: ASF GitHub Bot Created on: 07/May/18 22:35 Start Date: 07/May/18 22:35 Worklog Time Spent: 10m Work Description: apilloud commented on issue #5224: [BEAM-4044] [SQL] Add tables via TableStore in Schema, execute DDL in Calcite model URL: https://github.com/apache/beam/pull/5224#issuecomment-387227605 run java precommit This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org Issue Time Tracking --- Worklog Id: (was: 99242) Time Spent: 15h (was: 14h 50m) > Take advantage of Calcite DDL > - > > Key: BEAM-4044 > URL: https://issues.apache.org/jira/browse/BEAM-4044 > Project: Beam > Issue Type: New Feature > Components: dsl-sql >Reporter: Andrew Pilloud >Assignee: Andrew Pilloud >Priority: Major > Time Spent: 15h > Remaining Estimate: 0h > > In Calcite 1.15 support for abstract DDL moved into calcite core. We should > take advantage of that. -- This message was sent by Atlassian JIRA (v7.6.3#76005)
[jira] [Work logged] (BEAM-4044) Take advantage of Calcite DDL
[ https://issues.apache.org/jira/browse/BEAM-4044?focusedWorklogId=99219&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-99219 ] ASF GitHub Bot logged work on BEAM-4044: Author: ASF GitHub Bot Created on: 07/May/18 21:55 Start Date: 07/May/18 21:55 Worklog Time Spent: 10m Work Description: apilloud commented on issue #5224: [BEAM-4044] [SQL] Add tables via TableStore in Schema, execute DDL in Calcite model URL: https://github.com/apache/beam/pull/5224#issuecomment-387219176 run java precommit This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org Issue Time Tracking --- Worklog Id: (was: 99219) Time Spent: 14h 50m (was: 14h 40m) > Take advantage of Calcite DDL > - > > Key: BEAM-4044 > URL: https://issues.apache.org/jira/browse/BEAM-4044 > Project: Beam > Issue Type: New Feature > Components: dsl-sql >Reporter: Andrew Pilloud >Assignee: Andrew Pilloud >Priority: Major > Time Spent: 14h 50m > Remaining Estimate: 0h > > In Calcite 1.15 support for abstract DDL moved into calcite core. We should > take advantage of that. -- This message was sent by Atlassian JIRA (v7.6.3#76005)
[jira] [Work logged] (BEAM-4044) Take advantage of Calcite DDL
[ https://issues.apache.org/jira/browse/BEAM-4044?focusedWorklogId=99198&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-99198 ] ASF GitHub Bot logged work on BEAM-4044: Author: ASF GitHub Bot Created on: 07/May/18 21:19 Start Date: 07/May/18 21:19 Worklog Time Spent: 10m Work Description: apilloud commented on issue #5224: [BEAM-4044] [SQL] Add tables via TableStore in Schema, execute DDL in Calcite model URL: https://github.com/apache/beam/pull/5224#issuecomment-387209952 run java precommit This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org Issue Time Tracking --- Worklog Id: (was: 99198) Time Spent: 14h 40m (was: 14.5h) > Take advantage of Calcite DDL > - > > Key: BEAM-4044 > URL: https://issues.apache.org/jira/browse/BEAM-4044 > Project: Beam > Issue Type: New Feature > Components: dsl-sql >Reporter: Andrew Pilloud >Assignee: Andrew Pilloud >Priority: Major > Time Spent: 14h 40m > Remaining Estimate: 0h > > In Calcite 1.15 support for abstract DDL moved into calcite core. We should > take advantage of that. -- This message was sent by Atlassian JIRA (v7.6.3#76005)
[jira] [Work logged] (BEAM-4044) Take advantage of Calcite DDL
[ https://issues.apache.org/jira/browse/BEAM-4044?focusedWorklogId=98842&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-98842 ] ASF GitHub Bot logged work on BEAM-4044: Author: ASF GitHub Bot Created on: 07/May/18 00:09 Start Date: 07/May/18 00:09 Worklog Time Spent: 10m Work Description: kennknowles commented on issue #5224: [BEAM-4044] [SQL] Add tables via TableStore in Schema, execute DDL in Calcite model URL: https://github.com/apache/beam/pull/5224#issuecomment-386927290 run java precommit This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org Issue Time Tracking --- Worklog Id: (was: 98842) Time Spent: 14.5h (was: 14h 20m) > Take advantage of Calcite DDL > - > > Key: BEAM-4044 > URL: https://issues.apache.org/jira/browse/BEAM-4044 > Project: Beam > Issue Type: New Feature > Components: dsl-sql >Reporter: Andrew Pilloud >Assignee: Andrew Pilloud >Priority: Major > Time Spent: 14.5h > Remaining Estimate: 0h > > In Calcite 1.15 support for abstract DDL moved into calcite core. We should > take advantage of that. -- This message was sent by Atlassian JIRA (v7.6.3#76005)
[jira] [Work logged] (BEAM-4044) Take advantage of Calcite DDL
[ https://issues.apache.org/jira/browse/BEAM-4044?focusedWorklogId=98634&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-98634 ] ASF GitHub Bot logged work on BEAM-4044: Author: ASF GitHub Bot Created on: 04/May/18 23:19 Start Date: 04/May/18 23:19 Worklog Time Spent: 10m Work Description: apilloud commented on issue #5224: [BEAM-4044] [SQL] Add tables via TableStore in Schema, execute DDL in Calcite model URL: https://github.com/apache/beam/pull/5224#issuecomment-386758382 run java precommit This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org Issue Time Tracking --- Worklog Id: (was: 98634) Time Spent: 14h 10m (was: 14h) > Take advantage of Calcite DDL > - > > Key: BEAM-4044 > URL: https://issues.apache.org/jira/browse/BEAM-4044 > Project: Beam > Issue Type: New Feature > Components: dsl-sql >Reporter: Andrew Pilloud >Assignee: Andrew Pilloud >Priority: Major > Time Spent: 14h 10m > Remaining Estimate: 0h > > In Calcite 1.15 support for abstract DDL moved into calcite core. We should > take advantage of that. -- This message was sent by Atlassian JIRA (v7.6.3#76005)
[jira] [Work logged] (BEAM-4044) Take advantage of Calcite DDL
[ https://issues.apache.org/jira/browse/BEAM-4044?focusedWorklogId=98635&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-98635 ] ASF GitHub Bot logged work on BEAM-4044: Author: ASF GitHub Bot Created on: 04/May/18 23:19 Start Date: 04/May/18 23:19 Worklog Time Spent: 10m Work Description: apilloud commented on issue #5224: [BEAM-4044] [SQL] Add tables via TableStore in Schema, execute DDL in Calcite model URL: https://github.com/apache/beam/pull/5224#issuecomment-386758418 Got a Beam 1 Disconnected failure. This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org Issue Time Tracking --- Worklog Id: (was: 98635) Time Spent: 14h 20m (was: 14h 10m) > Take advantage of Calcite DDL > - > > Key: BEAM-4044 > URL: https://issues.apache.org/jira/browse/BEAM-4044 > Project: Beam > Issue Type: New Feature > Components: dsl-sql >Reporter: Andrew Pilloud >Assignee: Andrew Pilloud >Priority: Major > Time Spent: 14h 20m > Remaining Estimate: 0h > > In Calcite 1.15 support for abstract DDL moved into calcite core. We should > take advantage of that. -- This message was sent by Atlassian JIRA (v7.6.3#76005)
[jira] [Work logged] (BEAM-4044) Take advantage of Calcite DDL
[
https://issues.apache.org/jira/browse/BEAM-4044?focusedWorklogId=98611&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-98611
]
ASF GitHub Bot logged work on BEAM-4044:
Author: ASF GitHub Bot
Created on: 04/May/18 22:36
Start Date: 04/May/18 22:36
Worklog Time Spent: 10m
Work Description: apilloud commented on a change in pull request #5224:
[BEAM-4044] [SQL] Add tables via TableStore in Schema, execute DDL in Calcite
model
URL: https://github.com/apache/beam/pull/5224#discussion_r186236507
##
File path:
sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/BeamCalciteSchema.java
##
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.extensions.sql.impl;
+
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Set;
+import org.apache.beam.sdk.extensions.sql.meta.Table;
+import org.apache.beam.sdk.extensions.sql.meta.provider.TableProvider;
+import org.apache.calcite.linq4j.tree.Expression;
+import org.apache.calcite.schema.Function;
+import org.apache.calcite.schema.Schema;
+import org.apache.calcite.schema.SchemaPlus;
+import org.apache.calcite.schema.SchemaVersion;
+import org.apache.calcite.schema.Schemas;
+
+/**
+ * Adapter from {@link TableProvider} to {@link Schema}.
+ */
+public class BeamCalciteSchema implements Schema {
+ private TableProvider tableProvider;
+
+ public BeamCalciteSchema(TableProvider tableProvider) {
+this.tableProvider = tableProvider;
+ }
+
+ public TableProvider getTableProvider() {
+return tableProvider;
+ }
+
+ @Override
+ public boolean isMutable() {
+return true;
Review comment:
This means the DDL is allowed to introduce new tables, but I don't know of
anywhere it is actually checked or set to anything but true in calcite.
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
Issue Time Tracking
---
Worklog Id: (was: 98611)
Time Spent: 13h 50m (was: 13h 40m)
> Take advantage of Calcite DDL
> -
>
> Key: BEAM-4044
> URL: https://issues.apache.org/jira/browse/BEAM-4044
> Project: Beam
> Issue Type: New Feature
> Components: dsl-sql
>Reporter: Andrew Pilloud
>Assignee: Andrew Pilloud
>Priority: Major
> Time Spent: 13h 50m
> Remaining Estimate: 0h
>
> In Calcite 1.15 support for abstract DDL moved into calcite core. We should
> take advantage of that.
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
[jira] [Work logged] (BEAM-4044) Take advantage of Calcite DDL
[
https://issues.apache.org/jira/browse/BEAM-4044?focusedWorklogId=98608&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-98608
]
ASF GitHub Bot logged work on BEAM-4044:
Author: ASF GitHub Bot
Created on: 04/May/18 22:36
Start Date: 04/May/18 22:36
Worklog Time Spent: 10m
Work Description: apilloud commented on a change in pull request #5224:
[BEAM-4044] [SQL] Add tables via TableStore in Schema, execute DDL in Calcite
model
URL: https://github.com/apache/beam/pull/5224#discussion_r186236461
##
File path:
sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/BeamSqlCli.java
##
@@ -78,45 +67,15 @@ public void execute(String sqlString) throws Exception {
BeamSqlParser parser = new BeamSqlParser(sqlString);
SqlNode sqlNode = parser.impl().parseSqlStmtEof();
-if (sqlNode instanceof SqlCreateTable) {
- handleCreateTable((SqlCreateTable) sqlNode, metaStore);
-} else if (sqlNode instanceof SqlDropTable) {
- handleDropTable((SqlDropTable) sqlNode);
+if (sqlNode instanceof SqlExecutableStatement) {
+ ((SqlExecutableStatement) sqlNode).execute(env.getContext());
} else {
Review comment:
Comment added: DDL nodes are SqlExecutableStatement
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
Issue Time Tracking
---
Worklog Id: (was: 98608)
Time Spent: 13h 20m (was: 13h 10m)
> Take advantage of Calcite DDL
> -
>
> Key: BEAM-4044
> URL: https://issues.apache.org/jira/browse/BEAM-4044
> Project: Beam
> Issue Type: New Feature
> Components: dsl-sql
>Reporter: Andrew Pilloud
>Assignee: Andrew Pilloud
>Priority: Major
> Time Spent: 13h 20m
> Remaining Estimate: 0h
>
> In Calcite 1.15 support for abstract DDL moved into calcite core. We should
> take advantage of that.
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
[jira] [Work logged] (BEAM-4044) Take advantage of Calcite DDL
[
https://issues.apache.org/jira/browse/BEAM-4044?focusedWorklogId=98610&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-98610
]
ASF GitHub Bot logged work on BEAM-4044:
Author: ASF GitHub Bot
Created on: 04/May/18 22:36
Start Date: 04/May/18 22:36
Worklog Time Spent: 10m
Work Description: apilloud commented on a change in pull request #5224:
[BEAM-4044] [SQL] Add tables via TableStore in Schema, execute DDL in Calcite
model
URL: https://github.com/apache/beam/pull/5224#discussion_r186236487
##
File path:
sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/store/InMemoryMetaStore.java
##
@@ -37,7 +37,8 @@
private Map tables = new HashMap<>();
private Map providers = new HashMap<>();
- public InMemoryMetaStore() {
+ @Override public String getTableType() {
+return "";
Review comment:
Ok, I added type of `store`.
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
Issue Time Tracking
---
Worklog Id: (was: 98610)
Time Spent: 13h 40m (was: 13.5h)
> Take advantage of Calcite DDL
> -
>
> Key: BEAM-4044
> URL: https://issues.apache.org/jira/browse/BEAM-4044
> Project: Beam
> Issue Type: New Feature
> Components: dsl-sql
>Reporter: Andrew Pilloud
>Assignee: Andrew Pilloud
>Priority: Major
> Time Spent: 13h 40m
> Remaining Estimate: 0h
>
> In Calcite 1.15 support for abstract DDL moved into calcite core. We should
> take advantage of that.
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
[jira] [Work logged] (BEAM-4044) Take advantage of Calcite DDL
[
https://issues.apache.org/jira/browse/BEAM-4044?focusedWorklogId=98613&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-98613
]
ASF GitHub Bot logged work on BEAM-4044:
Author: ASF GitHub Bot
Created on: 04/May/18 22:36
Start Date: 04/May/18 22:36
Worklog Time Spent: 10m
Work Description: apilloud commented on a change in pull request #5224:
[BEAM-4044] [SQL] Add tables via TableStore in Schema, execute DDL in Calcite
model
URL: https://github.com/apache/beam/pull/5224#discussion_r186236524
##
File path:
sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamJoinRel.java
##
@@ -369,8 +370,8 @@ private Row buildNullRow(BeamRelNode relNode) {
private PCollection joinAsLookup(
BeamRelNode leftRelNode,
BeamRelNode rightRelNode,
- PCollectionTuple inputPCollections) {
-PCollection factStream =
inputPCollections.apply(leftRelNode.toPTransform());
+ PInput inputPCollections) {
+PCollection factStream =
inputPCollections.getPipeline().apply(leftRelNode.toPTransform());
Review comment:
The current model is wrong, the new model is wrong. I've dropped this change
and just added another `PCollectionTuple.empty(input.getPipeline())`. We can
discuss the right way to do it as a followup.
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
Issue Time Tracking
---
Worklog Id: (was: 98613)
Time Spent: 14h (was: 13h 50m)
> Take advantage of Calcite DDL
> -
>
> Key: BEAM-4044
> URL: https://issues.apache.org/jira/browse/BEAM-4044
> Project: Beam
> Issue Type: New Feature
> Components: dsl-sql
>Reporter: Andrew Pilloud
>Assignee: Andrew Pilloud
>Priority: Major
> Time Spent: 14h
> Remaining Estimate: 0h
>
> In Calcite 1.15 support for abstract DDL moved into calcite core. We should
> take advantage of that.
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
[jira] [Work logged] (BEAM-4044) Take advantage of Calcite DDL
[
https://issues.apache.org/jira/browse/BEAM-4044?focusedWorklogId=98609&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-98609
]
ASF GitHub Bot logged work on BEAM-4044:
Author: ASF GitHub Bot
Created on: 04/May/18 22:36
Start Date: 04/May/18 22:36
Worklog Time Spent: 10m
Work Description: apilloud commented on a change in pull request #5224:
[BEAM-4044] [SQL] Add tables via TableStore in Schema, execute DDL in Calcite
model
URL: https://github.com/apache/beam/pull/5224#discussion_r186236471
##
File path:
sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/BeamSqlEnv.java
##
@@ -103,105 +82,58 @@ public void registerUdaf(String functionName,
Combine.CombineFn combineFn) {
schema.plus().add(functionName, new UdafImpl(combineFn));
}
- /**
- * Registers {@link PCollection}s in {@link PCollectionTuple} as a tables.
- *
- * Assumes that {@link PCollection} elements are {@link Row}s.
- *
- * {@link TupleTag#getId()}s are used as table names.
- */
- public void registerPCollectionTuple(PCollectionTuple pCollectionTuple) {
-pCollectionTuple
-.getAll()
-.forEach((tag, pCollection) ->
-registerPCollection(tag.getId(), (PCollection)
pCollection));
- }
-
- /**
- * Registers {@link PCollection} of {@link Row}s as a table.
- *
- * Assumes that {@link PCollection#getCoder()} returns an instance of
{@link RowCoder}.
- */
- public void registerPCollection(String name, PCollection pCollection) {
-registerTable(name, pCollection, ((RowCoder)
pCollection.getCoder()).getSchema());
- }
-
- /**
- * Registers {@link PCollection} as a table.
- */
- public void registerTable(String tableName, PCollection pCollection,
Schema schema) {
-registerTable(tableName, new BeamPCollectionTable(pCollection, schema));
- }
-
- /**
- * Registers a {@link BaseBeamTable} which can be used for all subsequent
queries.
- */
- public void registerTable(String tableName, BeamSqlTable table) {
-schema.add(tableName, new BeamCalciteTable(table));
+ public BeamQueryPlanner getPlanner() {
+return planner;
}
- public void deregisterTable(String targetTableName) {
-schema.removeTable(targetTableName);
+ public CalcitePrepare.Context getContext() {
+return new ContextImpl();
}
- private static class BeamCalciteTable extends AbstractQueryableTable
- implements ModifiableTable, TranslatableTable {
-private BeamSqlTable beamTable;
-
-public BeamCalciteTable(BeamSqlTable beamTable) {
- super(Object[].class);
- this.beamTable = beamTable;
+ private class ContextImpl implements CalcitePrepare.Context {
+@Override
+public JavaTypeFactory getTypeFactory() {
+ throw new UnsupportedOperationException();
Review comment:
Nope, changed.
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
Issue Time Tracking
---
Worklog Id: (was: 98609)
Time Spent: 13.5h (was: 13h 20m)
> Take advantage of Calcite DDL
> -
>
> Key: BEAM-4044
> URL: https://issues.apache.org/jira/browse/BEAM-4044
> Project: Beam
> Issue Type: New Feature
> Components: dsl-sql
>Reporter: Andrew Pilloud
>Assignee: Andrew Pilloud
>Priority: Major
> Time Spent: 13.5h
> Remaining Estimate: 0h
>
> In Calcite 1.15 support for abstract DDL moved into calcite core. We should
> take advantage of that.
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
[jira] [Work logged] (BEAM-4044) Take advantage of Calcite DDL
[
https://issues.apache.org/jira/browse/BEAM-4044?focusedWorklogId=98607&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-98607
]
ASF GitHub Bot logged work on BEAM-4044:
Author: ASF GitHub Bot
Created on: 04/May/18 22:36
Start Date: 04/May/18 22:36
Worklog Time Spent: 10m
Work Description: apilloud commented on a change in pull request #5224:
[BEAM-4044] [SQL] Add tables via TableStore in Schema, execute DDL in Calcite
model
URL: https://github.com/apache/beam/pull/5224#discussion_r186236457
##
File path:
sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/BeamCalciteTable.java
##
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.extensions.sql.impl;
+
+import java.util.Collection;
+import java.util.List;
+import org.apache.beam.sdk.extensions.sql.BeamSqlTable;
+import org.apache.beam.sdk.extensions.sql.impl.planner.BeamQueryPlanner;
+import org.apache.beam.sdk.extensions.sql.impl.rel.BeamIOSinkRel;
+import org.apache.beam.sdk.extensions.sql.impl.rel.BeamIOSourceRel;
+import org.apache.beam.sdk.extensions.sql.impl.utils.CalciteUtils;
+import org.apache.calcite.adapter.java.AbstractQueryableTable;
+import org.apache.calcite.linq4j.QueryProvider;
+import org.apache.calcite.linq4j.Queryable;
+import org.apache.calcite.plan.RelOptCluster;
+import org.apache.calcite.plan.RelOptTable;
+import org.apache.calcite.prepare.Prepare;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.rel.core.TableModify;
+import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.rel.type.RelDataTypeFactory;
+import org.apache.calcite.rex.RexNode;
+import org.apache.calcite.schema.ModifiableTable;
+import org.apache.calcite.schema.SchemaPlus;
+import org.apache.calcite.schema.TranslatableTable;
+
+/**
+ * Adapter from {@link BeamSqlTable} to a calcite Table.
+ */
+class BeamCalciteTable extends AbstractQueryableTable
+implements ModifiableTable, TranslatableTable {
+ private BeamSqlTable beamTable;
+
+ public BeamCalciteTable(BeamSqlTable beamTable) {
+super(Object[].class);
+this.beamTable = beamTable;
+ }
+
+ @Override
+ public RelDataType getRowType(RelDataTypeFactory typeFactory) {
+return CalciteUtils.toCalciteRowType(this.beamTable.getSchema(),
BeamQueryPlanner.TYPE_FACTORY);
Review comment:
Sure.
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
Issue Time Tracking
---
Worklog Id: (was: 98607)
Time Spent: 13h 10m (was: 13h)
> Take advantage of Calcite DDL
> -
>
> Key: BEAM-4044
> URL: https://issues.apache.org/jira/browse/BEAM-4044
> Project: Beam
> Issue Type: New Feature
> Components: dsl-sql
>Reporter: Andrew Pilloud
>Assignee: Andrew Pilloud
>Priority: Major
> Time Spent: 13h 10m
> Remaining Estimate: 0h
>
> In Calcite 1.15 support for abstract DDL moved into calcite core. We should
> take advantage of that.
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
[jira] [Work logged] (BEAM-4044) Take advantage of Calcite DDL
[
https://issues.apache.org/jira/browse/BEAM-4044?focusedWorklogId=98605&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-98605
]
ASF GitHub Bot logged work on BEAM-4044:
Author: ASF GitHub Bot
Created on: 04/May/18 22:35
Start Date: 04/May/18 22:35
Worklog Time Spent: 10m
Work Description: apilloud commented on a change in pull request #5224:
[BEAM-4044] [SQL] Add tables via TableStore in Schema, execute DDL in Calcite
model
URL: https://github.com/apache/beam/pull/5224#discussion_r186236436
##
File path:
sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/BeamSqlTableProvider.java
##
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.extensions.sql.meta.provider;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import org.apache.beam.sdk.extensions.sql.BeamSqlTable;
+import org.apache.beam.sdk.extensions.sql.meta.Table;
+
+/**
+ * A {@code BeamSqlTableProvider} provides read only set of {@code
BeamSqlTable}.
+ */
+public class BeamSqlTableProvider implements TableProvider {
+ private final String typeName;
+ private final Map tables;
+
+ public BeamSqlTableProvider(String typeName, Map
tables) {
+this.typeName = typeName;
+this.tables = tables;
+ }
+
+ @Override public String getTableType() {
+return typeName;
+ }
+
+ @Override
+ public void createTable(Table table) {
+throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void dropTable(String tableName) {
+throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public List listTables() {
+List list = new ArrayList(tables.size());
+for (Map.Entry table : tables.entrySet()) {
+ list.add(Table.builder()
+ .type(getTableType())
+ .name(table.getKey())
+ .columns(Collections.emptyList())
+ .build());
Review comment:
And I would write this exactly how I did. I find for loops to be far easier
to read then java streams.
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
Issue Time Tracking
---
Worklog Id: (was: 98605)
Time Spent: 12h 50m (was: 12h 40m)
> Take advantage of Calcite DDL
> -
>
> Key: BEAM-4044
> URL: https://issues.apache.org/jira/browse/BEAM-4044
> Project: Beam
> Issue Type: New Feature
> Components: dsl-sql
>Reporter: Andrew Pilloud
>Assignee: Andrew Pilloud
>Priority: Major
> Time Spent: 12h 50m
> Remaining Estimate: 0h
>
> In Calcite 1.15 support for abstract DDL moved into calcite core. We should
> take advantage of that.
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
[jira] [Work logged] (BEAM-4044) Take advantage of Calcite DDL
[
https://issues.apache.org/jira/browse/BEAM-4044?focusedWorklogId=98606&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-98606
]
ASF GitHub Bot logged work on BEAM-4044:
Author: ASF GitHub Bot
Created on: 04/May/18 22:35
Start Date: 04/May/18 22:35
Worklog Time Spent: 10m
Work Description: apilloud commented on a change in pull request #5224:
[BEAM-4044] [SQL] Add tables via TableStore in Schema, execute DDL in Calcite
model
URL: https://github.com/apache/beam/pull/5224#discussion_r186236449
##
File path:
sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/QueryTransform.java
##
@@ -78,25 +76,18 @@
}
}
- private PCollectionTuple toPCollectionTuple(PInput inputs) {
-return (inputs instanceof PCollection)
-? PCollectionTuple.of(new TupleTag<>(PCOLLECTION_NAME), toRows(inputs))
-: tupleOfAllInputs(inputs.getPipeline(), inputs.expand());
- }
-
- private PCollectionTuple tupleOfAllInputs(
- Pipeline pipeline,
- Map, PValue> taggedInputs) {
-
-PCollectionTuple tuple = PCollectionTuple.empty(pipeline);
-
-for (Map.Entry, PValue> input : taggedInputs.entrySet()) {
- tuple = tuple.and(
- new TupleTag<>(input.getKey().getId()),
- toRows(input.getValue()));
+ private BeamSqlTableProvider toTableProvider(PInput inputs) {
+ImmutableMap.Builder tables = ImmutableMap.builder();
+if (inputs instanceof PCollection) {
+ tables.put(PCOLLECTION_NAME,
+ new BeamPCollectionTable(toRows(inputs)));
+} else {
+ for (Map.Entry, PValue> input : inputs.expand().entrySet()) {
+tables.put(input.getKey().getId(),
+new BeamPCollectionTable(toRows(input.getValue(;
+ }
Review comment:
I agree this is hard to read, I disagree that java streams makes it more
readable. I've restructured it like you've suggested otherwise and added the
comment.
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
Issue Time Tracking
---
Worklog Id: (was: 98606)
Time Spent: 13h (was: 12h 50m)
> Take advantage of Calcite DDL
> -
>
> Key: BEAM-4044
> URL: https://issues.apache.org/jira/browse/BEAM-4044
> Project: Beam
> Issue Type: New Feature
> Components: dsl-sql
>Reporter: Andrew Pilloud
>Assignee: Andrew Pilloud
>Priority: Major
> Time Spent: 13h
> Remaining Estimate: 0h
>
> In Calcite 1.15 support for abstract DDL moved into calcite core. We should
> take advantage of that.
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
[jira] [Work logged] (BEAM-4044) Take advantage of Calcite DDL
[
https://issues.apache.org/jira/browse/BEAM-4044?focusedWorklogId=98604&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-98604
]
ASF GitHub Bot logged work on BEAM-4044:
Author: ASF GitHub Bot
Created on: 04/May/18 22:35
Start Date: 04/May/18 22:35
Worklog Time Spent: 10m
Work Description: apilloud commented on a change in pull request #5224:
[BEAM-4044] [SQL] Add tables via TableStore in Schema, execute DDL in Calcite
model
URL: https://github.com/apache/beam/pull/5224#discussion_r186236410
##
File path:
sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/parser/SqlDropObject.java
##
@@ -51,8 +59,39 @@
name.unparse(writer, leftPrec, rightPrec);
}
- public String getNameSimple() {
-return name.getSimple().toLowerCase();
+ public void execute(CalcitePrepare.Context context) {
+final List path = context.getDefaultSchemaPath();
+CalciteSchema schema = context.getRootSchema();
+for (String p : path) {
+ schema = schema.getSubSchema(p, true);
+}
+final boolean existed;
+switch (getKind()) {
+case DROP_TABLE:
+case DROP_MATERIALIZED_VIEW:
Review comment:
Deleted. Calcite does a lot of things for us when we get out of the way,
views are probably one of them.
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
Issue Time Tracking
---
Worklog Id: (was: 98604)
Time Spent: 12h 40m (was: 12.5h)
> Take advantage of Calcite DDL
> -
>
> Key: BEAM-4044
> URL: https://issues.apache.org/jira/browse/BEAM-4044
> Project: Beam
> Issue Type: New Feature
> Components: dsl-sql
>Reporter: Andrew Pilloud
>Assignee: Andrew Pilloud
>Priority: Major
> Time Spent: 12h 40m
> Remaining Estimate: 0h
>
> In Calcite 1.15 support for abstract DDL moved into calcite core. We should
> take advantage of that.
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
[jira] [Work logged] (BEAM-4044) Take advantage of Calcite DDL
[
https://issues.apache.org/jira/browse/BEAM-4044?focusedWorklogId=98538&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-98538
]
ASF GitHub Bot logged work on BEAM-4044:
Author: ASF GitHub Bot
Created on: 04/May/18 18:14
Start Date: 04/May/18 18:14
Worklog Time Spent: 10m
Work Description: kennknowles commented on a change in pull request
#5224: [BEAM-4044] [SQL] Add tables via TableStore in Schema, execute DDL in
Calcite model
URL: https://github.com/apache/beam/pull/5224#discussion_r186170182
##
File path:
sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/BeamSqlCli.java
##
@@ -47,13 +42,7 @@
public BeamSqlCli metaStore(MetaStore metaStore) {
this.metaStore = metaStore;
-this.env = new BeamSqlEnv();
-
-// dump tables in metaStore into schema
-List tables = this.metaStore.listTables();
-for (Table table : tables) {
- env.registerTable(table.getName(),
metaStore.buildBeamSqlTable(table.getName()));
-}
+this.env = new BeamSqlEnv(metaStore);
Review comment:
nice
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
Issue Time Tracking
---
Worklog Id: (was: 98538)
Time Spent: 12.5h (was: 12h 20m)
> Take advantage of Calcite DDL
> -
>
> Key: BEAM-4044
> URL: https://issues.apache.org/jira/browse/BEAM-4044
> Project: Beam
> Issue Type: New Feature
> Components: dsl-sql
>Reporter: Andrew Pilloud
>Assignee: Andrew Pilloud
>Priority: Major
> Time Spent: 12.5h
> Remaining Estimate: 0h
>
> In Calcite 1.15 support for abstract DDL moved into calcite core. We should
> take advantage of that.
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
[jira] [Work logged] (BEAM-4044) Take advantage of Calcite DDL
[
https://issues.apache.org/jira/browse/BEAM-4044?focusedWorklogId=98533&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-98533
]
ASF GitHub Bot logged work on BEAM-4044:
Author: ASF GitHub Bot
Created on: 04/May/18 18:14
Start Date: 04/May/18 18:14
Worklog Time Spent: 10m
Work Description: kennknowles commented on a change in pull request
#5224: [BEAM-4044] [SQL] Add tables via TableStore in Schema, execute DDL in
Calcite model
URL: https://github.com/apache/beam/pull/5224#discussion_r186171758
##
File path:
sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/parser/SqlDropObject.java
##
@@ -51,8 +59,39 @@
name.unparse(writer, leftPrec, rightPrec);
}
- public String getNameSimple() {
-return name.getSimple().toLowerCase();
+ public void execute(CalcitePrepare.Context context) {
+final List path = context.getDefaultSchemaPath();
+CalciteSchema schema = context.getRootSchema();
+for (String p : path) {
+ schema = schema.getSubSchema(p, true);
+}
+final boolean existed;
+switch (getKind()) {
+case DROP_TABLE:
+case DROP_MATERIALIZED_VIEW:
Review comment:
Interesting question for the future. For pure SQL REPL use, we probably
would want something to name queries for reuse. Does calcite manage these for
us, and only delegates materialized views?
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
Issue Time Tracking
---
Worklog Id: (was: 98533)
Time Spent: 12h 20m (was: 12h 10m)
> Take advantage of Calcite DDL
> -
>
> Key: BEAM-4044
> URL: https://issues.apache.org/jira/browse/BEAM-4044
> Project: Beam
> Issue Type: New Feature
> Components: dsl-sql
>Reporter: Andrew Pilloud
>Assignee: Andrew Pilloud
>Priority: Major
> Time Spent: 12h 20m
> Remaining Estimate: 0h
>
> In Calcite 1.15 support for abstract DDL moved into calcite core. We should
> take advantage of that.
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
[jira] [Work logged] (BEAM-4044) Take advantage of Calcite DDL
[
https://issues.apache.org/jira/browse/BEAM-4044?focusedWorklogId=98537&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-98537
]
ASF GitHub Bot logged work on BEAM-4044:
Author: ASF GitHub Bot
Created on: 04/May/18 18:14
Start Date: 04/May/18 18:14
Worklog Time Spent: 10m
Work Description: kennknowles commented on a change in pull request
#5224: [BEAM-4044] [SQL] Add tables via TableStore in Schema, execute DDL in
Calcite model
URL: https://github.com/apache/beam/pull/5224#discussion_r186172690
##
File path:
sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamJoinRel.java
##
@@ -133,9 +133,10 @@ public Join copy(
.setCoder(CalciteUtils.toBeamSchema(getRowType()).getRowCoder());
}
- PCollection leftRows = inputPCollections.apply("left",
leftRelNode.toPTransform());
+ PCollection leftRows =
+ inputPCollections.getPipeline().apply("left",
leftRelNode.toPTransform());
PCollection rightRows =
- inputPCollections.apply("right", rightRelNode.toPTransform());
+ inputPCollections.getPipeline().apply("right",
rightRelNode.toPTransform());
Review comment:
Agree. Does this actually work? If so, is there a different data path
whereby the left and right collections are passed as input? This invocation
will not register them as inputs to the transform.
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
Issue Time Tracking
---
Worklog Id: (was: 98537)
Time Spent: 12.5h (was: 12h 20m)
> Take advantage of Calcite DDL
> -
>
> Key: BEAM-4044
> URL: https://issues.apache.org/jira/browse/BEAM-4044
> Project: Beam
> Issue Type: New Feature
> Components: dsl-sql
>Reporter: Andrew Pilloud
>Assignee: Andrew Pilloud
>Priority: Major
> Time Spent: 12.5h
> Remaining Estimate: 0h
>
> In Calcite 1.15 support for abstract DDL moved into calcite core. We should
> take advantage of that.
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
[jira] [Work logged] (BEAM-4044) Take advantage of Calcite DDL
[
https://issues.apache.org/jira/browse/BEAM-4044?focusedWorklogId=98539&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-98539
]
ASF GitHub Bot logged work on BEAM-4044:
Author: ASF GitHub Bot
Created on: 04/May/18 18:14
Start Date: 04/May/18 18:14
Worklog Time Spent: 10m
Work Description: kennknowles commented on a change in pull request
#5224: [BEAM-4044] [SQL] Add tables via TableStore in Schema, execute DDL in
Calcite model
URL: https://github.com/apache/beam/pull/5224#discussion_r186171129
##
File path:
sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/BeamCalciteSchema.java
##
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.extensions.sql.impl;
+
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Set;
+import org.apache.beam.sdk.extensions.sql.meta.Table;
+import org.apache.beam.sdk.extensions.sql.meta.provider.TableProvider;
+import org.apache.calcite.linq4j.tree.Expression;
+import org.apache.calcite.schema.Function;
+import org.apache.calcite.schema.Schema;
+import org.apache.calcite.schema.SchemaPlus;
+import org.apache.calcite.schema.SchemaVersion;
+import org.apache.calcite.schema.Schemas;
+
+/**
+ * Adapter from {@link TableProvider} to {@link Schema}.
+ */
+public class BeamCalciteSchema implements Schema {
+ private TableProvider tableProvider;
+
+ public BeamCalciteSchema(TableProvider tableProvider) {
+this.tableProvider = tableProvider;
+ }
+
+ public TableProvider getTableProvider() {
+return tableProvider;
+ }
+
+ @Override
+ public boolean isMutable() {
+return true;
Review comment:
Curious - why? Is it that the underlying `TableProvider` is mutable? Or does
this simply mean that the DDL is allowed to introduce new tables?
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
Issue Time Tracking
---
Worklog Id: (was: 98539)
> Take advantage of Calcite DDL
> -
>
> Key: BEAM-4044
> URL: https://issues.apache.org/jira/browse/BEAM-4044
> Project: Beam
> Issue Type: New Feature
> Components: dsl-sql
>Reporter: Andrew Pilloud
>Assignee: Andrew Pilloud
>Priority: Major
> Time Spent: 12.5h
> Remaining Estimate: 0h
>
> In Calcite 1.15 support for abstract DDL moved into calcite core. We should
> take advantage of that.
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
[jira] [Work logged] (BEAM-4044) Take advantage of Calcite DDL
[
https://issues.apache.org/jira/browse/BEAM-4044?focusedWorklogId=98536&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-98536
]
ASF GitHub Bot logged work on BEAM-4044:
Author: ASF GitHub Bot
Created on: 04/May/18 18:14
Start Date: 04/May/18 18:14
Worklog Time Spent: 10m
Work Description: kennknowles commented on a change in pull request
#5224: [BEAM-4044] [SQL] Add tables via TableStore in Schema, execute DDL in
Calcite model
URL: https://github.com/apache/beam/pull/5224#discussion_r186173034
##
File path:
sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamSetOperatorRelBase.java
##
@@ -61,12 +61,12 @@ public BeamSetOperatorRelBase(BeamRelNode beamRelNode,
OpType opType,
this.all = all;
}
- public PCollection buildBeamPipeline(PCollectionTuple
inputPCollections) {
+ public PCollection buildBeamPipeline(PInput inputPCollections) {
PCollection leftRows =
-inputPCollections.apply(
+inputPCollections.getPipeline().apply(
Review comment:
and here, and throughout
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
Issue Time Tracking
---
Worklog Id: (was: 98536)
Time Spent: 12.5h (was: 12h 20m)
> Take advantage of Calcite DDL
> -
>
> Key: BEAM-4044
> URL: https://issues.apache.org/jira/browse/BEAM-4044
> Project: Beam
> Issue Type: New Feature
> Components: dsl-sql
>Reporter: Andrew Pilloud
>Assignee: Andrew Pilloud
>Priority: Major
> Time Spent: 12.5h
> Remaining Estimate: 0h
>
> In Calcite 1.15 support for abstract DDL moved into calcite core. We should
> take advantage of that.
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
[jira] [Work logged] (BEAM-4044) Take advantage of Calcite DDL
[
https://issues.apache.org/jira/browse/BEAM-4044?focusedWorklogId=98534&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-98534
]
ASF GitHub Bot logged work on BEAM-4044:
Author: ASF GitHub Bot
Created on: 04/May/18 18:14
Start Date: 04/May/18 18:14
Worklog Time Spent: 10m
Work Description: kennknowles commented on a change in pull request
#5224: [BEAM-4044] [SQL] Add tables via TableStore in Schema, execute DDL in
Calcite model
URL: https://github.com/apache/beam/pull/5224#discussion_r186170150
##
File path:
sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/BeamSqlCli.java
##
@@ -78,45 +67,15 @@ public void execute(String sqlString) throws Exception {
BeamSqlParser parser = new BeamSqlParser(sqlString);
SqlNode sqlNode = parser.impl().parseSqlStmtEof();
-if (sqlNode instanceof SqlCreateTable) {
- handleCreateTable((SqlCreateTable) sqlNode, metaStore);
-} else if (sqlNode instanceof SqlDropTable) {
- handleDropTable((SqlDropTable) sqlNode);
+if (sqlNode instanceof SqlExecutableStatement) {
+ ((SqlExecutableStatement) sqlNode).execute(env.getContext());
Review comment:
nice
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
Issue Time Tracking
---
Worklog Id: (was: 98534)
Time Spent: 12.5h (was: 12h 20m)
> Take advantage of Calcite DDL
> -
>
> Key: BEAM-4044
> URL: https://issues.apache.org/jira/browse/BEAM-4044
> Project: Beam
> Issue Type: New Feature
> Components: dsl-sql
>Reporter: Andrew Pilloud
>Assignee: Andrew Pilloud
>Priority: Major
> Time Spent: 12.5h
> Remaining Estimate: 0h
>
> In Calcite 1.15 support for abstract DDL moved into calcite core. We should
> take advantage of that.
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
[jira] [Work logged] (BEAM-4044) Take advantage of Calcite DDL
[
https://issues.apache.org/jira/browse/BEAM-4044?focusedWorklogId=98535&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-98535
]
ASF GitHub Bot logged work on BEAM-4044:
Author: ASF GitHub Bot
Created on: 04/May/18 18:14
Start Date: 04/May/18 18:14
Worklog Time Spent: 10m
Work Description: kennknowles commented on a change in pull request
#5224: [BEAM-4044] [SQL] Add tables via TableStore in Schema, execute DDL in
Calcite model
URL: https://github.com/apache/beam/pull/5224#discussion_r186170860
##
File path:
sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/QueryTransform.java
##
@@ -78,25 +76,18 @@
}
}
- private PCollectionTuple toPCollectionTuple(PInput inputs) {
-return (inputs instanceof PCollection)
-? PCollectionTuple.of(new TupleTag<>(PCOLLECTION_NAME), toRows(inputs))
-: tupleOfAllInputs(inputs.getPipeline(), inputs.expand());
- }
-
- private PCollectionTuple tupleOfAllInputs(
- Pipeline pipeline,
- Map, PValue> taggedInputs) {
-
-PCollectionTuple tuple = PCollectionTuple.empty(pipeline);
-
-for (Map.Entry, PValue> input : taggedInputs.entrySet()) {
- tuple = tuple.and(
- new TupleTag<>(input.getKey().getId()),
- toRows(input.getValue()));
+ private BeamSqlTableProvider toTableProvider(PInput inputs) {
+ImmutableMap.Builder tables = ImmutableMap.builder();
+if (inputs instanceof PCollection) {
+ tables.put(PCOLLECTION_NAME,
+ new BeamPCollectionTable(toRows(inputs)));
+} else {
+ for (Map.Entry, PValue> input : inputs.expand().entrySet()) {
+tables.put(input.getKey().getId(),
+new BeamPCollectionTable(toRows(input.getValue(;
+ }
Review comment:
Incidentally, here (or maybe it is nearby beneath a fold) seems like a good
place to (possibly redundantly) explain that a PCollection makes a single magic
table while any other kind of input uses `expand()` to make many tables using
the tags as names.
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
Issue Time Tracking
---
Worklog Id: (was: 98535)
Time Spent: 12.5h (was: 12h 20m)
> Take advantage of Calcite DDL
> -
>
> Key: BEAM-4044
> URL: https://issues.apache.org/jira/browse/BEAM-4044
> Project: Beam
> Issue Type: New Feature
> Components: dsl-sql
>Reporter: Andrew Pilloud
>Assignee: Andrew Pilloud
>Priority: Major
> Time Spent: 12.5h
> Remaining Estimate: 0h
>
> In Calcite 1.15 support for abstract DDL moved into calcite core. We should
> take advantage of that.
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
[jira] [Work logged] (BEAM-4044) Take advantage of Calcite DDL
[
https://issues.apache.org/jira/browse/BEAM-4044?focusedWorklogId=98530&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-98530
]
ASF GitHub Bot logged work on BEAM-4044:
Author: ASF GitHub Bot
Created on: 04/May/18 18:14
Start Date: 04/May/18 18:14
Worklog Time Spent: 10m
Work Description: kennknowles commented on a change in pull request
#5224: [BEAM-4044] [SQL] Add tables via TableStore in Schema, execute DDL in
Calcite model
URL: https://github.com/apache/beam/pull/5224#discussion_r186172960
##
File path:
sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamJoinRel.java
##
@@ -369,8 +370,8 @@ private Row buildNullRow(BeamRelNode relNode) {
private PCollection joinAsLookup(
BeamRelNode leftRelNode,
BeamRelNode rightRelNode,
- PCollectionTuple inputPCollections) {
-PCollection factStream =
inputPCollections.apply(leftRelNode.toPTransform());
+ PInput inputPCollections) {
+PCollection factStream =
inputPCollections.getPipeline().apply(leftRelNode.toPTransform());
Review comment:
`getPipeline().apply()` is probably not what you want here, either. It is
actually a bad method - it is the same as `getPipeline().begin().apply()` so it
always starts a new initial pipeline segment.
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
Issue Time Tracking
---
Worklog Id: (was: 98530)
Time Spent: 11h 50m (was: 11h 40m)
> Take advantage of Calcite DDL
> -
>
> Key: BEAM-4044
> URL: https://issues.apache.org/jira/browse/BEAM-4044
> Project: Beam
> Issue Type: New Feature
> Components: dsl-sql
>Reporter: Andrew Pilloud
>Assignee: Andrew Pilloud
>Priority: Major
> Time Spent: 11h 50m
> Remaining Estimate: 0h
>
> In Calcite 1.15 support for abstract DDL moved into calcite core. We should
> take advantage of that.
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
[jira] [Work logged] (BEAM-4044) Take advantage of Calcite DDL
[
https://issues.apache.org/jira/browse/BEAM-4044?focusedWorklogId=98532&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-98532
]
ASF GitHub Bot logged work on BEAM-4044:
Author: ASF GitHub Bot
Created on: 04/May/18 18:14
Start Date: 04/May/18 18:14
Worklog Time Spent: 10m
Work Description: kennknowles commented on a change in pull request
#5224: [BEAM-4044] [SQL] Add tables via TableStore in Schema, execute DDL in
Calcite model
URL: https://github.com/apache/beam/pull/5224#discussion_r186170113
##
File path:
sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/BeamSqlCli.java
##
@@ -78,45 +67,15 @@ public void execute(String sqlString) throws Exception {
BeamSqlParser parser = new BeamSqlParser(sqlString);
SqlNode sqlNode = parser.impl().parseSqlStmtEof();
-if (sqlNode instanceof SqlCreateTable) {
- handleCreateTable((SqlCreateTable) sqlNode, metaStore);
-} else if (sqlNode instanceof SqlDropTable) {
- handleDropTable((SqlDropTable) sqlNode);
+if (sqlNode instanceof SqlExecutableStatement) {
+ ((SqlExecutableStatement) sqlNode).execute(env.getContext());
} else {
PipelineOptions options = PipelineOptionsFactory.fromArgs(new String[]
{}).withValidation()
.as(PipelineOptions.class);
options.setJobName("BeamPlanCreator");
Pipeline pipeline = Pipeline.create(options);
- compilePipeline(sqlString, pipeline, env);
+ env.getPlanner().compileBeamPipeline(sqlString, pipeline);
Review comment:
nice
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
Issue Time Tracking
---
Worklog Id: (was: 98532)
Time Spent: 12h 10m (was: 12h)
> Take advantage of Calcite DDL
> -
>
> Key: BEAM-4044
> URL: https://issues.apache.org/jira/browse/BEAM-4044
> Project: Beam
> Issue Type: New Feature
> Components: dsl-sql
>Reporter: Andrew Pilloud
>Assignee: Andrew Pilloud
>Priority: Major
> Time Spent: 12h 10m
> Remaining Estimate: 0h
>
> In Calcite 1.15 support for abstract DDL moved into calcite core. We should
> take advantage of that.
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
[jira] [Work logged] (BEAM-4044) Take advantage of Calcite DDL
[
https://issues.apache.org/jira/browse/BEAM-4044?focusedWorklogId=98531&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-98531
]
ASF GitHub Bot logged work on BEAM-4044:
Author: ASF GitHub Bot
Created on: 04/May/18 18:14
Start Date: 04/May/18 18:14
Worklog Time Spent: 10m
Work Description: kennknowles commented on a change in pull request
#5224: [BEAM-4044] [SQL] Add tables via TableStore in Schema, execute DDL in
Calcite model
URL: https://github.com/apache/beam/pull/5224#discussion_r186170078
##
File path:
sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/QueryTransform.java
##
@@ -55,20 +56,17 @@
@Override
public PCollection expand(PInput input) {
-PCollectionTuple inputTuple = toPCollectionTuple(input);
-
-BeamSqlEnv sqlEnv = new BeamSqlEnv();
+BeamSqlEnv sqlEnv = new BeamSqlEnv(toTableProvider(input));
Review comment:
nice
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
Issue Time Tracking
---
Worklog Id: (was: 98531)
Time Spent: 12h (was: 11h 50m)
> Take advantage of Calcite DDL
> -
>
> Key: BEAM-4044
> URL: https://issues.apache.org/jira/browse/BEAM-4044
> Project: Beam
> Issue Type: New Feature
> Components: dsl-sql
>Reporter: Andrew Pilloud
>Assignee: Andrew Pilloud
>Priority: Major
> Time Spent: 12h
> Remaining Estimate: 0h
>
> In Calcite 1.15 support for abstract DDL moved into calcite core. We should
> take advantage of that.
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
[jira] [Work logged] (BEAM-4044) Take advantage of Calcite DDL
[
https://issues.apache.org/jira/browse/BEAM-4044?focusedWorklogId=98491&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-98491
]
ASF GitHub Bot logged work on BEAM-4044:
Author: ASF GitHub Bot
Created on: 04/May/18 16:31
Start Date: 04/May/18 16:31
Worklog Time Spent: 10m
Work Description: apilloud commented on a change in pull request #5224:
[BEAM-4044] [SQL] Add tables via TableStore in Schema, execute DDL in Calcite
model
URL: https://github.com/apache/beam/pull/5224#discussion_r186138546
##
File path:
sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/BeamCalciteSchema.java
##
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.extensions.sql.impl;
+
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Set;
+import org.apache.beam.sdk.extensions.sql.meta.Table;
+import org.apache.beam.sdk.extensions.sql.meta.provider.TableProvider;
+import org.apache.calcite.linq4j.tree.Expression;
+import org.apache.calcite.schema.Function;
+import org.apache.calcite.schema.Schema;
+import org.apache.calcite.schema.SchemaPlus;
+import org.apache.calcite.schema.SchemaVersion;
+import org.apache.calcite.schema.Schemas;
+
+/**
+ * Adapter from {@link TableProvider} to {@link Schema}.
+ */
+public class BeamCalciteSchema implements Schema {
+ private TableProvider tableProvider;
+
+ public BeamCalciteSchema(TableProvider tableProvider) {
+this.tableProvider = tableProvider;
+ }
+
+ public TableProvider getTableProvider() {
+return tableProvider;
+ }
+
+ @Override
+ public boolean isMutable() {
+return true;
+ }
+
+ @Override
+ public Schema snapshot(SchemaVersion version) {
+return this;
+ }
+
+ @Override
+ public Expression getExpression(SchemaPlus parentSchema, String name) {
+return Schemas.subSchemaExpression(parentSchema, name, getClass());
+ }
+
+ @Override
+ public Set getTableNames() {
+Set tables = new HashSet();
+for (Table table : tableProvider.listTables()) {
+ tables.add(table.getName());
+}
+return tables;
+ }
+
+ @Override
+ public org.apache.calcite.schema.Table getTable(String name) {
+for (Table table : tableProvider.listTables()) {
+ if (table.getName().equals(name)) {
+return new BeamCalciteTable(tableProvider.buildBeamSqlTable(table));
+ }
+}
Review comment:
I'm assuming this comment is on the line with `tableProvider.listTables()`
not `}`? If so, the output of that function can not be cached. I do however
agree that it makes sense to change the return value of that API to
`Map`. Simplifies a lot of code all over, so I'll do that.
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
Issue Time Tracking
---
Worklog Id: (was: 98491)
Time Spent: 11h 40m (was: 11.5h)
> Take advantage of Calcite DDL
> -
>
> Key: BEAM-4044
> URL: https://issues.apache.org/jira/browse/BEAM-4044
> Project: Beam
> Issue Type: New Feature
> Components: dsl-sql
>Reporter: Andrew Pilloud
>Assignee: Andrew Pilloud
>Priority: Major
> Time Spent: 11h 40m
> Remaining Estimate: 0h
>
> In Calcite 1.15 support for abstract DDL moved into calcite core. We should
> take advantage of that.
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
[jira] [Work logged] (BEAM-4044) Take advantage of Calcite DDL
[
https://issues.apache.org/jira/browse/BEAM-4044?focusedWorklogId=97994&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-97994
]
ASF GitHub Bot logged work on BEAM-4044:
Author: ASF GitHub Bot
Created on: 03/May/18 18:16
Start Date: 03/May/18 18:16
Worklog Time Spent: 10m
Work Description: akedin commented on a change in pull request #5224:
[BEAM-4044] [SQL] Add tables via TableStore in Schema, execute DDL in Calcite
model
URL: https://github.com/apache/beam/pull/5224#discussion_r185884947
##
File path:
sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/BeamSqlTableProvider.java
##
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.extensions.sql.meta.provider;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import org.apache.beam.sdk.extensions.sql.BeamSqlTable;
+import org.apache.beam.sdk.extensions.sql.meta.Table;
+
+/**
+ * A {@code BeamSqlTableProvider} provides read only set of {@code
BeamSqlTable}.
+ */
+public class BeamSqlTableProvider implements TableProvider {
+ private final String typeName;
+ private final Map tables;
+
+ public BeamSqlTableProvider(String typeName, Map
tables) {
+this.typeName = typeName;
+this.tables = tables;
+ }
+
+ @Override public String getTableType() {
+return typeName;
+ }
+
+ @Override
+ public void createTable(Table table) {
+throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void dropTable(String tableName) {
+throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public List listTables() {
+List list = new ArrayList(tables.size());
+for (Map.Entry table : tables.entrySet()) {
+ list.add(Table.builder()
+ .type(getTableType())
+ .name(table.getKey())
+ .columns(Collections.emptyList())
+ .build());
Review comment:
nit: I would rewrite it this way:
```java
tables
.values()
.stream()
.map(sqlTable ->
Table
.builder()
.type(getTableType())
.name(sqlTable.getKey())
.columns(Collections.emptyList())
.build())
.collect(toList());
```
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
Issue Time Tracking
---
Worklog Id: (was: 97994)
Time Spent: 10h 40m (was: 10.5h)
> Take advantage of Calcite DDL
> -
>
> Key: BEAM-4044
> URL: https://issues.apache.org/jira/browse/BEAM-4044
> Project: Beam
> Issue Type: New Feature
> Components: dsl-sql
>Reporter: Andrew Pilloud
>Assignee: Andrew Pilloud
>Priority: Major
> Time Spent: 10h 40m
> Remaining Estimate: 0h
>
> In Calcite 1.15 support for abstract DDL moved into calcite core. We should
> take advantage of that.
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
[jira] [Work logged] (BEAM-4044) Take advantage of Calcite DDL
[
https://issues.apache.org/jira/browse/BEAM-4044?focusedWorklogId=98000&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-98000
]
ASF GitHub Bot logged work on BEAM-4044:
Author: ASF GitHub Bot
Created on: 03/May/18 18:16
Start Date: 03/May/18 18:16
Worklog Time Spent: 10m
Work Description: akedin commented on a change in pull request #5224:
[BEAM-4044] [SQL] Add tables via TableStore in Schema, execute DDL in Calcite
model
URL: https://github.com/apache/beam/pull/5224#discussion_r185883258
##
File path:
sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/BeamSqlTableProvider.java
##
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.extensions.sql.meta.provider;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import org.apache.beam.sdk.extensions.sql.BeamSqlTable;
+import org.apache.beam.sdk.extensions.sql.meta.Table;
+
+/**
+ * A {@code BeamSqlTableProvider} provides read only set of {@code
BeamSqlTable}.
+ */
+public class BeamSqlTableProvider implements TableProvider {
Review comment:
make it `@AutoValue`+`Builder`?
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
Issue Time Tracking
---
Worklog Id: (was: 98000)
Time Spent: 11.5h (was: 11h 20m)
> Take advantage of Calcite DDL
> -
>
> Key: BEAM-4044
> URL: https://issues.apache.org/jira/browse/BEAM-4044
> Project: Beam
> Issue Type: New Feature
> Components: dsl-sql
>Reporter: Andrew Pilloud
>Assignee: Andrew Pilloud
>Priority: Major
> Time Spent: 11.5h
> Remaining Estimate: 0h
>
> In Calcite 1.15 support for abstract DDL moved into calcite core. We should
> take advantage of that.
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
[jira] [Work logged] (BEAM-4044) Take advantage of Calcite DDL
[
https://issues.apache.org/jira/browse/BEAM-4044?focusedWorklogId=97995&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-97995
]
ASF GitHub Bot logged work on BEAM-4044:
Author: ASF GitHub Bot
Created on: 03/May/18 18:16
Start Date: 03/May/18 18:16
Worklog Time Spent: 10m
Work Description: akedin commented on a change in pull request #5224:
[BEAM-4044] [SQL] Add tables via TableStore in Schema, execute DDL in Calcite
model
URL: https://github.com/apache/beam/pull/5224#discussion_r185889574
##
File path:
sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/BeamSqlCli.java
##
@@ -78,45 +67,15 @@ public void execute(String sqlString) throws Exception {
BeamSqlParser parser = new BeamSqlParser(sqlString);
SqlNode sqlNode = parser.impl().parseSqlStmtEof();
-if (sqlNode instanceof SqlCreateTable) {
- handleCreateTable((SqlCreateTable) sqlNode, metaStore);
-} else if (sqlNode instanceof SqlDropTable) {
- handleDropTable((SqlDropTable) sqlNode);
+if (sqlNode instanceof SqlExecutableStatement) {
+ ((SqlExecutableStatement) sqlNode).execute(env.getContext());
} else {
Review comment:
nit: add a comment what is executable statement, what is not?
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
Issue Time Tracking
---
Worklog Id: (was: 97995)
Time Spent: 10h 50m (was: 10h 40m)
> Take advantage of Calcite DDL
> -
>
> Key: BEAM-4044
> URL: https://issues.apache.org/jira/browse/BEAM-4044
> Project: Beam
> Issue Type: New Feature
> Components: dsl-sql
>Reporter: Andrew Pilloud
>Assignee: Andrew Pilloud
>Priority: Major
> Time Spent: 10h 50m
> Remaining Estimate: 0h
>
> In Calcite 1.15 support for abstract DDL moved into calcite core. We should
> take advantage of that.
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
[jira] [Work logged] (BEAM-4044) Take advantage of Calcite DDL
[
https://issues.apache.org/jira/browse/BEAM-4044?focusedWorklogId=97991&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-97991
]
ASF GitHub Bot logged work on BEAM-4044:
Author: ASF GitHub Bot
Created on: 03/May/18 18:16
Start Date: 03/May/18 18:16
Worklog Time Spent: 10m
Work Description: akedin commented on a change in pull request #5224:
[BEAM-4044] [SQL] Add tables via TableStore in Schema, execute DDL in Calcite
model
URL: https://github.com/apache/beam/pull/5224#discussion_r185892321
##
File path:
sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/store/InMemoryMetaStore.java
##
@@ -37,7 +37,8 @@
private Map tables = new HashMap<>();
private Map providers = new HashMap<>();
- public InMemoryMetaStore() {
+ @Override public String getTableType() {
+return "";
Review comment:
I think it should have its own table type still
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
Issue Time Tracking
---
Worklog Id: (was: 97991)
Time Spent: 10h 20m (was: 10h 10m)
> Take advantage of Calcite DDL
> -
>
> Key: BEAM-4044
> URL: https://issues.apache.org/jira/browse/BEAM-4044
> Project: Beam
> Issue Type: New Feature
> Components: dsl-sql
>Reporter: Andrew Pilloud
>Assignee: Andrew Pilloud
>Priority: Major
> Time Spent: 10h 20m
> Remaining Estimate: 0h
>
> In Calcite 1.15 support for abstract DDL moved into calcite core. We should
> take advantage of that.
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
[jira] [Work logged] (BEAM-4044) Take advantage of Calcite DDL
[
https://issues.apache.org/jira/browse/BEAM-4044?focusedWorklogId=97998&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-97998
]
ASF GitHub Bot logged work on BEAM-4044:
Author: ASF GitHub Bot
Created on: 03/May/18 18:16
Start Date: 03/May/18 18:16
Worklog Time Spent: 10m
Work Description: akedin commented on a change in pull request #5224:
[BEAM-4044] [SQL] Add tables via TableStore in Schema, execute DDL in Calcite
model
URL: https://github.com/apache/beam/pull/5224#discussion_r185889031
##
File path:
sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/BeamCalciteTable.java
##
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.extensions.sql.impl;
+
+import java.util.Collection;
+import java.util.List;
+import org.apache.beam.sdk.extensions.sql.BeamSqlTable;
+import org.apache.beam.sdk.extensions.sql.impl.planner.BeamQueryPlanner;
+import org.apache.beam.sdk.extensions.sql.impl.rel.BeamIOSinkRel;
+import org.apache.beam.sdk.extensions.sql.impl.rel.BeamIOSourceRel;
+import org.apache.beam.sdk.extensions.sql.impl.utils.CalciteUtils;
+import org.apache.calcite.adapter.java.AbstractQueryableTable;
+import org.apache.calcite.linq4j.QueryProvider;
+import org.apache.calcite.linq4j.Queryable;
+import org.apache.calcite.plan.RelOptCluster;
+import org.apache.calcite.plan.RelOptTable;
+import org.apache.calcite.prepare.Prepare;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.rel.core.TableModify;
+import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.rel.type.RelDataTypeFactory;
+import org.apache.calcite.rex.RexNode;
+import org.apache.calcite.schema.ModifiableTable;
+import org.apache.calcite.schema.SchemaPlus;
+import org.apache.calcite.schema.TranslatableTable;
+
+/**
+ * Adapter from {@link BeamSqlTable} to a calcite Table.
+ */
+class BeamCalciteTable extends AbstractQueryableTable
+implements ModifiableTable, TranslatableTable {
+ private BeamSqlTable beamTable;
+
+ public BeamCalciteTable(BeamSqlTable beamTable) {
+super(Object[].class);
+this.beamTable = beamTable;
+ }
+
+ @Override
+ public RelDataType getRowType(RelDataTypeFactory typeFactory) {
+return CalciteUtils.toCalciteRowType(this.beamTable.getSchema(),
BeamQueryPlanner.TYPE_FACTORY);
Review comment:
Create this in constructor?
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
Issue Time Tracking
---
Worklog Id: (was: 97998)
Time Spent: 11h 10m (was: 11h)
> Take advantage of Calcite DDL
> -
>
> Key: BEAM-4044
> URL: https://issues.apache.org/jira/browse/BEAM-4044
> Project: Beam
> Issue Type: New Feature
> Components: dsl-sql
>Reporter: Andrew Pilloud
>Assignee: Andrew Pilloud
>Priority: Major
> Time Spent: 11h 10m
> Remaining Estimate: 0h
>
> In Calcite 1.15 support for abstract DDL moved into calcite core. We should
> take advantage of that.
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
[jira] [Work logged] (BEAM-4044) Take advantage of Calcite DDL
[
https://issues.apache.org/jira/browse/BEAM-4044?focusedWorklogId=97992&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-97992
]
ASF GitHub Bot logged work on BEAM-4044:
Author: ASF GitHub Bot
Created on: 03/May/18 18:16
Start Date: 03/May/18 18:16
Worklog Time Spent: 10m
Work Description: akedin commented on a change in pull request #5224:
[BEAM-4044] [SQL] Add tables via TableStore in Schema, execute DDL in Calcite
model
URL: https://github.com/apache/beam/pull/5224#discussion_r185880448
##
File path:
sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/parser/SqlDropObject.java
##
@@ -51,8 +59,39 @@
name.unparse(writer, leftPrec, rightPrec);
}
- public String getNameSimple() {
-return name.getSimple().toLowerCase();
+ public void execute(CalcitePrepare.Context context) {
+final List path = context.getDefaultSchemaPath();
+CalciteSchema schema = context.getRootSchema();
+for (String p : path) {
+ schema = schema.getSubSchema(p, true);
+}
+final boolean existed;
+switch (getKind()) {
+case DROP_TABLE:
+case DROP_MATERIALIZED_VIEW:
Review comment:
Do we support views? If we don't have concrete plans to support them i'd
rather remove all related code
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
Issue Time Tracking
---
Worklog Id: (was: 97992)
> Take advantage of Calcite DDL
> -
>
> Key: BEAM-4044
> URL: https://issues.apache.org/jira/browse/BEAM-4044
> Project: Beam
> Issue Type: New Feature
> Components: dsl-sql
>Reporter: Andrew Pilloud
>Assignee: Andrew Pilloud
>Priority: Major
> Time Spent: 10h 20m
> Remaining Estimate: 0h
>
> In Calcite 1.15 support for abstract DDL moved into calcite core. We should
> take advantage of that.
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
[jira] [Work logged] (BEAM-4044) Take advantage of Calcite DDL
[
https://issues.apache.org/jira/browse/BEAM-4044?focusedWorklogId=97997&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-97997
]
ASF GitHub Bot logged work on BEAM-4044:
Author: ASF GitHub Bot
Created on: 03/May/18 18:16
Start Date: 03/May/18 18:16
Worklog Time Spent: 10m
Work Description: akedin commented on a change in pull request #5224:
[BEAM-4044] [SQL] Add tables via TableStore in Schema, execute DDL in Calcite
model
URL: https://github.com/apache/beam/pull/5224#discussion_r185890896
##
File path:
sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/BeamCalciteSchema.java
##
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.extensions.sql.impl;
+
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Set;
+import org.apache.beam.sdk.extensions.sql.meta.Table;
+import org.apache.beam.sdk.extensions.sql.meta.provider.TableProvider;
+import org.apache.calcite.linq4j.tree.Expression;
+import org.apache.calcite.schema.Function;
+import org.apache.calcite.schema.Schema;
+import org.apache.calcite.schema.SchemaPlus;
+import org.apache.calcite.schema.SchemaVersion;
+import org.apache.calcite.schema.Schemas;
+
+/**
+ * Adapter from {@link TableProvider} to {@link Schema}.
+ */
+public class BeamCalciteSchema implements Schema {
+ private TableProvider tableProvider;
+
+ public BeamCalciteSchema(TableProvider tableProvider) {
+this.tableProvider = tableProvider;
+ }
+
+ public TableProvider getTableProvider() {
+return tableProvider;
+ }
+
+ @Override
+ public boolean isMutable() {
+return true;
+ }
+
+ @Override
+ public Schema snapshot(SchemaVersion version) {
+return this;
+ }
+
+ @Override
+ public Expression getExpression(SchemaPlus parentSchema, String name) {
+return Schemas.subSchemaExpression(parentSchema, name, getClass());
+ }
+
+ @Override
+ public Set getTableNames() {
+Set tables = new HashSet();
+for (Table table : tableProvider.listTables()) {
+ tables.add(table.getName());
+}
+return tables;
+ }
+
+ @Override
+ public org.apache.calcite.schema.Table getTable(String name) {
+for (Table table : tableProvider.listTables()) {
+ if (table.getName().equals(name)) {
+return new BeamCalciteTable(tableProvider.buildBeamSqlTable(table));
+ }
+}
Review comment:
nit: looks like it would be better to convert this to a `Map` once in constructor, this way you wouldn't need to implement
`map.keySet()` or `map.get()`
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
Issue Time Tracking
---
Worklog Id: (was: 97997)
Time Spent: 11h (was: 10h 50m)
> Take advantage of Calcite DDL
> -
>
> Key: BEAM-4044
> URL: https://issues.apache.org/jira/browse/BEAM-4044
> Project: Beam
> Issue Type: New Feature
> Components: dsl-sql
>Reporter: Andrew Pilloud
>Assignee: Andrew Pilloud
>Priority: Major
> Time Spent: 11h
> Remaining Estimate: 0h
>
> In Calcite 1.15 support for abstract DDL moved into calcite core. We should
> take advantage of that.
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
[jira] [Work logged] (BEAM-4044) Take advantage of Calcite DDL
[
https://issues.apache.org/jira/browse/BEAM-4044?focusedWorklogId=97993&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-97993
]
ASF GitHub Bot logged work on BEAM-4044:
Author: ASF GitHub Bot
Created on: 03/May/18 18:16
Start Date: 03/May/18 18:16
Worklog Time Spent: 10m
Work Description: akedin commented on a change in pull request #5224:
[BEAM-4044] [SQL] Add tables via TableStore in Schema, execute DDL in Calcite
model
URL: https://github.com/apache/beam/pull/5224#discussion_r185888542
##
File path:
sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/QueryTransform.java
##
@@ -78,25 +76,18 @@
}
}
- private PCollectionTuple toPCollectionTuple(PInput inputs) {
-return (inputs instanceof PCollection)
-? PCollectionTuple.of(new TupleTag<>(PCOLLECTION_NAME), toRows(inputs))
-: tupleOfAllInputs(inputs.getPipeline(), inputs.expand());
- }
-
- private PCollectionTuple tupleOfAllInputs(
- Pipeline pipeline,
- Map, PValue> taggedInputs) {
-
-PCollectionTuple tuple = PCollectionTuple.empty(pipeline);
-
-for (Map.Entry, PValue> input : taggedInputs.entrySet()) {
- tuple = tuple.and(
- new TupleTag<>(input.getKey().getId()),
- toRows(input.getValue()));
+ private BeamSqlTableProvider toTableProvider(PInput inputs) {
+ImmutableMap.Builder tables = ImmutableMap.builder();
+if (inputs instanceof PCollection) {
+ tables.put(PCOLLECTION_NAME,
+ new BeamPCollectionTable(toRows(inputs)));
+} else {
+ for (Map.Entry, PValue> input : inputs.expand().entrySet()) {
+tables.put(input.getKey().getId(),
+new BeamPCollectionTable(toRows(input.getValue(;
+ }
Review comment:
nit: I would avoid stateful if/else with loops with generics, hurts
readability. Might consider extracting something like this:
```java
if (input instanceof PCollection) {
return
ImmuableMap.of(
PCOLLECTION_NAME,
new BeamPCollectionTable(toRows(inputs)))
}
return
inputs
.expand()
.entrySet()
.stream()
.collect(
toMap(
keyedPCollection -> keyedPCollection.getKey().getId(),
keyedPCollection -> keyedPCollection.getValue()))
```
and then create BeamSqlTableProvider outside
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
Issue Time Tracking
---
Worklog Id: (was: 97993)
Time Spent: 10.5h (was: 10h 20m)
> Take advantage of Calcite DDL
> -
>
> Key: BEAM-4044
> URL: https://issues.apache.org/jira/browse/BEAM-4044
> Project: Beam
> Issue Type: New Feature
> Components: dsl-sql
>Reporter: Andrew Pilloud
>Assignee: Andrew Pilloud
>Priority: Major
> Time Spent: 10.5h
> Remaining Estimate: 0h
>
> In Calcite 1.15 support for abstract DDL moved into calcite core. We should
> take advantage of that.
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
[jira] [Work logged] (BEAM-4044) Take advantage of Calcite DDL
[
https://issues.apache.org/jira/browse/BEAM-4044?focusedWorklogId=97999&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-97999
]
ASF GitHub Bot logged work on BEAM-4044:
Author: ASF GitHub Bot
Created on: 03/May/18 18:16
Start Date: 03/May/18 18:16
Worklog Time Spent: 10m
Work Description: akedin commented on a change in pull request #5224:
[BEAM-4044] [SQL] Add tables via TableStore in Schema, execute DDL in Calcite
model
URL: https://github.com/apache/beam/pull/5224#discussion_r185882943
##
File path:
sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamJoinRel.java
##
@@ -133,9 +133,10 @@ public Join copy(
.setCoder(CalciteUtils.toBeamSchema(getRowType()).getRowCoder());
}
- PCollection leftRows = inputPCollections.apply("left",
leftRelNode.toPTransform());
+ PCollection leftRows =
+ inputPCollections.getPipeline().apply("left",
leftRelNode.toPTransform());
PCollection rightRows =
- inputPCollections.apply("right", rightRelNode.toPTransform());
+ inputPCollections.getPipeline().apply("right",
rightRelNode.toPTransform());
Review comment:
Not sure if it's the right thing to access the pipeline directly. Who knows
what's there? Does it have a source so that it can produce elements? With
PCollections I would have at least an expectation that there should be elements
in it
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
Issue Time Tracking
---
Worklog Id: (was: 97999)
Time Spent: 11h 20m (was: 11h 10m)
> Take advantage of Calcite DDL
> -
>
> Key: BEAM-4044
> URL: https://issues.apache.org/jira/browse/BEAM-4044
> Project: Beam
> Issue Type: New Feature
> Components: dsl-sql
>Reporter: Andrew Pilloud
>Assignee: Andrew Pilloud
>Priority: Major
> Time Spent: 11h 20m
> Remaining Estimate: 0h
>
> In Calcite 1.15 support for abstract DDL moved into calcite core. We should
> take advantage of that.
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
[jira] [Work logged] (BEAM-4044) Take advantage of Calcite DDL
[
https://issues.apache.org/jira/browse/BEAM-4044?focusedWorklogId=97996&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-97996
]
ASF GitHub Bot logged work on BEAM-4044:
Author: ASF GitHub Bot
Created on: 03/May/18 18:16
Start Date: 03/May/18 18:16
Worklog Time Spent: 10m
Work Description: akedin commented on a change in pull request #5224:
[BEAM-4044] [SQL] Add tables via TableStore in Schema, execute DDL in Calcite
model
URL: https://github.com/apache/beam/pull/5224#discussion_r185891412
##
File path:
sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/BeamSqlEnv.java
##
@@ -103,105 +82,58 @@ public void registerUdaf(String functionName,
Combine.CombineFn combineFn) {
schema.plus().add(functionName, new UdafImpl(combineFn));
}
- /**
- * Registers {@link PCollection}s in {@link PCollectionTuple} as a tables.
- *
- * Assumes that {@link PCollection} elements are {@link Row}s.
- *
- * {@link TupleTag#getId()}s are used as table names.
- */
- public void registerPCollectionTuple(PCollectionTuple pCollectionTuple) {
-pCollectionTuple
-.getAll()
-.forEach((tag, pCollection) ->
-registerPCollection(tag.getId(), (PCollection)
pCollection));
- }
-
- /**
- * Registers {@link PCollection} of {@link Row}s as a table.
- *
- * Assumes that {@link PCollection#getCoder()} returns an instance of
{@link RowCoder}.
- */
- public void registerPCollection(String name, PCollection pCollection) {
-registerTable(name, pCollection, ((RowCoder)
pCollection.getCoder()).getSchema());
- }
-
- /**
- * Registers {@link PCollection} as a table.
- */
- public void registerTable(String tableName, PCollection pCollection,
Schema schema) {
-registerTable(tableName, new BeamPCollectionTable(pCollection, schema));
- }
-
- /**
- * Registers a {@link BaseBeamTable} which can be used for all subsequent
queries.
- */
- public void registerTable(String tableName, BeamSqlTable table) {
-schema.add(tableName, new BeamCalciteTable(table));
+ public BeamQueryPlanner getPlanner() {
+return planner;
}
- public void deregisterTable(String targetTableName) {
-schema.removeTable(targetTableName);
+ public CalcitePrepare.Context getContext() {
+return new ContextImpl();
}
- private static class BeamCalciteTable extends AbstractQueryableTable
- implements ModifiableTable, TranslatableTable {
-private BeamSqlTable beamTable;
-
-public BeamCalciteTable(BeamSqlTable beamTable) {
- super(Object[].class);
- this.beamTable = beamTable;
+ private class ContextImpl implements CalcitePrepare.Context {
+@Override
+public JavaTypeFactory getTypeFactory() {
+ throw new UnsupportedOperationException();
Review comment:
Would it be wrong to return `BeamQueryPlanner.TYPE_FACTORY`?
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
Issue Time Tracking
---
Worklog Id: (was: 97996)
Time Spent: 10h 50m (was: 10h 40m)
> Take advantage of Calcite DDL
> -
>
> Key: BEAM-4044
> URL: https://issues.apache.org/jira/browse/BEAM-4044
> Project: Beam
> Issue Type: New Feature
> Components: dsl-sql
>Reporter: Andrew Pilloud
>Assignee: Andrew Pilloud
>Priority: Major
> Time Spent: 10h 50m
> Remaining Estimate: 0h
>
> In Calcite 1.15 support for abstract DDL moved into calcite core. We should
> take advantage of that.
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
[jira] [Work logged] (BEAM-4044) Take advantage of Calcite DDL
[ https://issues.apache.org/jira/browse/BEAM-4044?focusedWorklogId=97978&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-97978 ] ASF GitHub Bot logged work on BEAM-4044: Author: ASF GitHub Bot Created on: 03/May/18 17:35 Start Date: 03/May/18 17:35 Worklog Time Spent: 10m Work Description: apilloud commented on issue #5224: [BEAM-4044] [SQL] Add tables via TableStore in Schema, execute DDL in Calcite model URL: https://github.com/apache/beam/pull/5224#issuecomment-386375614 R: @kennknowles This gets us 90% of the way to sqlline. This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org Issue Time Tracking --- Worklog Id: (was: 97978) Time Spent: 10h 10m (was: 10h) > Take advantage of Calcite DDL > - > > Key: BEAM-4044 > URL: https://issues.apache.org/jira/browse/BEAM-4044 > Project: Beam > Issue Type: New Feature > Components: dsl-sql >Reporter: Andrew Pilloud >Assignee: Andrew Pilloud >Priority: Major > Time Spent: 10h 10m > Remaining Estimate: 0h > > In Calcite 1.15 support for abstract DDL moved into calcite core. We should > take advantage of that. -- This message was sent by Atlassian JIRA (v7.6.3#76005)
[jira] [Work logged] (BEAM-4044) Take advantage of Calcite DDL
[ https://issues.apache.org/jira/browse/BEAM-4044?focusedWorklogId=97972&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-97972 ] ASF GitHub Bot logged work on BEAM-4044: Author: ASF GitHub Bot Created on: 03/May/18 17:25 Start Date: 03/May/18 17:25 Worklog Time Spent: 10m Work Description: apilloud commented on issue #5254: [BEAM-4044] [SQL] Simplify TableProvider interface URL: https://github.com/apache/beam/pull/5254#issuecomment-386372372 Merging this into mega PR #5224 This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org Issue Time Tracking --- Worklog Id: (was: 97972) Time Spent: 9h 50m (was: 9h 40m) > Take advantage of Calcite DDL > - > > Key: BEAM-4044 > URL: https://issues.apache.org/jira/browse/BEAM-4044 > Project: Beam > Issue Type: New Feature > Components: dsl-sql >Reporter: Andrew Pilloud >Assignee: Andrew Pilloud >Priority: Major > Time Spent: 9h 50m > Remaining Estimate: 0h > > In Calcite 1.15 support for abstract DDL moved into calcite core. We should > take advantage of that. -- This message was sent by Atlassian JIRA (v7.6.3#76005)
[jira] [Work logged] (BEAM-4044) Take advantage of Calcite DDL
[
https://issues.apache.org/jira/browse/BEAM-4044?focusedWorklogId=97973&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-97973
]
ASF GitHub Bot logged work on BEAM-4044:
Author: ASF GitHub Bot
Created on: 03/May/18 17:25
Start Date: 03/May/18 17:25
Worklog Time Spent: 10m
Work Description: apilloud closed pull request #5254: [BEAM-4044] [SQL]
Simplify TableProvider interface
URL: https://github.com/apache/beam/pull/5254
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git
a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/InMemoryMetaTableProvider.java
b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/InMemoryMetaTableProvider.java
new file mode 100644
index 000..bfed36132ca
--- /dev/null
+++
b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/InMemoryMetaTableProvider.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.extensions.sql.meta.provider;
+
+import java.util.Collections;
+import java.util.List;
+import org.apache.beam.sdk.extensions.sql.meta.Table;
+
+/**
+ * A {@code InMemoryMetaTableProvider} is an abstract {@code TableProvider}
for in-memory types.
+ */
+public abstract class InMemoryMetaTableProvider implements TableProvider {
+
+ @Override
+ public void createTable(Table table) {
+// No-op
+ }
+
+ @Override
+ public void dropTable(String tableName) {
+// No-op
+ }
+
+ @Override
+ public List listTables() {
+return Collections.emptyList();
+ }
+}
diff --git
a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/TableProvider.java
b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/TableProvider.java
index 5bbadb13eeb..e52021cada9 100644
---
a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/TableProvider.java
+++
b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/TableProvider.java
@@ -29,11 +29,6 @@
* handle MySQL based tables, a provider to handle Casandra based tables etc.
*/
public interface TableProvider {
- /**
- * Init the provider.
- */
- void init();
-
/**
* Gets the table type this provider handles.
*/
@@ -60,9 +55,4 @@
* Build a {@link BeamSqlTable} using the given table meta info.
*/
BeamSqlTable buildBeamSqlTable(Table table);
-
- /**
- * Close the provider.
- */
- void close();
}
diff --git
a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/bigquery/BigQueryTableProvider.java
b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/bigquery/BigQueryTableProvider.java
index dd728d0faeb..21483af63dc 100644
---
a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/bigquery/BigQueryTableProvider.java
+++
b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/bigquery/BigQueryTableProvider.java
@@ -20,11 +20,9 @@
import static
org.apache.beam.sdk.extensions.sql.meta.provider.MetaUtils.getRowTypeFromTable;
-import java.util.Collections;
-import java.util.List;
import org.apache.beam.sdk.extensions.sql.BeamSqlTable;
import org.apache.beam.sdk.extensions.sql.meta.Table;
-import org.apache.beam.sdk.extensions.sql.meta.provider.TableProvider;
+import
org.apache.beam.sdk.extensions.sql.meta.provider.InMemoryMetaTableProvider;
import org.apache.beam.sdk.schemas.Schema;
/**
@@ -41,7 +39,7 @@
* LOCATION '[PROJECT_ID]:[DATASET].[TABLE]'
* }
*/
-public class BigQueryTableProvider implements TableProvider {
+public class BigQueryTableProvider extends InMemoryMetaTableProvider {
@Override public String getTableType()
[jira] [Work logged] (BEAM-4044) Take advantage of Calcite DDL
[
https://issues.apache.org/jira/browse/BEAM-4044?focusedWorklogId=97708&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-97708
]
ASF GitHub Bot logged work on BEAM-4044:
Author: ASF GitHub Bot
Created on: 02/May/18 22:40
Start Date: 02/May/18 22:40
Worklog Time Spent: 10m
Work Description: apilloud commented on a change in pull request #5254:
[BEAM-4044] [SQL] Simplify TableProvider interface
URL: https://github.com/apache/beam/pull/5254#discussion_r185659851
##
File path:
sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/bigquery/BigQueryTableProvider.java
##
@@ -41,7 +39,7 @@
* LOCATION '[PROJECT_ID]:[DATASET].[TABLE]'
* }
*/
-public class BigQueryTableProvider implements TableProvider {
+public class BigQueryTableProvider extends InMemoryTableProvider {
Review comment:
Ok, I've renamed this to `InMemoryMetaTableProvider`.
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
Issue Time Tracking
---
Worklog Id: (was: 97708)
Time Spent: 9h 40m (was: 9.5h)
> Take advantage of Calcite DDL
> -
>
> Key: BEAM-4044
> URL: https://issues.apache.org/jira/browse/BEAM-4044
> Project: Beam
> Issue Type: New Feature
> Components: dsl-sql
>Reporter: Andrew Pilloud
>Assignee: Andrew Pilloud
>Priority: Major
> Time Spent: 9h 40m
> Remaining Estimate: 0h
>
> In Calcite 1.15 support for abstract DDL moved into calcite core. We should
> take advantage of that.
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
[jira] [Work logged] (BEAM-4044) Take advantage of Calcite DDL
[
https://issues.apache.org/jira/browse/BEAM-4044?focusedWorklogId=97181&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-97181
]
ASF GitHub Bot logged work on BEAM-4044:
Author: ASF GitHub Bot
Created on: 01/May/18 18:47
Start Date: 01/May/18 18:47
Worklog Time Spent: 10m
Work Description: apilloud commented on a change in pull request #5254:
[BEAM-4044] [SQL] Simplify TableProvider interface
URL: https://github.com/apache/beam/pull/5254#discussion_r185300879
##
File path:
sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/bigquery/BigQueryTableProvider.java
##
@@ -41,7 +39,7 @@
* LOCATION '[PROJECT_ID]:[DATASET].[TABLE]'
* }
*/
-public class BigQueryTableProvider implements TableProvider {
+public class BigQueryTableProvider extends InMemoryTableProvider {
Review comment:
It is going to be in-memory in the sense that the metadata is in-memory
only. I'm happy to change the name to something less confusing. Do you have any
suggestions?
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
Issue Time Tracking
---
Worklog Id: (was: 97181)
Time Spent: 9.5h (was: 9h 20m)
> Take advantage of Calcite DDL
> -
>
> Key: BEAM-4044
> URL: https://issues.apache.org/jira/browse/BEAM-4044
> Project: Beam
> Issue Type: New Feature
> Components: dsl-sql
>Reporter: Andrew Pilloud
>Assignee: Andrew Pilloud
>Priority: Major
> Time Spent: 9.5h
> Remaining Estimate: 0h
>
> In Calcite 1.15 support for abstract DDL moved into calcite core. We should
> take advantage of that.
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
[jira] [Work logged] (BEAM-4044) Take advantage of Calcite DDL
[
https://issues.apache.org/jira/browse/BEAM-4044?focusedWorklogId=97176&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-97176
]
ASF GitHub Bot logged work on BEAM-4044:
Author: ASF GitHub Bot
Created on: 01/May/18 18:23
Start Date: 01/May/18 18:23
Worklog Time Spent: 10m
Work Description: akedin commented on a change in pull request #5254:
[BEAM-4044] [SQL] Simplify TableProvider interface
URL: https://github.com/apache/beam/pull/5254#discussion_r185294687
##
File path:
sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/bigquery/BigQueryTableProvider.java
##
@@ -41,7 +39,7 @@
* LOCATION '[PROJECT_ID]:[DATASET].[TABLE]'
* }
*/
-public class BigQueryTableProvider implements TableProvider {
+public class BigQueryTableProvider extends InMemoryTableProvider {
Review comment:
it reads as if BigQuery table (with all data) is going to be in-memory, is
it the case?
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
Issue Time Tracking
---
Worklog Id: (was: 97176)
Time Spent: 9h 20m (was: 9h 10m)
> Take advantage of Calcite DDL
> -
>
> Key: BEAM-4044
> URL: https://issues.apache.org/jira/browse/BEAM-4044
> Project: Beam
> Issue Type: New Feature
> Components: dsl-sql
>Reporter: Andrew Pilloud
>Assignee: Andrew Pilloud
>Priority: Major
> Time Spent: 9h 20m
> Remaining Estimate: 0h
>
> In Calcite 1.15 support for abstract DDL moved into calcite core. We should
> take advantage of that.
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
[jira] [Work logged] (BEAM-4044) Take advantage of Calcite DDL
[ https://issues.apache.org/jira/browse/BEAM-4044?focusedWorklogId=97172&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-97172 ] ASF GitHub Bot logged work on BEAM-4044: Author: ASF GitHub Bot Created on: 01/May/18 18:17 Start Date: 01/May/18 18:17 Worklog Time Spent: 10m Work Description: apilloud commented on issue #5254: [BEAM-4044] [SQL] Simplify TableProvider interface URL: https://github.com/apache/beam/pull/5254#issuecomment-385746088 R: @kennknowles This is some cleanup you requested in #5220. Should make it clear why dropTable is a no-op. cc: @akedin This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org Issue Time Tracking --- Worklog Id: (was: 97172) Time Spent: 9h 10m (was: 9h) > Take advantage of Calcite DDL > - > > Key: BEAM-4044 > URL: https://issues.apache.org/jira/browse/BEAM-4044 > Project: Beam > Issue Type: New Feature > Components: dsl-sql >Reporter: Andrew Pilloud >Assignee: Andrew Pilloud >Priority: Major > Time Spent: 9h 10m > Remaining Estimate: 0h > > In Calcite 1.15 support for abstract DDL moved into calcite core. We should > take advantage of that. -- This message was sent by Atlassian JIRA (v7.6.3#76005)
[jira] [Work logged] (BEAM-4044) Take advantage of Calcite DDL
[ https://issues.apache.org/jira/browse/BEAM-4044?focusedWorklogId=97171&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-97171 ] ASF GitHub Bot logged work on BEAM-4044: Author: ASF GitHub Bot Created on: 01/May/18 18:15 Start Date: 01/May/18 18:15 Worklog Time Spent: 10m Work Description: apilloud opened a new pull request #5254: [BEAM-4044] [SQL] Simplify TableProvider interface URL: https://github.com/apache/beam/pull/5254 This should make it clear that tables are in memory. Also deleted some unused functions from the interface. Follow this checklist to help us incorporate your contribution quickly and easily: - [X] Make sure there is a [JIRA issue](https://issues.apache.org/jira/projects/BEAM/issues/) filed for the change (usually before you start working on it). Trivial changes like typos do not require a JIRA issue. Your pull request should address just this issue, without pulling in other changes. - [X] Format the pull request title like `[BEAM-XXX] Fixes bug in ApproximateQuantiles`, where you replace `BEAM-XXX` with the appropriate JIRA issue. - [X] Write a pull request description that is detailed enough to understand: - [X] What the pull request does - [X] Why it does it - [X] How it does it - [X] Why this approach - [X] Each commit in the pull request should have a meaningful subject line and body. - [X] Run `./gradlew build` to make sure basic checks pass. A more thorough check will be performed on your pull request automatically. - [ ] If this contribution is large, please file an Apache [Individual Contributor License Agreement](https://www.apache.org/licenses/icla.pdf). This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org Issue Time Tracking --- Worklog Id: (was: 97171) Time Spent: 9h (was: 8h 50m) > Take advantage of Calcite DDL > - > > Key: BEAM-4044 > URL: https://issues.apache.org/jira/browse/BEAM-4044 > Project: Beam > Issue Type: New Feature > Components: dsl-sql >Reporter: Andrew Pilloud >Assignee: Andrew Pilloud >Priority: Major > Time Spent: 9h > Remaining Estimate: 0h > > In Calcite 1.15 support for abstract DDL moved into calcite core. We should > take advantage of that. -- This message was sent by Atlassian JIRA (v7.6.3#76005)
[jira] [Work logged] (BEAM-4044) Take advantage of Calcite DDL
[ https://issues.apache.org/jira/browse/BEAM-4044?focusedWorklogId=95297&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-95297 ] ASF GitHub Bot logged work on BEAM-4044: Author: ASF GitHub Bot Created on: 25/Apr/18 23:12 Start Date: 25/Apr/18 23:12 Worklog Time Spent: 10m Work Description: apilloud commented on issue #5224: [BEAM-4044] [SQL] Cleanout unneeded sqlEnv URL: https://github.com/apache/beam/pull/5224#issuecomment-384462294 R: @XuMingmin @xumingming This is a trivial change, but it does change the public interface around BeamQueryPlanner. This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org Issue Time Tracking --- Worklog Id: (was: 95297) Time Spent: 8h 50m (was: 8h 40m) > Take advantage of Calcite DDL > - > > Key: BEAM-4044 > URL: https://issues.apache.org/jira/browse/BEAM-4044 > Project: Beam > Issue Type: New Feature > Components: dsl-sql >Reporter: Andrew Pilloud >Assignee: Andrew Pilloud >Priority: Major > Time Spent: 8h 50m > Remaining Estimate: 0h > > In Calcite 1.15 support for abstract DDL moved into calcite core. We should > take advantage of that. -- This message was sent by Atlassian JIRA (v7.6.3#76005)
[jira] [Work logged] (BEAM-4044) Take advantage of Calcite DDL
[ https://issues.apache.org/jira/browse/BEAM-4044?focusedWorklogId=95201&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-95201 ] ASF GitHub Bot logged work on BEAM-4044: Author: ASF GitHub Bot Created on: 25/Apr/18 19:41 Start Date: 25/Apr/18 19:41 Worklog Time Spent: 10m Work Description: akedin commented on issue #5224: [BEAM-4044] [SQL] Cleanout unneeded sqlEnv URL: https://github.com/apache/beam/pull/5224#issuecomment-384409940 LGTM This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org Issue Time Tracking --- Worklog Id: (was: 95201) Time Spent: 8h 40m (was: 8.5h) > Take advantage of Calcite DDL > - > > Key: BEAM-4044 > URL: https://issues.apache.org/jira/browse/BEAM-4044 > Project: Beam > Issue Type: New Feature > Components: dsl-sql >Reporter: Andrew Pilloud >Assignee: Andrew Pilloud >Priority: Major > Time Spent: 8h 40m > Remaining Estimate: 0h > > In Calcite 1.15 support for abstract DDL moved into calcite core. We should > take advantage of that. -- This message was sent by Atlassian JIRA (v7.6.3#76005)
[jira] [Work logged] (BEAM-4044) Take advantage of Calcite DDL
[ https://issues.apache.org/jira/browse/BEAM-4044?focusedWorklogId=95113&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-95113 ] ASF GitHub Bot logged work on BEAM-4044: Author: ASF GitHub Bot Created on: 25/Apr/18 17:06 Start Date: 25/Apr/18 17:06 Worklog Time Spent: 10m Work Description: apilloud opened a new pull request #5224: [BEAM-4044] [SQL] Cleanout unneeded sqlEnv URL: https://github.com/apache/beam/pull/5224 This is a followup to #5154. It has no functional changes, but it does change tests and public APIs. Follow this checklist to help us incorporate your contribution quickly and easily: - [X] Make sure there is a [JIRA issue](https://issues.apache.org/jira/projects/BEAM/issues/) filed for the change (usually before you start working on it). Trivial changes like typos do not require a JIRA issue. Your pull request should address just this issue, without pulling in other changes. - [X] Format the pull request title like `[BEAM-XXX] Fixes bug in ApproximateQuantiles`, where you replace `BEAM-XXX` with the appropriate JIRA issue. - [X] Write a pull request description that is detailed enough to understand: - [X] What the pull request does - [X] Why it does it - [X] How it does it - [X] Why this approach - [X] Each commit in the pull request should have a meaningful subject line and body. - [ ] Run `./gradlew build` to make sure basic checks pass. A more thorough check will be performed on your pull request automatically. - [ ] If this contribution is large, please file an Apache [Individual Contributor License Agreement](https://www.apache.org/licenses/icla.pdf). This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org Issue Time Tracking --- Worklog Id: (was: 95113) Time Spent: 8.5h (was: 8h 20m) > Take advantage of Calcite DDL > - > > Key: BEAM-4044 > URL: https://issues.apache.org/jira/browse/BEAM-4044 > Project: Beam > Issue Type: New Feature > Components: dsl-sql >Reporter: Andrew Pilloud >Assignee: Andrew Pilloud >Priority: Major > Time Spent: 8.5h > Remaining Estimate: 0h > > In Calcite 1.15 support for abstract DDL moved into calcite core. We should > take advantage of that. -- This message was sent by Atlassian JIRA (v7.6.3#76005)
[jira] [Work logged] (BEAM-4044) Take advantage of Calcite DDL
[
https://issues.apache.org/jira/browse/BEAM-4044?focusedWorklogId=94439&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-94439
]
ASF GitHub Bot logged work on BEAM-4044:
Author: ASF GitHub Bot
Created on: 24/Apr/18 03:35
Start Date: 24/Apr/18 03:35
Worklog Time Spent: 10m
Work Description: kennknowles closed pull request #5154: [BEAM-4044]
[SQL] Make BeamCalciteTable self planning
URL: https://github.com/apache/beam/pull/5154
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git
a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/BeamSqlEnv.java
b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/BeamSqlEnv.java
index 7f77668379c..eeaf36e7af3 100644
---
a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/BeamSqlEnv.java
+++
b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/BeamSqlEnv.java
@@ -20,8 +20,8 @@
import java.io.IOException;
import java.io.ObjectInputStream;
import java.io.Serializable;
-import java.util.HashMap;
-import java.util.Map;
+import java.util.Collection;
+import java.util.List;
import org.apache.beam.sdk.coders.RowCoder;
import org.apache.beam.sdk.extensions.sql.BeamSql;
import org.apache.beam.sdk.extensions.sql.BeamSqlCli;
@@ -29,6 +29,8 @@
import org.apache.beam.sdk.extensions.sql.BeamSqlUdf;
import org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.UdafImpl;
import org.apache.beam.sdk.extensions.sql.impl.planner.BeamQueryPlanner;
+import org.apache.beam.sdk.extensions.sql.impl.rel.BeamIOSinkRel;
+import org.apache.beam.sdk.extensions.sql.impl.rel.BeamIOSourceRel;
import org.apache.beam.sdk.extensions.sql.impl.schema.BaseBeamTable;
import org.apache.beam.sdk.extensions.sql.impl.schema.BeamPCollectionTable;
import org.apache.beam.sdk.extensions.sql.impl.utils.CalciteUtils;
@@ -39,19 +41,22 @@
import org.apache.beam.sdk.values.PCollectionTuple;
import org.apache.beam.sdk.values.Row;
import org.apache.beam.sdk.values.TupleTag;
-import org.apache.calcite.DataContext;
-import org.apache.calcite.config.CalciteConnectionConfig;
-import org.apache.calcite.linq4j.Enumerable;
+import org.apache.calcite.adapter.java.AbstractQueryableTable;
+import org.apache.calcite.jdbc.CalciteSchema;
+import org.apache.calcite.linq4j.QueryProvider;
+import org.apache.calcite.linq4j.Queryable;
+import org.apache.calcite.plan.RelOptCluster;
+import org.apache.calcite.plan.RelOptTable;
+import org.apache.calcite.prepare.Prepare;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.rel.core.TableModify;
import org.apache.calcite.rel.type.RelDataType;
import org.apache.calcite.rel.type.RelDataTypeFactory;
-import org.apache.calcite.schema.ScannableTable;
+import org.apache.calcite.rex.RexNode;
+import org.apache.calcite.schema.ModifiableTable;
import org.apache.calcite.schema.SchemaPlus;
-import org.apache.calcite.schema.Statistic;
-import org.apache.calcite.schema.Statistics;
+import org.apache.calcite.schema.TranslatableTable;
import org.apache.calcite.schema.impl.ScalarFunctionImpl;
-import org.apache.calcite.sql.SqlCall;
-import org.apache.calcite.sql.SqlNode;
-import org.apache.calcite.tools.Frameworks;
/**
* {@link BeamSqlEnv} prepares the execution context for {@link BeamSql} and
{@link BeamSqlCli}.
@@ -60,21 +65,19 @@
* {@link BeamQueryPlanner} which parse/validate/optimize/translate input SQL
queries.
*/
public class BeamSqlEnv implements Serializable {
- transient SchemaPlus schema;
+ transient CalciteSchema schema;
transient BeamQueryPlanner planner;
- transient Map tables;
public BeamSqlEnv() {
-tables = new HashMap<>(16);
-schema = Frameworks.createRootSchema(true);
-planner = new BeamQueryPlanner(this, schema);
+schema = CalciteSchema.createRootSchema(true);
+planner = new BeamQueryPlanner(this, schema.plus());
}
/**
* Register a UDF function which can be used in SQL expression.
*/
public void registerUdf(String functionName, Class clazz, String method) {
-schema.add(functionName, ScalarFunctionImpl.create(clazz, method));
+schema.plus().add(functionName, ScalarFunctionImpl.create(clazz, method));
}
/**
@@ -97,7 +100,7 @@ public void registerUdf(String functionName,
SerializableFunction sfn) {
* See {@link org.apache.beam.sdk.transforms.Combine.CombineFn} on how to
implement a UDAF.
*/
public void registerUdaf(String functionName, Combine.CombineFn combineFn) {
-schema.add(functionName, new UdafImpl(combineFn));
+schema.plus().add(functionName, new UdafImpl(combineFn));
}
/
[jira] [Work logged] (BEAM-4044) Take advantage of Calcite DDL
[ https://issues.apache.org/jira/browse/BEAM-4044?focusedWorklogId=94204&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-94204 ] ASF GitHub Bot logged work on BEAM-4044: Author: ASF GitHub Bot Created on: 23/Apr/18 17:49 Start Date: 23/Apr/18 17:49 Worklog Time Spent: 10m Work Description: akedin commented on issue #5154: [BEAM-4044] [SQL] Make BeamCalciteTable self planning URL: https://github.com/apache/beam/pull/5154#issuecomment-383663560 LGTM This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org Issue Time Tracking --- Worklog Id: (was: 94204) Time Spent: 8h 10m (was: 8h) > Take advantage of Calcite DDL > - > > Key: BEAM-4044 > URL: https://issues.apache.org/jira/browse/BEAM-4044 > Project: Beam > Issue Type: New Feature > Components: dsl-sql >Reporter: Andrew Pilloud >Assignee: Andrew Pilloud >Priority: Major > Time Spent: 8h 10m > Remaining Estimate: 0h > > In Calcite 1.15 support for abstract DDL moved into calcite core. We should > take advantage of that. -- This message was sent by Atlassian JIRA (v7.6.3#76005)
[jira] [Work logged] (BEAM-4044) Take advantage of Calcite DDL
[ https://issues.apache.org/jira/browse/BEAM-4044?focusedWorklogId=94191&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-94191 ] ASF GitHub Bot logged work on BEAM-4044: Author: ASF GitHub Bot Created on: 23/Apr/18 17:38 Start Date: 23/Apr/18 17:38 Worklog Time Spent: 10m Work Description: apilloud commented on issue #5154: [BEAM-4044] [SQL] Make BeamCalciteTable self planning URL: https://github.com/apache/beam/pull/5154#issuecomment-383164913 run java precommit This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org Issue Time Tracking --- Worklog Id: (was: 94191) Time Spent: 8h (was: 7h 50m) > Take advantage of Calcite DDL > - > > Key: BEAM-4044 > URL: https://issues.apache.org/jira/browse/BEAM-4044 > Project: Beam > Issue Type: New Feature > Components: dsl-sql >Reporter: Andrew Pilloud >Assignee: Andrew Pilloud >Priority: Major > Time Spent: 8h > Remaining Estimate: 0h > > In Calcite 1.15 support for abstract DDL moved into calcite core. We should > take advantage of that. -- This message was sent by Atlassian JIRA (v7.6.3#76005)
[jira] [Work logged] (BEAM-4044) Take advantage of Calcite DDL
[ https://issues.apache.org/jira/browse/BEAM-4044?focusedWorklogId=94188&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-94188 ] ASF GitHub Bot logged work on BEAM-4044: Author: ASF GitHub Bot Created on: 23/Apr/18 17:37 Start Date: 23/Apr/18 17:37 Worklog Time Spent: 10m Work Description: apilloud commented on issue #5154: [BEAM-4044] [SQL] Make BeamCalciteTable self planning URL: https://github.com/apache/beam/pull/5154#issuecomment-383237065 run java precommit This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org Issue Time Tracking --- Worklog Id: (was: 94188) Time Spent: 7.5h (was: 7h 20m) > Take advantage of Calcite DDL > - > > Key: BEAM-4044 > URL: https://issues.apache.org/jira/browse/BEAM-4044 > Project: Beam > Issue Type: New Feature > Components: dsl-sql >Reporter: Andrew Pilloud >Assignee: Andrew Pilloud >Priority: Major > Time Spent: 7.5h > Remaining Estimate: 0h > > In Calcite 1.15 support for abstract DDL moved into calcite core. We should > take advantage of that. -- This message was sent by Atlassian JIRA (v7.6.3#76005)
[jira] [Work logged] (BEAM-4044) Take advantage of Calcite DDL
[ https://issues.apache.org/jira/browse/BEAM-4044?focusedWorklogId=94190&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-94190 ] ASF GitHub Bot logged work on BEAM-4044: Author: ASF GitHub Bot Created on: 23/Apr/18 17:37 Start Date: 23/Apr/18 17:37 Worklog Time Spent: 10m Work Description: apilloud commented on issue #5154: [BEAM-4044] [SQL] Make BeamCalciteTable self planning URL: https://github.com/apache/beam/pull/5154#issuecomment-383168674 run java precommit This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org Issue Time Tracking --- Worklog Id: (was: 94190) Time Spent: 7h 50m (was: 7h 40m) > Take advantage of Calcite DDL > - > > Key: BEAM-4044 > URL: https://issues.apache.org/jira/browse/BEAM-4044 > Project: Beam > Issue Type: New Feature > Components: dsl-sql >Reporter: Andrew Pilloud >Assignee: Andrew Pilloud >Priority: Major > Time Spent: 7h 50m > Remaining Estimate: 0h > > In Calcite 1.15 support for abstract DDL moved into calcite core. We should > take advantage of that. -- This message was sent by Atlassian JIRA (v7.6.3#76005)
[jira] [Work logged] (BEAM-4044) Take advantage of Calcite DDL
[ https://issues.apache.org/jira/browse/BEAM-4044?focusedWorklogId=94189&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-94189 ] ASF GitHub Bot logged work on BEAM-4044: Author: ASF GitHub Bot Created on: 23/Apr/18 17:37 Start Date: 23/Apr/18 17:37 Worklog Time Spent: 10m Work Description: apilloud commented on issue #5154: [BEAM-4044] [SQL] Make BeamCalciteTable self planning URL: https://github.com/apache/beam/pull/5154#issuecomment-383183640 run java precommit This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org Issue Time Tracking --- Worklog Id: (was: 94189) Time Spent: 7h 40m (was: 7.5h) > Take advantage of Calcite DDL > - > > Key: BEAM-4044 > URL: https://issues.apache.org/jira/browse/BEAM-4044 > Project: Beam > Issue Type: New Feature > Components: dsl-sql >Reporter: Andrew Pilloud >Assignee: Andrew Pilloud >Priority: Major > Time Spent: 7h 40m > Remaining Estimate: 0h > > In Calcite 1.15 support for abstract DDL moved into calcite core. We should > take advantage of that. -- This message was sent by Atlassian JIRA (v7.6.3#76005)
[jira] [Work logged] (BEAM-4044) Take advantage of Calcite DDL
[ https://issues.apache.org/jira/browse/BEAM-4044?focusedWorklogId=93457&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-93457 ] ASF GitHub Bot logged work on BEAM-4044: Author: ASF GitHub Bot Created on: 20/Apr/18 22:19 Start Date: 20/Apr/18 22:19 Worklog Time Spent: 10m Work Description: apilloud commented on issue #5154: [BEAM-4044] [SQL] Make BeamCalciteTable self planning URL: https://github.com/apache/beam/pull/5154#issuecomment-383237065 run java precommit This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org Issue Time Tracking --- Worklog Id: (was: 93457) Time Spent: 7h 20m (was: 7h 10m) > Take advantage of Calcite DDL > - > > Key: BEAM-4044 > URL: https://issues.apache.org/jira/browse/BEAM-4044 > Project: Beam > Issue Type: New Feature > Components: dsl-sql >Reporter: Andrew Pilloud >Assignee: Andrew Pilloud >Priority: Major > Time Spent: 7h 20m > Remaining Estimate: 0h > > In Calcite 1.15 support for abstract DDL moved into calcite core. We should > take advantage of that. -- This message was sent by Atlassian JIRA (v7.6.3#76005)
[jira] [Work logged] (BEAM-4044) Take advantage of Calcite DDL
[ https://issues.apache.org/jira/browse/BEAM-4044?focusedWorklogId=93450&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-93450 ] ASF GitHub Bot logged work on BEAM-4044: Author: ASF GitHub Bot Created on: 20/Apr/18 21:23 Start Date: 20/Apr/18 21:23 Worklog Time Spent: 10m Work Description: apilloud commented on issue #5154: [BEAM-4044] [SQL] Make BeamCalciteTable self planning URL: https://github.com/apache/beam/pull/5154#issuecomment-383226481 I dropped the Serializable from the table class, which should fix it. The error message suggests adding readObject which makes other code odities make more sense. This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org Issue Time Tracking --- Worklog Id: (was: 93450) Time Spent: 7h 10m (was: 7h) > Take advantage of Calcite DDL > - > > Key: BEAM-4044 > URL: https://issues.apache.org/jira/browse/BEAM-4044 > Project: Beam > Issue Type: New Feature > Components: dsl-sql >Reporter: Andrew Pilloud >Assignee: Andrew Pilloud >Priority: Major > Time Spent: 7h 10m > Remaining Estimate: 0h > > In Calcite 1.15 support for abstract DDL moved into calcite core. We should > take advantage of that. -- This message was sent by Atlassian JIRA (v7.6.3#76005)
[jira] [Work logged] (BEAM-4044) Take advantage of Calcite DDL
[ https://issues.apache.org/jira/browse/BEAM-4044?focusedWorklogId=93449&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-93449 ] ASF GitHub Bot logged work on BEAM-4044: Author: ASF GitHub Bot Created on: 20/Apr/18 21:19 Start Date: 20/Apr/18 21:19 Worklog Time Spent: 10m Work Description: kennknowles commented on issue #5154: [BEAM-4044] [SQL] Make BeamCalciteTable self planning URL: https://github.com/apache/beam/pull/5154#issuecomment-383225644 Personally, I think findbugs is not worthwhile once we have Errorprone or Checker. So anything expedient to get past this is fine by me. This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org Issue Time Tracking --- Worklog Id: (was: 93449) Time Spent: 7h (was: 6h 50m) > Take advantage of Calcite DDL > - > > Key: BEAM-4044 > URL: https://issues.apache.org/jira/browse/BEAM-4044 > Project: Beam > Issue Type: New Feature > Components: dsl-sql >Reporter: Andrew Pilloud >Assignee: Andrew Pilloud >Priority: Major > Time Spent: 7h > Remaining Estimate: 0h > > In Calcite 1.15 support for abstract DDL moved into calcite core. We should > take advantage of that. -- This message was sent by Atlassian JIRA (v7.6.3#76005)
[jira] [Work logged] (BEAM-4044) Take advantage of Calcite DDL
[ https://issues.apache.org/jira/browse/BEAM-4044?focusedWorklogId=93371&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-93371 ] ASF GitHub Bot logged work on BEAM-4044: Author: ASF GitHub Bot Created on: 20/Apr/18 18:56 Start Date: 20/Apr/18 18:56 Worklog Time Spent: 10m Work Description: apilloud commented on issue #5154: [BEAM-4044] [SQL] Make BeamCalciteTable self planning URL: https://github.com/apache/beam/pull/5154#issuecomment-383190573 The set of findbugs tests run by maven and gradle are not the same. This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org Issue Time Tracking --- Worklog Id: (was: 93371) Time Spent: 6h 50m (was: 6h 40m) > Take advantage of Calcite DDL > - > > Key: BEAM-4044 > URL: https://issues.apache.org/jira/browse/BEAM-4044 > Project: Beam > Issue Type: New Feature > Components: dsl-sql >Reporter: Andrew Pilloud >Assignee: Andrew Pilloud >Priority: Major > Time Spent: 6h 50m > Remaining Estimate: 0h > > In Calcite 1.15 support for abstract DDL moved into calcite core. We should > take advantage of that. -- This message was sent by Atlassian JIRA (v7.6.3#76005)
[jira] [Work logged] (BEAM-4044) Take advantage of Calcite DDL
[ https://issues.apache.org/jira/browse/BEAM-4044?focusedWorklogId=93308&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-93308 ] ASF GitHub Bot logged work on BEAM-4044: Author: ASF GitHub Bot Created on: 20/Apr/18 17:35 Start Date: 20/Apr/18 17:35 Worklog Time Spent: 10m Work Description: apilloud commented on issue #5154: [BEAM-4044] [SQL] Make BeamCalciteTable self planning URL: https://github.com/apache/beam/pull/5154#issuecomment-383168674 run java precommit This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org Issue Time Tracking --- Worklog Id: (was: 93308) Time Spent: 6h 40m (was: 6.5h) > Take advantage of Calcite DDL > - > > Key: BEAM-4044 > URL: https://issues.apache.org/jira/browse/BEAM-4044 > Project: Beam > Issue Type: New Feature > Components: dsl-sql >Reporter: Andrew Pilloud >Assignee: Andrew Pilloud >Priority: Major > Time Spent: 6h 40m > Remaining Estimate: 0h > > In Calcite 1.15 support for abstract DDL moved into calcite core. We should > take advantage of that. -- This message was sent by Atlassian JIRA (v7.6.3#76005)
[jira] [Work logged] (BEAM-4044) Take advantage of Calcite DDL
[ https://issues.apache.org/jira/browse/BEAM-4044?focusedWorklogId=93298&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-93298 ] ASF GitHub Bot logged work on BEAM-4044: Author: ASF GitHub Bot Created on: 20/Apr/18 17:21 Start Date: 20/Apr/18 17:21 Worklog Time Spent: 10m Work Description: apilloud commented on issue #5154: [BEAM-4044] [SQL] Make BeamCalciteTable self planning URL: https://github.com/apache/beam/pull/5154#issuecomment-383164913 run java precommit This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org Issue Time Tracking --- Worklog Id: (was: 93298) Time Spent: 6.5h (was: 6h 20m) > Take advantage of Calcite DDL > - > > Key: BEAM-4044 > URL: https://issues.apache.org/jira/browse/BEAM-4044 > Project: Beam > Issue Type: New Feature > Components: dsl-sql >Reporter: Andrew Pilloud >Assignee: Andrew Pilloud >Priority: Major > Time Spent: 6.5h > Remaining Estimate: 0h > > In Calcite 1.15 support for abstract DDL moved into calcite core. We should > take advantage of that. -- This message was sent by Atlassian JIRA (v7.6.3#76005)
[jira] [Work logged] (BEAM-4044) Take advantage of Calcite DDL
[
https://issues.apache.org/jira/browse/BEAM-4044?focusedWorklogId=93263&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-93263
]
ASF GitHub Bot logged work on BEAM-4044:
Author: ASF GitHub Bot
Created on: 20/Apr/18 16:05
Start Date: 20/Apr/18 16:05
Worklog Time Spent: 10m
Work Description: apilloud commented on a change in pull request #5154:
[BEAM-4044] [SQL] Make BeamCalciteTable self planning
URL: https://github.com/apache/beam/pull/5154#discussion_r183098485
##
File path:
sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamIOSinkRel.java
##
@@ -20,61 +20,81 @@
import com.google.common.base.Joiner;
import java.util.List;
import org.apache.beam.sdk.extensions.sql.BeamSqlTable;
-import org.apache.beam.sdk.extensions.sql.impl.BeamSqlEnv;
+import org.apache.beam.sdk.extensions.sql.impl.rule.BeamIOSinkRule;
import org.apache.beam.sdk.transforms.PTransform;
import org.apache.beam.sdk.values.PCollection;
import org.apache.beam.sdk.values.PCollectionTuple;
import org.apache.beam.sdk.values.Row;
import org.apache.calcite.plan.RelOptCluster;
+import org.apache.calcite.plan.RelOptPlanner;
import org.apache.calcite.plan.RelOptTable;
import org.apache.calcite.plan.RelTraitSet;
import org.apache.calcite.prepare.Prepare;
import org.apache.calcite.rel.RelNode;
import org.apache.calcite.rel.core.TableModify;
import org.apache.calcite.rex.RexNode;
+import org.apache.calcite.sql2rel.RelStructuredTypeFlattener;
/** BeamRelNode to replace a {@code TableModify} node. */
-public class BeamIOSinkRel extends TableModify implements BeamRelNode {
+public class BeamIOSinkRel extends TableModify
+implements BeamRelNode, RelStructuredTypeFlattener.SelfFlatteningRel {
- private final BeamSqlEnv sqlEnv;
+ private final BeamSqlTable sqlTable;
+ private boolean isFlattening = false;
public BeamIOSinkRel(
- BeamSqlEnv sqlEnv,
RelOptCluster cluster,
- RelTraitSet traits,
RelOptTable table,
Prepare.CatalogReader catalogReader,
RelNode child,
Operation operation,
List updateColumnList,
List sourceExpressionList,
- boolean flattened) {
+ boolean flattened,
+ BeamSqlTable sqlTable) {
super(
cluster,
-traits,
+cluster.traitSetOf(BeamLogicalConvention.INSTANCE),
table,
catalogReader,
child,
operation,
updateColumnList,
sourceExpressionList,
flattened);
-this.sqlEnv = sqlEnv;
+this.sqlTable = sqlTable;
}
@Override
public RelNode copy(RelTraitSet traitSet, List inputs) {
-return new BeamIOSinkRel(
-sqlEnv,
+boolean flattened = isFlattened() || isFlattening;
+BeamIOSinkRel newRel = new BeamIOSinkRel(
getCluster(),
-traitSet,
getTable(),
getCatalogReader(),
sole(inputs),
getOperation(),
getUpdateColumnList(),
getSourceExpressionList(),
-isFlattened());
+flattened,
+sqlTable);
+newRel.traitSet = traitSet;
+return newRel;
+ }
+
+ @Override
+ public void flattenRel(RelStructuredTypeFlattener flattener) {
+// rewriteGeneric calls this.copy. Setting isFlattining passes
+// this context into copy for modificaiton of the flattened flag.
Review comment:
Fixed! Thanks.
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
Issue Time Tracking
---
Worklog Id: (was: 93263)
Time Spent: 6h 20m (was: 6h 10m)
> Take advantage of Calcite DDL
> -
>
> Key: BEAM-4044
> URL: https://issues.apache.org/jira/browse/BEAM-4044
> Project: Beam
> Issue Type: New Feature
> Components: dsl-sql
>Reporter: Andrew Pilloud
>Assignee: Andrew Pilloud
>Priority: Major
> Time Spent: 6h 20m
> Remaining Estimate: 0h
>
> In Calcite 1.15 support for abstract DDL moved into calcite core. We should
> take advantage of that.
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
[jira] [Work logged] (BEAM-4044) Take advantage of Calcite DDL
[
https://issues.apache.org/jira/browse/BEAM-4044?focusedWorklogId=93068&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-93068
]
ASF GitHub Bot logged work on BEAM-4044:
Author: ASF GitHub Bot
Created on: 20/Apr/18 04:25
Start Date: 20/Apr/18 04:25
Worklog Time Spent: 10m
Work Description: kennknowles commented on a change in pull request
#5154: [BEAM-4044] [SQL] Make BeamCalciteTable self planning
URL: https://github.com/apache/beam/pull/5154#discussion_r182943163
##
File path:
sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamIOSinkRel.java
##
@@ -20,61 +20,81 @@
import com.google.common.base.Joiner;
import java.util.List;
import org.apache.beam.sdk.extensions.sql.BeamSqlTable;
-import org.apache.beam.sdk.extensions.sql.impl.BeamSqlEnv;
+import org.apache.beam.sdk.extensions.sql.impl.rule.BeamIOSinkRule;
import org.apache.beam.sdk.transforms.PTransform;
import org.apache.beam.sdk.values.PCollection;
import org.apache.beam.sdk.values.PCollectionTuple;
import org.apache.beam.sdk.values.Row;
import org.apache.calcite.plan.RelOptCluster;
+import org.apache.calcite.plan.RelOptPlanner;
import org.apache.calcite.plan.RelOptTable;
import org.apache.calcite.plan.RelTraitSet;
import org.apache.calcite.prepare.Prepare;
import org.apache.calcite.rel.RelNode;
import org.apache.calcite.rel.core.TableModify;
import org.apache.calcite.rex.RexNode;
+import org.apache.calcite.sql2rel.RelStructuredTypeFlattener;
/** BeamRelNode to replace a {@code TableModify} node. */
-public class BeamIOSinkRel extends TableModify implements BeamRelNode {
+public class BeamIOSinkRel extends TableModify
+implements BeamRelNode, RelStructuredTypeFlattener.SelfFlatteningRel {
- private final BeamSqlEnv sqlEnv;
+ private final BeamSqlTable sqlTable;
+ private boolean isFlattening = false;
public BeamIOSinkRel(
- BeamSqlEnv sqlEnv,
RelOptCluster cluster,
- RelTraitSet traits,
RelOptTable table,
Prepare.CatalogReader catalogReader,
RelNode child,
Operation operation,
List updateColumnList,
List sourceExpressionList,
- boolean flattened) {
+ boolean flattened,
+ BeamSqlTable sqlTable) {
super(
cluster,
-traits,
+cluster.traitSetOf(BeamLogicalConvention.INSTANCE),
table,
catalogReader,
child,
operation,
updateColumnList,
sourceExpressionList,
flattened);
-this.sqlEnv = sqlEnv;
+this.sqlTable = sqlTable;
}
@Override
public RelNode copy(RelTraitSet traitSet, List inputs) {
-return new BeamIOSinkRel(
-sqlEnv,
+boolean flattened = isFlattened() || isFlattening;
+BeamIOSinkRel newRel = new BeamIOSinkRel(
getCluster(),
-traitSet,
getTable(),
getCatalogReader(),
sole(inputs),
getOperation(),
getUpdateColumnList(),
getSourceExpressionList(),
-isFlattened());
+flattened,
+sqlTable);
+newRel.traitSet = traitSet;
+return newRel;
+ }
+
+ @Override
+ public void flattenRel(RelStructuredTypeFlattener flattener) {
+// rewriteGeneric calls this.copy. Setting isFlattining passes
+// this context into copy for modificaiton of the flattened flag.
Review comment:
`modification`
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
Issue Time Tracking
---
Worklog Id: (was: 93068)
Time Spent: 6h 10m (was: 6h)
> Take advantage of Calcite DDL
> -
>
> Key: BEAM-4044
> URL: https://issues.apache.org/jira/browse/BEAM-4044
> Project: Beam
> Issue Type: New Feature
> Components: dsl-sql
>Reporter: Andrew Pilloud
>Assignee: Andrew Pilloud
>Priority: Major
> Time Spent: 6h 10m
> Remaining Estimate: 0h
>
> In Calcite 1.15 support for abstract DDL moved into calcite core. We should
> take advantage of that.
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
[jira] [Work logged] (BEAM-4044) Take advantage of Calcite DDL
[ https://issues.apache.org/jira/browse/BEAM-4044?focusedWorklogId=92961&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-92961 ] ASF GitHub Bot logged work on BEAM-4044: Author: ASF GitHub Bot Created on: 20/Apr/18 00:16 Start Date: 20/Apr/18 00:16 Worklog Time Spent: 10m Work Description: apilloud commented on issue #5154: [BEAM-4044] [SQL] Make BeamCalciteTable self planning URL: https://github.com/apache/beam/pull/5154#issuecomment-382921290 run java precommit This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org Issue Time Tracking --- Worklog Id: (was: 92961) Time Spent: 6h (was: 5h 50m) > Take advantage of Calcite DDL > - > > Key: BEAM-4044 > URL: https://issues.apache.org/jira/browse/BEAM-4044 > Project: Beam > Issue Type: New Feature > Components: dsl-sql >Reporter: Andrew Pilloud >Assignee: Andrew Pilloud >Priority: Major > Time Spent: 6h > Remaining Estimate: 0h > > In Calcite 1.15 support for abstract DDL moved into calcite core. We should > take advantage of that. -- This message was sent by Atlassian JIRA (v7.6.3#76005)
[jira] [Work logged] (BEAM-4044) Take advantage of Calcite DDL
[ https://issues.apache.org/jira/browse/BEAM-4044?focusedWorklogId=92887&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-92887 ] ASF GitHub Bot logged work on BEAM-4044: Author: ASF GitHub Bot Created on: 19/Apr/18 22:35 Start Date: 19/Apr/18 22:35 Worklog Time Spent: 10m Work Description: apilloud commented on issue #5154: [BEAM-4044] [SQL] Make BeamCalciteTable self planning URL: https://github.com/apache/beam/pull/5154#issuecomment-382901428 run java precommit This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org Issue Time Tracking --- Worklog Id: (was: 92887) Time Spent: 5h 50m (was: 5h 40m) > Take advantage of Calcite DDL > - > > Key: BEAM-4044 > URL: https://issues.apache.org/jira/browse/BEAM-4044 > Project: Beam > Issue Type: New Feature > Components: dsl-sql >Reporter: Andrew Pilloud >Assignee: Andrew Pilloud >Priority: Major > Time Spent: 5h 50m > Remaining Estimate: 0h > > In Calcite 1.15 support for abstract DDL moved into calcite core. We should > take advantage of that. -- This message was sent by Atlassian JIRA (v7.6.3#76005)
[jira] [Work logged] (BEAM-4044) Take advantage of Calcite DDL
[ https://issues.apache.org/jira/browse/BEAM-4044?focusedWorklogId=92688&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-92688 ] ASF GitHub Bot logged work on BEAM-4044: Author: ASF GitHub Bot Created on: 19/Apr/18 17:06 Start Date: 19/Apr/18 17:06 Worklog Time Spent: 10m Work Description: apilloud commented on issue #5154: [BEAM-4044] [SQL] Make BeamCalciteTable self planning URL: https://github.com/apache/beam/pull/5154#issuecomment-382810452 run java precommit This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org Issue Time Tracking --- Worklog Id: (was: 92688) Time Spent: 5h 40m (was: 5.5h) > Take advantage of Calcite DDL > - > > Key: BEAM-4044 > URL: https://issues.apache.org/jira/browse/BEAM-4044 > Project: Beam > Issue Type: New Feature > Components: dsl-sql >Reporter: Andrew Pilloud >Assignee: Andrew Pilloud >Priority: Major > Time Spent: 5h 40m > Remaining Estimate: 0h > > In Calcite 1.15 support for abstract DDL moved into calcite core. We should > take advantage of that. -- This message was sent by Atlassian JIRA (v7.6.3#76005)
[jira] [Work logged] (BEAM-4044) Take advantage of Calcite DDL
[ https://issues.apache.org/jira/browse/BEAM-4044?focusedWorklogId=92646&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-92646 ] ASF GitHub Bot logged work on BEAM-4044: Author: ASF GitHub Bot Created on: 19/Apr/18 15:58 Start Date: 19/Apr/18 15:58 Worklog Time Spent: 10m Work Description: apilloud commented on issue #5154: [BEAM-4044] [SQL] Make BeamCalciteTable self planning URL: https://github.com/apache/beam/pull/5154#issuecomment-382789150 run java precommit This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org Issue Time Tracking --- Worklog Id: (was: 92646) Time Spent: 5.5h (was: 5h 20m) > Take advantage of Calcite DDL > - > > Key: BEAM-4044 > URL: https://issues.apache.org/jira/browse/BEAM-4044 > Project: Beam > Issue Type: New Feature > Components: dsl-sql >Reporter: Andrew Pilloud >Assignee: Andrew Pilloud >Priority: Major > Time Spent: 5.5h > Remaining Estimate: 0h > > In Calcite 1.15 support for abstract DDL moved into calcite core. We should > take advantage of that. -- This message was sent by Atlassian JIRA (v7.6.3#76005)
[jira] [Work logged] (BEAM-4044) Take advantage of Calcite DDL
[ https://issues.apache.org/jira/browse/BEAM-4044?focusedWorklogId=92605&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-92605 ] ASF GitHub Bot logged work on BEAM-4044: Author: ASF GitHub Bot Created on: 19/Apr/18 15:06 Start Date: 19/Apr/18 15:06 Worklog Time Spent: 10m Work Description: apilloud commented on issue #5154: [BEAM-4044] [SQL] Make BeamCalciteTable self planning URL: https://github.com/apache/beam/pull/5154#issuecomment-382771277 run java precommit This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org Issue Time Tracking --- Worklog Id: (was: 92605) Time Spent: 5h 20m (was: 5h 10m) > Take advantage of Calcite DDL > - > > Key: BEAM-4044 > URL: https://issues.apache.org/jira/browse/BEAM-4044 > Project: Beam > Issue Type: New Feature > Components: dsl-sql >Reporter: Andrew Pilloud >Assignee: Andrew Pilloud >Priority: Major > Time Spent: 5h 20m > Remaining Estimate: 0h > > In Calcite 1.15 support for abstract DDL moved into calcite core. We should > take advantage of that. -- This message was sent by Atlassian JIRA (v7.6.3#76005)
[jira] [Work logged] (BEAM-4044) Take advantage of Calcite DDL
[ https://issues.apache.org/jira/browse/BEAM-4044?focusedWorklogId=92284&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-92284 ] ASF GitHub Bot logged work on BEAM-4044: Author: ASF GitHub Bot Created on: 18/Apr/18 23:04 Start Date: 18/Apr/18 23:04 Worklog Time Spent: 10m Work Description: apilloud commented on issue #5154: [BEAM-4044] [SQL] Make BeamCalciteTable self planning URL: https://github.com/apache/beam/pull/5154#issuecomment-382556305 run java precommit This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org Issue Time Tracking --- Worklog Id: (was: 92284) Time Spent: 5h 10m (was: 5h) > Take advantage of Calcite DDL > - > > Key: BEAM-4044 > URL: https://issues.apache.org/jira/browse/BEAM-4044 > Project: Beam > Issue Type: New Feature > Components: dsl-sql >Reporter: Andrew Pilloud >Assignee: Andrew Pilloud >Priority: Major > Time Spent: 5h 10m > Remaining Estimate: 0h > > In Calcite 1.15 support for abstract DDL moved into calcite core. We should > take advantage of that. -- This message was sent by Atlassian JIRA (v7.6.3#76005)
[jira] [Work logged] (BEAM-4044) Take advantage of Calcite DDL
[ https://issues.apache.org/jira/browse/BEAM-4044?focusedWorklogId=92283&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-92283 ] ASF GitHub Bot logged work on BEAM-4044: Author: ASF GitHub Bot Created on: 18/Apr/18 23:04 Start Date: 18/Apr/18 23:04 Worklog Time Spent: 10m Work Description: apilloud commented on issue #5154: [BEAM-4044] [SQL] Make BeamCalciteTable self planning URL: https://github.com/apache/beam/pull/5154#issuecomment-382553227 run java precommit This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org Issue Time Tracking --- Worklog Id: (was: 92283) Time Spent: 5h (was: 4h 50m) > Take advantage of Calcite DDL > - > > Key: BEAM-4044 > URL: https://issues.apache.org/jira/browse/BEAM-4044 > Project: Beam > Issue Type: New Feature > Components: dsl-sql >Reporter: Andrew Pilloud >Assignee: Andrew Pilloud >Priority: Major > Time Spent: 5h > Remaining Estimate: 0h > > In Calcite 1.15 support for abstract DDL moved into calcite core. We should > take advantage of that. -- This message was sent by Atlassian JIRA (v7.6.3#76005)
[jira] [Work logged] (BEAM-4044) Take advantage of Calcite DDL
[ https://issues.apache.org/jira/browse/BEAM-4044?focusedWorklogId=92282&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-92282 ] ASF GitHub Bot logged work on BEAM-4044: Author: ASF GitHub Bot Created on: 18/Apr/18 23:03 Start Date: 18/Apr/18 23:03 Worklog Time Spent: 10m Work Description: apilloud commented on issue #5154: [BEAM-4044] [SQL] Make BeamCalciteTable self planning URL: https://github.com/apache/beam/pull/5154#issuecomment-382556305 run java precommit This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org Issue Time Tracking --- Worklog Id: (was: 92282) Time Spent: 4h 50m (was: 4h 40m) > Take advantage of Calcite DDL > - > > Key: BEAM-4044 > URL: https://issues.apache.org/jira/browse/BEAM-4044 > Project: Beam > Issue Type: New Feature > Components: dsl-sql >Reporter: Andrew Pilloud >Assignee: Andrew Pilloud >Priority: Major > Time Spent: 4h 50m > Remaining Estimate: 0h > > In Calcite 1.15 support for abstract DDL moved into calcite core. We should > take advantage of that. -- This message was sent by Atlassian JIRA (v7.6.3#76005)
[jira] [Work logged] (BEAM-4044) Take advantage of Calcite DDL
[ https://issues.apache.org/jira/browse/BEAM-4044?focusedWorklogId=92267&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-92267 ] ASF GitHub Bot logged work on BEAM-4044: Author: ASF GitHub Bot Created on: 18/Apr/18 22:48 Start Date: 18/Apr/18 22:48 Worklog Time Spent: 10m Work Description: apilloud commented on issue #5154: [BEAM-4044] [SQL] Make BeamCalciteTable self planning URL: https://github.com/apache/beam/pull/5154#issuecomment-382553227 run java precommit This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org Issue Time Tracking --- Worklog Id: (was: 92267) Time Spent: 4h 40m (was: 4.5h) > Take advantage of Calcite DDL > - > > Key: BEAM-4044 > URL: https://issues.apache.org/jira/browse/BEAM-4044 > Project: Beam > Issue Type: New Feature > Components: dsl-sql >Reporter: Andrew Pilloud >Assignee: Andrew Pilloud >Priority: Major > Time Spent: 4h 40m > Remaining Estimate: 0h > > In Calcite 1.15 support for abstract DDL moved into calcite core. We should > take advantage of that. -- This message was sent by Atlassian JIRA (v7.6.3#76005)
[jira] [Work logged] (BEAM-4044) Take advantage of Calcite DDL
[ https://issues.apache.org/jira/browse/BEAM-4044?focusedWorklogId=92159&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-92159 ] ASF GitHub Bot logged work on BEAM-4044: Author: ASF GitHub Bot Created on: 18/Apr/18 16:33 Start Date: 18/Apr/18 16:33 Worklog Time Spent: 10m Work Description: apilloud commented on issue #5154: [BEAM-4044] [SQL] Make BeamCalciteTable self planning URL: https://github.com/apache/beam/pull/5154#issuecomment-382449051 run java precommit This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org Issue Time Tracking --- Worklog Id: (was: 92159) Time Spent: 4.5h (was: 4h 20m) > Take advantage of Calcite DDL > - > > Key: BEAM-4044 > URL: https://issues.apache.org/jira/browse/BEAM-4044 > Project: Beam > Issue Type: New Feature > Components: dsl-sql >Reporter: Andrew Pilloud >Assignee: Andrew Pilloud >Priority: Major > Time Spent: 4.5h > Remaining Estimate: 0h > > In Calcite 1.15 support for abstract DDL moved into calcite core. We should > take advantage of that. -- This message was sent by Atlassian JIRA (v7.6.3#76005)
[jira] [Work logged] (BEAM-4044) Take advantage of Calcite DDL
[ https://issues.apache.org/jira/browse/BEAM-4044?focusedWorklogId=92158&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-92158 ] ASF GitHub Bot logged work on BEAM-4044: Author: ASF GitHub Bot Created on: 18/Apr/18 16:33 Start Date: 18/Apr/18 16:33 Worklog Time Spent: 10m Work Description: apilloud commented on issue #5154: [BEAM-4044] [SQL] Make BeamCalciteTable self planning URL: https://github.com/apache/beam/pull/5154#issuecomment-382449051 run java precommit This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org Issue Time Tracking --- Worklog Id: (was: 92158) Time Spent: 4h 20m (was: 4h 10m) > Take advantage of Calcite DDL > - > > Key: BEAM-4044 > URL: https://issues.apache.org/jira/browse/BEAM-4044 > Project: Beam > Issue Type: New Feature > Components: dsl-sql >Reporter: Andrew Pilloud >Assignee: Andrew Pilloud >Priority: Major > Time Spent: 4h 20m > Remaining Estimate: 0h > > In Calcite 1.15 support for abstract DDL moved into calcite core. We should > take advantage of that. -- This message was sent by Atlassian JIRA (v7.6.3#76005)
[jira] [Work logged] (BEAM-4044) Take advantage of Calcite DDL
[ https://issues.apache.org/jira/browse/BEAM-4044?focusedWorklogId=91821&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-91821 ] ASF GitHub Bot logged work on BEAM-4044: Author: ASF GitHub Bot Created on: 17/Apr/18 16:57 Start Date: 17/Apr/18 16:57 Worklog Time Spent: 10m Work Description: apilloud commented on issue #5154: [BEAM-4044] [SQL] Make BeamCalciteTable self planning URL: https://github.com/apache/beam/pull/5154#issuecomment-382066494 R: @kennknowles This is a refactor PR contained entirely in `sql/impl`. cc: @XuMingmin This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org Issue Time Tracking --- Worklog Id: (was: 91821) Time Spent: 4h 10m (was: 4h) > Take advantage of Calcite DDL > - > > Key: BEAM-4044 > URL: https://issues.apache.org/jira/browse/BEAM-4044 > Project: Beam > Issue Type: New Feature > Components: dsl-sql >Reporter: Andrew Pilloud >Assignee: Andrew Pilloud >Priority: Major > Time Spent: 4h 10m > Remaining Estimate: 0h > > In Calcite 1.15 support for abstract DDL moved into calcite core. We should > take advantage of that. -- This message was sent by Atlassian JIRA (v7.6.3#76005)
[jira] [Work logged] (BEAM-4044) Take advantage of Calcite DDL
[ https://issues.apache.org/jira/browse/BEAM-4044?focusedWorklogId=91819&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-91819 ] ASF GitHub Bot logged work on BEAM-4044: Author: ASF GitHub Bot Created on: 17/Apr/18 16:52 Start Date: 17/Apr/18 16:52 Worklog Time Spent: 10m Work Description: apilloud opened a new pull request #5154: [BEAM-4044] [SQL] Make BeamCalciteTable self planning URL: https://github.com/apache/beam/pull/5154 This is some cleanup moving us towards only using the calcite environment. This is required for calcite's builtin DDL support. There is no test changes, functional changes, or public API changes. The core nugget of this change is that the planner rules no longer require access to the BeamSqlEnv. Follow this checklist to help us incorporate your contribution quickly and easily: - [X] Make sure there is a [JIRA issue](https://issues.apache.org/jira/projects/BEAM/issues/) filed for the change (usually before you start working on it). Trivial changes like typos do not require a JIRA issue. Your pull request should address just this issue, without pulling in other changes. - [X] Format the pull request title like `[BEAM-XXX] Fixes bug in ApproximateQuantiles`, where you replace `BEAM-XXX` with the appropriate JIRA issue. - [X] Write a pull request description that is detailed enough to understand: - [X] What the pull request does - [X] Why it does it - [X] How it does it - [X] Why this approach - [X] Each commit in the pull request should have a meaningful subject line and body. - [X] Run `mvn clean verify` to make sure basic checks pass. A more thorough check will be performed on your pull request automatically. - [ ] If this contribution is large, please file an Apache [Individual Contributor License Agreement](https://www.apache.org/licenses/icla.pdf). This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org Issue Time Tracking --- Worklog Id: (was: 91819) Time Spent: 4h (was: 3h 50m) > Take advantage of Calcite DDL > - > > Key: BEAM-4044 > URL: https://issues.apache.org/jira/browse/BEAM-4044 > Project: Beam > Issue Type: New Feature > Components: dsl-sql >Reporter: Andrew Pilloud >Assignee: Andrew Pilloud >Priority: Major > Time Spent: 4h > Remaining Estimate: 0h > > In Calcite 1.15 support for abstract DDL moved into calcite core. We should > take advantage of that. -- This message was sent by Atlassian JIRA (v7.6.3#76005)
[jira] [Work logged] (BEAM-4044) Take advantage of Calcite DDL
[
https://issues.apache.org/jira/browse/BEAM-4044?focusedWorklogId=91592&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-91592
]
ASF GitHub Bot logged work on BEAM-4044:
Author: ASF GitHub Bot
Created on: 17/Apr/18 04:13
Start Date: 17/Apr/18 04:13
Worklog Time Spent: 10m
Work Description: XuMingmin closed pull request #5040: [BEAM-4044] [SQL]
Refresh DDL from 1.16
URL: https://github.com/apache/beam/pull/5040
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/sdks/java/extensions/sql/src/main/codegen/config.fmpp
b/sdks/java/extensions/sql/src/main/codegen/config.fmpp
index 61645e29f55..5ecb3d53fe4 100644
--- a/sdks/java/extensions/sql/src/main/codegen/config.fmpp
+++ b/sdks/java/extensions/sql/src/main/codegen/config.fmpp
@@ -1,10 +1,9 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to you under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
@@ -15,9 +14,79 @@
# limitations under the License.
data: {
- parser: tdd(../data/Parser.tdd)
-}
+parser: {
+ # Generated parser implementation class package and name
+ package: "org.apache.beam.sdk.extensions.sql.impl.parser.impl",
+ class: "BeamSqlParserImpl",
+
+ # List of import statements.
+ imports: [
+"org.apache.calcite.schema.ColumnStrategy"
+"org.apache.calcite.sql.SqlCreate"
+"org.apache.calcite.sql.SqlDrop"
+"org.apache.beam.sdk.extensions.sql.impl.parser.SqlDdlNodes"
+ ]
+
+ # List of keywords.
+ keywords: [
+"COMMENT"
+"IF"
+ "LOCATION"
+ "TBLPROPERTIES"
+ ]
+
+ # List of keywords from "keywords" section that are not reserved.
+ nonReservedKeywords: [
+"COMMENT"
+"IF"
+ "LOCATION"
+ "TBLPROPERTIES"
+ ]
+
+ # List of methods for parsing custom SQL statements.
+ statementParserMethods: [
+ ]
+
+ # List of methods for parsing custom literals.
+ # Example: ParseJsonLiteral().
+ literalParserMethods: [
+ ]
+
+ # List of methods for parsing custom data types.
+ dataTypeParserMethods: [
+ ]
+ # List of methods for parsing extensions to "ALTER " calls.
+ # Each must accept arguments "(SqlParserPos pos, String scope)".
+ alterStatementParserMethods: [
+ ]
+
+ # List of methods for parsing extensions to "CREATE [OR REPLACE]" calls.
+ # Each must accept arguments "(SqlParserPos pos, boolean replace)".
+ createStatementParserMethods: [
+"SqlCreateTable"
+ ]
+
+ # List of methods for parsing extensions to "DROP" calls.
+ # Each must accept arguments "(SqlParserPos pos)".
+ dropStatementParserMethods: [
+"SqlDropTable"
+ ]
+
+ # List of files in @includes directory that have parser method
+ # implementations for parsing custom SQL statements, literals or types
+ # given as part of "statementParserMethods", "literalParserMethods" or
+ # "dataTypeParserMethods".
+ implementationFiles: [
+"parserImpls.ftl"
+ ]
+
+ includeCompoundIdentifier: true
+ includeBraces: true
+ includeAdditionalDeclarations: false
+
+}
+}
freemarkerLinks: {
- includes: includes/
+includes: includes/
}
diff --git a/sdks/java/extensions/sql/src/main/codegen/data/Parser.tdd
b/sdks/java/extensions/sql/src/main/codegen/data/Parser.tdd
deleted file mode 100644
index 1afa73d255b..000
--- a/sdks/java/extensions/sql/src/main/codegen/data/Parser.tdd
+++ /dev/null
@@ -1,76 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not u
[jira] [Work logged] (BEAM-4044) Take advantage of Calcite DDL
[ https://issues.apache.org/jira/browse/BEAM-4044?focusedWorklogId=91524&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-91524 ] ASF GitHub Bot logged work on BEAM-4044: Author: ASF GitHub Bot Created on: 16/Apr/18 22:13 Start Date: 16/Apr/18 22:13 Worklog Time Spent: 10m Work Description: apilloud commented on issue #5040: [BEAM-4044] [SQL] Refresh DDL from 1.16 URL: https://github.com/apache/beam/pull/5040#issuecomment-381767118 I've made the requested changes and this is green. Can I get someone to merge? This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org Issue Time Tracking --- Worklog Id: (was: 91524) Time Spent: 3h 40m (was: 3.5h) > Take advantage of Calcite DDL > - > > Key: BEAM-4044 > URL: https://issues.apache.org/jira/browse/BEAM-4044 > Project: Beam > Issue Type: New Feature > Components: dsl-sql >Reporter: Andrew Pilloud >Assignee: Andrew Pilloud >Priority: Major > Time Spent: 3h 40m > Remaining Estimate: 0h > > In Calcite 1.15 support for abstract DDL moved into calcite core. We should > take advantage of that. -- This message was sent by Atlassian JIRA (v7.6.3#76005)
[jira] [Work logged] (BEAM-4044) Take advantage of Calcite DDL
[ https://issues.apache.org/jira/browse/BEAM-4044?focusedWorklogId=90577&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-90577 ] ASF GitHub Bot logged work on BEAM-4044: Author: ASF GitHub Bot Created on: 12/Apr/18 21:00 Start Date: 12/Apr/18 21:00 Worklog Time Spent: 10m Work Description: apilloud commented on issue #5040: [BEAM-4044] [SQL] Refresh DDL from 1.16 URL: https://github.com/apache/beam/pull/5040#issuecomment-380942823 run java precommit This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org Issue Time Tracking --- Worklog Id: (was: 90577) Time Spent: 3.5h (was: 3h 20m) > Take advantage of Calcite DDL > - > > Key: BEAM-4044 > URL: https://issues.apache.org/jira/browse/BEAM-4044 > Project: Beam > Issue Type: New Feature > Components: dsl-sql >Reporter: Andrew Pilloud >Assignee: Andrew Pilloud >Priority: Major > Time Spent: 3.5h > Remaining Estimate: 0h > > In Calcite 1.15 support for abstract DDL moved into calcite core. We should > take advantage of that. -- This message was sent by Atlassian JIRA (v7.6.3#76005)
[jira] [Work logged] (BEAM-4044) Take advantage of Calcite DDL
[ https://issues.apache.org/jira/browse/BEAM-4044?focusedWorklogId=90550&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-90550 ] ASF GitHub Bot logged work on BEAM-4044: Author: ASF GitHub Bot Created on: 12/Apr/18 19:56 Start Date: 12/Apr/18 19:56 Worklog Time Spent: 10m Work Description: apilloud commented on issue #5040: [BEAM-4044] [SQL] Refresh DDL from 1.16 URL: https://github.com/apache/beam/pull/5040#issuecomment-380925898 This is rebased and I changed the copyright message. This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org Issue Time Tracking --- Worklog Id: (was: 90550) Time Spent: 3h 20m (was: 3h 10m) > Take advantage of Calcite DDL > - > > Key: BEAM-4044 > URL: https://issues.apache.org/jira/browse/BEAM-4044 > Project: Beam > Issue Type: New Feature > Components: dsl-sql >Reporter: Andrew Pilloud >Assignee: Andrew Pilloud >Priority: Major > Time Spent: 3h 20m > Remaining Estimate: 0h > > In Calcite 1.15 support for abstract DDL moved into calcite core. We should > take advantage of that. -- This message was sent by Atlassian JIRA (v7.6.3#76005)
[jira] [Work logged] (BEAM-4044) Take advantage of Calcite DDL
[ https://issues.apache.org/jira/browse/BEAM-4044?focusedWorklogId=90534&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-90534 ] ASF GitHub Bot logged work on BEAM-4044: Author: ASF GitHub Bot Created on: 12/Apr/18 18:34 Start Date: 12/Apr/18 18:34 Worklog Time Spent: 10m Work Description: XuMingmin commented on issue #5040: [BEAM-4044] [SQL] Refresh DDL from 1.16 URL: https://github.com/apache/beam/pull/5040#issuecomment-380903084 can you rebase the PR as some files are also changed by #4991 This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org Issue Time Tracking --- Worklog Id: (was: 90534) Time Spent: 3h 10m (was: 3h) > Take advantage of Calcite DDL > - > > Key: BEAM-4044 > URL: https://issues.apache.org/jira/browse/BEAM-4044 > Project: Beam > Issue Type: New Feature > Components: dsl-sql >Reporter: Andrew Pilloud >Assignee: Andrew Pilloud >Priority: Major > Time Spent: 3h 10m > Remaining Estimate: 0h > > In Calcite 1.15 support for abstract DDL moved into calcite core. We should > take advantage of that. -- This message was sent by Atlassian JIRA (v7.6.3#76005)
[jira] [Work logged] (BEAM-4044) Take advantage of Calcite DDL
[ https://issues.apache.org/jira/browse/BEAM-4044?focusedWorklogId=90530&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-90530 ] ASF GitHub Bot logged work on BEAM-4044: Author: ASF GitHub Bot Created on: 12/Apr/18 18:30 Start Date: 12/Apr/18 18:30 Worklog Time Spent: 10m Work Description: XuMingmin commented on a change in pull request #5040: [BEAM-4044] [SQL] Refresh DDL from 1.16 URL: https://github.com/apache/beam/pull/5040#discussion_r181179993 ## File path: sdks/java/extensions/sql/src/main/codegen/includes/parserImpls.ftl ## @@ -1,106 +1,162 @@ -<#-- Licensed to the Apache Software Foundation (ASF) under one or more contributor - license agreements. See the NOTICE file distributed with this work for additional - information regarding copyright ownership. The ASF licenses this file to - You under the Apache License, Version 2.0 (the "License"); you may not use - this file except in compliance with the License. You may obtain a copy of - the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required - by applicable law or agreed to in writing, software distributed under the - License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS - OF ANY KIND, either express or implied. See the License for the specific - language governing permissions and limitations under the License. --> +<#-- +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to you under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with Review comment: I remember there was a checkstyle error before, and James exclude the file and comment it. @apilloud could you revert these lines? This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org Issue Time Tracking --- Worklog Id: (was: 90530) Time Spent: 3h (was: 2h 50m) > Take advantage of Calcite DDL > - > > Key: BEAM-4044 > URL: https://issues.apache.org/jira/browse/BEAM-4044 > Project: Beam > Issue Type: New Feature > Components: dsl-sql >Reporter: Andrew Pilloud >Assignee: Andrew Pilloud >Priority: Major > Time Spent: 3h > Remaining Estimate: 0h > > In Calcite 1.15 support for abstract DDL moved into calcite core. We should > take advantage of that. -- This message was sent by Atlassian JIRA (v7.6.3#76005)
[jira] [Work logged] (BEAM-4044) Take advantage of Calcite DDL
[ https://issues.apache.org/jira/browse/BEAM-4044?focusedWorklogId=90479&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-90479 ] ASF GitHub Bot logged work on BEAM-4044: Author: ASF GitHub Bot Created on: 12/Apr/18 16:44 Start Date: 12/Apr/18 16:44 Worklog Time Spent: 10m Work Description: akedin commented on a change in pull request #5040: [BEAM-4044] [SQL] Refresh DDL from 1.16 URL: https://github.com/apache/beam/pull/5040#discussion_r181148031 ## File path: sdks/java/extensions/sql/src/main/codegen/includes/parserImpls.ftl ## @@ -1,106 +1,162 @@ -<#-- Licensed to the Apache Software Foundation (ASF) under one or more contributor - license agreements. See the NOTICE file distributed with this work for additional - information regarding copyright ownership. The ASF licenses this file to - You under the Apache License, Version 2.0 (the "License"); you may not use - this file except in compliance with the License. You may obtain a copy of - the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required - by applicable law or agreed to in writing, software distributed under the - License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS - OF ANY KIND, either express or implied. See the License for the specific - language governing permissions and limitations under the License. --> +<#-- +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to you under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with Review comment: uncomment? This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org Issue Time Tracking --- Worklog Id: (was: 90479) Time Spent: 2h 50m (was: 2h 40m) > Take advantage of Calcite DDL > - > > Key: BEAM-4044 > URL: https://issues.apache.org/jira/browse/BEAM-4044 > Project: Beam > Issue Type: New Feature > Components: dsl-sql >Reporter: Andrew Pilloud >Assignee: Andrew Pilloud >Priority: Major > Time Spent: 2h 50m > Remaining Estimate: 0h > > In Calcite 1.15 support for abstract DDL moved into calcite core. We should > take advantage of that. -- This message was sent by Atlassian JIRA (v7.6.3#76005)
[jira] [Work logged] (BEAM-4044) Take advantage of Calcite DDL
[
https://issues.apache.org/jira/browse/BEAM-4044?focusedWorklogId=90160&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-90160
]
ASF GitHub Bot logged work on BEAM-4044:
Author: ASF GitHub Bot
Created on: 11/Apr/18 20:42
Start Date: 11/Apr/18 20:42
Worklog Time Spent: 10m
Work Description: kennknowles commented on a change in pull request
#5040: [BEAM-4044] [SQL] Refresh DDL from 1.16
URL: https://github.com/apache/beam/pull/5040#discussion_r180890261
##
File path:
sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/kafka/KafkaTableProviderTest.java
##
@@ -65,17 +64,17 @@ private static Table mockTable(String name) {
return Table.builder()
.name(name)
.comment(name + " table")
-.location(URI.create("kafka://localhost:2181/brokers?topic=test"))
+.location("kafka://localhost:2181/brokers?topic=test")
Review comment:
So we remove the U from URL and then put it back via `TYPE`. Can't object.
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
Issue Time Tracking
---
Worklog Id: (was: 90160)
Time Spent: 2h 40m (was: 2.5h)
> Take advantage of Calcite DDL
> -
>
> Key: BEAM-4044
> URL: https://issues.apache.org/jira/browse/BEAM-4044
> Project: Beam
> Issue Type: New Feature
> Components: dsl-sql
>Reporter: Andrew Pilloud
>Assignee: Andrew Pilloud
>Priority: Major
> Time Spent: 2h 40m
> Remaining Estimate: 0h
>
> In Calcite 1.15 support for abstract DDL moved into calcite core. We should
> take advantage of that.
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
[jira] [Work logged] (BEAM-4044) Take advantage of Calcite DDL
[ https://issues.apache.org/jira/browse/BEAM-4044?focusedWorklogId=90038&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-90038 ] ASF GitHub Bot logged work on BEAM-4044: Author: ASF GitHub Bot Created on: 11/Apr/18 16:48 Start Date: 11/Apr/18 16:48 Worklog Time Spent: 10m Work Description: apilloud commented on issue #5040: [BEAM-4044] [SQL] Refresh DDL from 1.16 URL: https://github.com/apache/beam/pull/5040#issuecomment-380520597 run java precommit This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org Issue Time Tracking --- Worklog Id: (was: 90038) Time Spent: 2.5h (was: 2h 20m) > Take advantage of Calcite DDL > - > > Key: BEAM-4044 > URL: https://issues.apache.org/jira/browse/BEAM-4044 > Project: Beam > Issue Type: New Feature > Components: dsl-sql >Reporter: Andrew Pilloud >Assignee: Andrew Pilloud >Priority: Major > Time Spent: 2.5h > Remaining Estimate: 0h > > In Calcite 1.15 support for abstract DDL moved into calcite core. We should > take advantage of that. -- This message was sent by Atlassian JIRA (v7.6.3#76005)
[jira] [Work logged] (BEAM-4044) Take advantage of Calcite DDL
[
https://issues.apache.org/jira/browse/BEAM-4044?focusedWorklogId=90024&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-90024
]
ASF GitHub Bot logged work on BEAM-4044:
Author: ASF GitHub Bot
Created on: 11/Apr/18 16:10
Start Date: 11/Apr/18 16:10
Worklog Time Spent: 10m
Work Description: apilloud commented on a change in pull request #5040:
[BEAM-4044] [SQL] Refresh DDL from 1.16
URL: https://github.com/apache/beam/pull/5040#discussion_r180812108
##
File path: sdks/java/extensions/sql/src/main/codegen/includes/parserImpls.ftl
##
@@ -1,106 +1,154 @@
-<#-- Licensed to the Apache Software Foundation (ASF) under one or more
contributor
- license agreements. See the NOTICE file distributed with this work for
additional
- information regarding copyright ownership. The ASF licenses this file to
- You under the Apache License, Version 2.0 (the "License"); you may not use
- this file except in compliance with the License. You may obtain a copy of
- the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required
- by applicable law or agreed to in writing, software distributed under the
- License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS
- OF ANY KIND, either express or implied. See the License for the specific
- language governing permissions and limitations under the License. -->
+<#--
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to you under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+-->
+boolean IfNotExistsOpt() :
+{
+}
+{
+ { return true; }
+|
+{ return false; }
+}
-private void ColumnDef(List list) :
+boolean IfExistsOpt() :
{
-SqlParserPos pos;
-SqlIdentifier name;
-SqlDataTypeSpec type;
-ColumnConstraint constraint = null;
-SqlNode comment = null;
}
{
-name = SimpleIdentifier() { pos = getPos(); }
-type = DataType()
-[
-
- { constraint = new ColumnConstraint.PrimaryKey(getPos()); }
-]
+ { return true; }
+|
+{ return false; }
+}
+
+SqlNodeList Options() :
+{
+final Span s;
+final List list = Lists.newArrayList();
+}
+{
+ { s = span(); }
[
- comment = StringLiteral()
+Option(list)
+(
+
+Option(list)
+)*
]
-{
-list.add(new ColumnDefinition(name, type, constraint, comment, pos));
+ {
+return new SqlNodeList(list, s.end(this));
}
}
-SqlNodeList ColumnDefinitionList() :
+void Option(List list) :
{
-SqlParserPos pos;
-List list = Lists.newArrayList();
+final SqlIdentifier id;
+final SqlNode value;
}
{
- { pos = getPos(); }
-ColumnDef(list)
-( ColumnDef(list) )*
+id = SimpleIdentifier()
+value = Literal() {
+list.add(id);
+list.add(value);
+}
+}
+
+SqlNodeList TableElementList() :
+{
+final Span s;
+final List list = Lists.newArrayList();
+}
+{
+ { s = span(); }
+TableElement(list)
+(
+ TableElement(list)
+)*
{
-return new SqlNodeList(list, pos.plus(getPos()));
+return new SqlNodeList(list, s.end(this));
+}
+}
+
+void TableElement(List list) :
+{
+final SqlIdentifier id;
+final SqlDataTypeSpec type;
+final boolean nullable;
+SqlNode comment = null;
+final Span s = Span.of();
+}
+{
+id = SimpleIdentifier()
+(
+type = DataType()
+(
+ { nullable = true; }
+|
+ { nullable = false; }
+|
+{ nullable = true; }
+)
+[ comment = StringLiteral() ]
+{
+list.add(
+SqlDdlNodes.column(s.add(id).end(this), id,
+type.withNullable(nullable), comment));
+}
+|
+{ list.add(id); }
+)
+|
+id = SimpleIdentifier() {
+list.add(id);
}
}
-/**
- * CREATE TABLE ( IF NOT EXISTS )?
- * ( database_name '.' )? table_name ( '(' column_def ( ',' column_def )* ')'
- * ( STORED AS INPUTFORMAT input_format_classname OUTPUTFORMAT
output_format_classname )?
- * LOCATION location_uri
- * ( TBLPROPERTIES tbl_properties )?
- * ( AS select_stmt )
[jira] [Work logged] (BEAM-4044) Take advantage of Calcite DDL
[
https://issues.apache.org/jira/browse/BEAM-4044?focusedWorklogId=90021&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-90021
]
ASF GitHub Bot logged work on BEAM-4044:
Author: ASF GitHub Bot
Created on: 11/Apr/18 16:09
Start Date: 11/Apr/18 16:09
Worklog Time Spent: 10m
Work Description: apilloud commented on a change in pull request #5040:
[BEAM-4044] [SQL] Refresh DDL from 1.16
URL: https://github.com/apache/beam/pull/5040#discussion_r180811943
##
File path: sdks/java/extensions/sql/src/main/codegen/includes/parserImpls.ftl
##
@@ -1,106 +1,154 @@
-<#-- Licensed to the Apache Software Foundation (ASF) under one or more
contributor
- license agreements. See the NOTICE file distributed with this work for
additional
- information regarding copyright ownership. The ASF licenses this file to
- You under the Apache License, Version 2.0 (the "License"); you may not use
- this file except in compliance with the License. You may obtain a copy of
- the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required
- by applicable law or agreed to in writing, software distributed under the
- License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS
- OF ANY KIND, either express or implied. See the License for the specific
- language governing permissions and limitations under the License. -->
+<#--
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to you under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+-->
+boolean IfNotExistsOpt() :
+{
+}
+{
+ { return true; }
+|
+{ return false; }
+}
-private void ColumnDef(List list) :
+boolean IfExistsOpt() :
{
-SqlParserPos pos;
-SqlIdentifier name;
-SqlDataTypeSpec type;
-ColumnConstraint constraint = null;
-SqlNode comment = null;
}
{
-name = SimpleIdentifier() { pos = getPos(); }
-type = DataType()
-[
-
- { constraint = new ColumnConstraint.PrimaryKey(getPos()); }
-]
+ { return true; }
+|
+{ return false; }
+}
+
+SqlNodeList Options() :
+{
+final Span s;
+final List list = Lists.newArrayList();
+}
+{
+ { s = span(); }
[
- comment = StringLiteral()
+Option(list)
+(
+
+Option(list)
+)*
]
-{
-list.add(new ColumnDefinition(name, type, constraint, comment, pos));
+ {
+return new SqlNodeList(list, s.end(this));
}
}
-SqlNodeList ColumnDefinitionList() :
+void Option(List list) :
{
-SqlParserPos pos;
-List list = Lists.newArrayList();
+final SqlIdentifier id;
+final SqlNode value;
}
{
- { pos = getPos(); }
-ColumnDef(list)
-( ColumnDef(list) )*
+id = SimpleIdentifier()
+value = Literal() {
+list.add(id);
+list.add(value);
+}
+}
+
+SqlNodeList TableElementList() :
+{
+final Span s;
+final List list = Lists.newArrayList();
+}
+{
+ { s = span(); }
+TableElement(list)
+(
+ TableElement(list)
+)*
{
-return new SqlNodeList(list, pos.plus(getPos()));
+return new SqlNodeList(list, s.end(this));
+}
+}
+
+void TableElement(List list) :
+{
+final SqlIdentifier id;
+final SqlDataTypeSpec type;
+final boolean nullable;
+SqlNode comment = null;
+final Span s = Span.of();
+}
+{
+id = SimpleIdentifier()
+(
+type = DataType()
+(
+ { nullable = true; }
+|
+ { nullable = false; }
+|
+{ nullable = true; }
+)
+[ comment = StringLiteral() ]
+{
+list.add(
+SqlDdlNodes.column(s.add(id).end(this), id,
+type.withNullable(nullable), comment));
+}
+|
+{ list.add(id); }
+)
+|
+id = SimpleIdentifier() {
+list.add(id);
}
}
-/**
- * CREATE TABLE ( IF NOT EXISTS )?
- * ( database_name '.' )? table_name ( '(' column_def ( ',' column_def )* ')'
- * ( STORED AS INPUTFORMAT input_format_classname OUTPUTFORMAT
output_format_classname )?
- * LOCATION location_uri
- * ( TBLPROPERTIES tbl_properties )?
- * ( AS select_stmt )
-
[jira] [Work logged] (BEAM-4044) Take advantage of Calcite DDL
[
https://issues.apache.org/jira/browse/BEAM-4044?focusedWorklogId=89807&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-89807
]
ASF GitHub Bot logged work on BEAM-4044:
Author: ASF GitHub Bot
Created on: 11/Apr/18 06:48
Start Date: 11/Apr/18 06:48
Worklog Time Spent: 10m
Work Description: XuMingmin commented on a change in pull request #5040:
[BEAM-4044] [SQL] Refresh DDL from 1.16
URL: https://github.com/apache/beam/pull/5040#discussion_r180646402
##
File path: sdks/java/extensions/sql/src/main/codegen/includes/parserImpls.ftl
##
@@ -1,106 +1,154 @@
-<#-- Licensed to the Apache Software Foundation (ASF) under one or more
contributor
- license agreements. See the NOTICE file distributed with this work for
additional
- information regarding copyright ownership. The ASF licenses this file to
- You under the Apache License, Version 2.0 (the "License"); you may not use
- this file except in compliance with the License. You may obtain a copy of
- the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required
- by applicable law or agreed to in writing, software distributed under the
- License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS
- OF ANY KIND, either express or implied. See the License for the specific
- language governing permissions and limitations under the License. -->
+<#--
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to you under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+-->
+boolean IfNotExistsOpt() :
+{
+}
+{
+ { return true; }
+|
+{ return false; }
+}
-private void ColumnDef(List list) :
+boolean IfExistsOpt() :
{
-SqlParserPos pos;
-SqlIdentifier name;
-SqlDataTypeSpec type;
-ColumnConstraint constraint = null;
-SqlNode comment = null;
}
{
-name = SimpleIdentifier() { pos = getPos(); }
-type = DataType()
-[
-
- { constraint = new ColumnConstraint.PrimaryKey(getPos()); }
-]
+ { return true; }
+|
+{ return false; }
+}
+
+SqlNodeList Options() :
+{
+final Span s;
+final List list = Lists.newArrayList();
+}
+{
+ { s = span(); }
[
- comment = StringLiteral()
+Option(list)
+(
+
+Option(list)
+)*
]
-{
-list.add(new ColumnDefinition(name, type, constraint, comment, pos));
+ {
+return new SqlNodeList(list, s.end(this));
}
}
-SqlNodeList ColumnDefinitionList() :
+void Option(List list) :
{
-SqlParserPos pos;
-List list = Lists.newArrayList();
+final SqlIdentifier id;
+final SqlNode value;
}
{
- { pos = getPos(); }
-ColumnDef(list)
-( ColumnDef(list) )*
+id = SimpleIdentifier()
+value = Literal() {
+list.add(id);
+list.add(value);
+}
+}
+
+SqlNodeList TableElementList() :
+{
+final Span s;
+final List list = Lists.newArrayList();
+}
+{
+ { s = span(); }
+TableElement(list)
+(
+ TableElement(list)
+)*
{
-return new SqlNodeList(list, pos.plus(getPos()));
+return new SqlNodeList(list, s.end(this));
+}
+}
+
+void TableElement(List list) :
+{
+final SqlIdentifier id;
+final SqlDataTypeSpec type;
+final boolean nullable;
+SqlNode comment = null;
+final Span s = Span.of();
+}
+{
+id = SimpleIdentifier()
+(
+type = DataType()
+(
+ { nullable = true; }
+|
+ { nullable = false; }
+|
+{ nullable = true; }
+)
+[ comment = StringLiteral() ]
+{
+list.add(
+SqlDdlNodes.column(s.add(id).end(this), id,
+type.withNullable(nullable), comment));
+}
+|
+{ list.add(id); }
+)
+|
+id = SimpleIdentifier() {
+list.add(id);
}
}
-/**
- * CREATE TABLE ( IF NOT EXISTS )?
- * ( database_name '.' )? table_name ( '(' column_def ( ',' column_def )* ')'
- * ( STORED AS INPUTFORMAT input_format_classname OUTPUTFORMAT
output_format_classname )?
- * LOCATION location_uri
- * ( TBLPROPERTIES tbl_properties )?
- * ( AS select_stmt )
[jira] [Work logged] (BEAM-4044) Take advantage of Calcite DDL
[
https://issues.apache.org/jira/browse/BEAM-4044?focusedWorklogId=89806&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-89806
]
ASF GitHub Bot logged work on BEAM-4044:
Author: ASF GitHub Bot
Created on: 11/Apr/18 06:48
Start Date: 11/Apr/18 06:48
Worklog Time Spent: 10m
Work Description: XuMingmin commented on a change in pull request #5040:
[BEAM-4044] [SQL] Refresh DDL from 1.16
URL: https://github.com/apache/beam/pull/5040#discussion_r180647797
##
File path: sdks/java/extensions/sql/src/main/codegen/includes/parserImpls.ftl
##
@@ -1,106 +1,154 @@
-<#-- Licensed to the Apache Software Foundation (ASF) under one or more
contributor
- license agreements. See the NOTICE file distributed with this work for
additional
- information regarding copyright ownership. The ASF licenses this file to
- You under the Apache License, Version 2.0 (the "License"); you may not use
- this file except in compliance with the License. You may obtain a copy of
- the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required
- by applicable law or agreed to in writing, software distributed under the
- License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS
- OF ANY KIND, either express or implied. See the License for the specific
- language governing permissions and limitations under the License. -->
+<#--
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to you under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+-->
+boolean IfNotExistsOpt() :
+{
+}
+{
+ { return true; }
+|
+{ return false; }
+}
-private void ColumnDef(List list) :
+boolean IfExistsOpt() :
{
-SqlParserPos pos;
-SqlIdentifier name;
-SqlDataTypeSpec type;
-ColumnConstraint constraint = null;
-SqlNode comment = null;
}
{
-name = SimpleIdentifier() { pos = getPos(); }
-type = DataType()
-[
-
- { constraint = new ColumnConstraint.PrimaryKey(getPos()); }
-]
+ { return true; }
+|
+{ return false; }
+}
+
+SqlNodeList Options() :
+{
+final Span s;
+final List list = Lists.newArrayList();
+}
+{
+ { s = span(); }
[
- comment = StringLiteral()
+Option(list)
+(
+
+Option(list)
+)*
]
-{
-list.add(new ColumnDefinition(name, type, constraint, comment, pos));
+ {
+return new SqlNodeList(list, s.end(this));
}
}
-SqlNodeList ColumnDefinitionList() :
+void Option(List list) :
{
-SqlParserPos pos;
-List list = Lists.newArrayList();
+final SqlIdentifier id;
+final SqlNode value;
}
{
- { pos = getPos(); }
-ColumnDef(list)
-( ColumnDef(list) )*
+id = SimpleIdentifier()
+value = Literal() {
+list.add(id);
+list.add(value);
+}
+}
+
+SqlNodeList TableElementList() :
+{
+final Span s;
+final List list = Lists.newArrayList();
+}
+{
+ { s = span(); }
+TableElement(list)
+(
+ TableElement(list)
+)*
{
-return new SqlNodeList(list, pos.plus(getPos()));
+return new SqlNodeList(list, s.end(this));
+}
+}
+
+void TableElement(List list) :
+{
+final SqlIdentifier id;
+final SqlDataTypeSpec type;
+final boolean nullable;
+SqlNode comment = null;
+final Span s = Span.of();
+}
+{
+id = SimpleIdentifier()
+(
+type = DataType()
+(
+ { nullable = true; }
+|
+ { nullable = false; }
+|
+{ nullable = true; }
+)
+[ comment = StringLiteral() ]
+{
+list.add(
+SqlDdlNodes.column(s.add(id).end(this), id,
+type.withNullable(nullable), comment));
+}
+|
+{ list.add(id); }
+)
+|
+id = SimpleIdentifier() {
+list.add(id);
}
}
-/**
- * CREATE TABLE ( IF NOT EXISTS )?
- * ( database_name '.' )? table_name ( '(' column_def ( ',' column_def )* ')'
- * ( STORED AS INPUTFORMAT input_format_classname OUTPUTFORMAT
output_format_classname )?
- * LOCATION location_uri
- * ( TBLPROPERTIES tbl_properties )?
- * ( AS select_stmt )
-
[jira] [Work logged] (BEAM-4044) Take advantage of Calcite DDL
[
https://issues.apache.org/jira/browse/BEAM-4044?focusedWorklogId=89717&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-89717
]
ASF GitHub Bot logged work on BEAM-4044:
Author: ASF GitHub Bot
Created on: 10/Apr/18 23:46
Start Date: 10/Apr/18 23:46
Worklog Time Spent: 10m
Work Description: apilloud commented on a change in pull request #5040:
[BEAM-4044] [SQL] Refresh DDL from 1.16
URL: https://github.com/apache/beam/pull/5040#discussion_r180599529
##
File path:
sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/kafka/KafkaTableProviderTest.java
##
@@ -65,17 +64,17 @@ private static Table mockTable(String name) {
return Table.builder()
.name(name)
.comment(name + " table")
-.location(URI.create("kafka://localhost:2181/brokers?topic=test"))
+.location("kafka://localhost:2181/brokers?topic=test")
Review comment:
The DDL already requires a type. Syntax for this looks something like
`CREATE TABLE mytable TYPE kafka LOCATION
"kafka://localhost:2181/brokers?topic=test"`
PR #4991 links to #4947 which adds bigquery. I was honestly expecting both
to be merged before this one was reviewed, but I haven't been able to get
reviews on #4991.
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
Issue Time Tracking
---
Worklog Id: (was: 89717)
Time Spent: 1h 40m (was: 1.5h)
> Take advantage of Calcite DDL
> -
>
> Key: BEAM-4044
> URL: https://issues.apache.org/jira/browse/BEAM-4044
> Project: Beam
> Issue Type: New Feature
> Components: dsl-sql
>Reporter: Andrew Pilloud
>Assignee: Andrew Pilloud
>Priority: Major
> Time Spent: 1h 40m
> Remaining Estimate: 0h
>
> In Calcite 1.15 support for abstract DDL moved into calcite core. We should
> take advantage of that.
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
[jira] [Work logged] (BEAM-4044) Take advantage of Calcite DDL
[
https://issues.apache.org/jira/browse/BEAM-4044?focusedWorklogId=89708&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-89708
]
ASF GitHub Bot logged work on BEAM-4044:
Author: ASF GitHub Bot
Created on: 10/Apr/18 23:37
Start Date: 10/Apr/18 23:37
Worklog Time Spent: 10m
Work Description: akedin commented on a change in pull request #5040:
[BEAM-4044] [SQL] Refresh DDL from 1.16
URL: https://github.com/apache/beam/pull/5040#discussion_r180598326
##
File path:
sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/kafka/KafkaTableProviderTest.java
##
@@ -65,17 +64,17 @@ private static Table mockTable(String name) {
return Table.builder()
.name(name)
.comment(name + " table")
-.location(URI.create("kafka://localhost:2181/brokers?topic=test"))
+.location("kafka://localhost:2181/brokers?topic=test")
Review comment:
Got it. The #4991 doesn't show any BQ-specific examples so this wasn't
clear. How do we know that the table is a BQ table then if we can't use
Location to identify it?
And I think that URI format actually supports the syntax for BQ table IDs,
but not sure how semantically correct that would be. It would probably
technically be a URN at that point which our code has to resolve using some
other data.
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
Issue Time Tracking
---
Worklog Id: (was: 89708)
Time Spent: 1.5h (was: 1h 20m)
> Take advantage of Calcite DDL
> -
>
> Key: BEAM-4044
> URL: https://issues.apache.org/jira/browse/BEAM-4044
> Project: Beam
> Issue Type: New Feature
> Components: dsl-sql
>Reporter: Andrew Pilloud
>Assignee: Andrew Pilloud
>Priority: Major
> Time Spent: 1.5h
> Remaining Estimate: 0h
>
> In Calcite 1.15 support for abstract DDL moved into calcite core. We should
> take advantage of that.
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
[jira] [Work logged] (BEAM-4044) Take advantage of Calcite DDL
[ https://issues.apache.org/jira/browse/BEAM-4044?focusedWorklogId=89665&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-89665 ] ASF GitHub Bot logged work on BEAM-4044: Author: ASF GitHub Bot Created on: 10/Apr/18 23:05 Start Date: 10/Apr/18 23:05 Worklog Time Spent: 10m Work Description: akedin commented on a change in pull request #5040: [BEAM-4044] [SQL] Refresh DDL from 1.16 URL: https://github.com/apache/beam/pull/5040#discussion_r180593027 ## File path: sdks/java/extensions/sql/src/main/codegen/includes/parserImpls.ftl ## @@ -1,106 +1,154 @@ -<#-- Licensed to the Apache Software Foundation (ASF) under one or more contributor - license agreements. See the NOTICE file distributed with this work for additional - information regarding copyright ownership. The ASF licenses this file to - You under the Apache License, Version 2.0 (the "License"); you may not use - this file except in compliance with the License. You may obtain a copy of - the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required - by applicable law or agreed to in writing, software distributed under the - License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS - OF ANY KIND, either express or implied. See the License for the specific - language governing permissions and limitations under the License. --> +<#-- +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to you under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +--> +boolean IfNotExistsOpt() : Review comment: This link is enough, i think. Do we need to update parser tests though? This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org Issue Time Tracking --- Worklog Id: (was: 89665) Time Spent: 1h 20m (was: 1h 10m) > Take advantage of Calcite DDL > - > > Key: BEAM-4044 > URL: https://issues.apache.org/jira/browse/BEAM-4044 > Project: Beam > Issue Type: New Feature > Components: dsl-sql >Reporter: Andrew Pilloud >Assignee: Andrew Pilloud >Priority: Major > Time Spent: 1h 20m > Remaining Estimate: 0h > > In Calcite 1.15 support for abstract DDL moved into calcite core. We should > take advantage of that. -- This message was sent by Atlassian JIRA (v7.6.3#76005)
[jira] [Work logged] (BEAM-4044) Take advantage of Calcite DDL
[ https://issues.apache.org/jira/browse/BEAM-4044?focusedWorklogId=89647&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-89647 ] ASF GitHub Bot logged work on BEAM-4044: Author: ASF GitHub Bot Created on: 10/Apr/18 22:06 Start Date: 10/Apr/18 22:06 Worklog Time Spent: 10m Work Description: apilloud commented on issue #5040: [BEAM-4044] [SQL] Refresh DDL from 1.16 URL: https://github.com/apache/beam/pull/5040#issuecomment-380262902 Tests are in a few places. These are the main ones: https://github.com/apache/beam/blob/master/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/parser/BeamSqlParserTest.java This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org Issue Time Tracking --- Worklog Id: (was: 89647) Time Spent: 1h 10m (was: 1h) > Take advantage of Calcite DDL > - > > Key: BEAM-4044 > URL: https://issues.apache.org/jira/browse/BEAM-4044 > Project: Beam > Issue Type: New Feature > Components: dsl-sql >Reporter: Andrew Pilloud >Assignee: Andrew Pilloud >Priority: Major > Time Spent: 1h 10m > Remaining Estimate: 0h > > In Calcite 1.15 support for abstract DDL moved into calcite core. We should > take advantage of that. -- This message was sent by Atlassian JIRA (v7.6.3#76005)
[jira] [Work logged] (BEAM-4044) Take advantage of Calcite DDL
[
https://issues.apache.org/jira/browse/BEAM-4044?focusedWorklogId=89646&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-89646
]
ASF GitHub Bot logged work on BEAM-4044:
Author: ASF GitHub Bot
Created on: 10/Apr/18 22:04
Start Date: 10/Apr/18 22:04
Worklog Time Spent: 10m
Work Description: apilloud commented on a change in pull request #5040:
[BEAM-4044] [SQL] Refresh DDL from 1.16
URL: https://github.com/apache/beam/pull/5040#discussion_r180581448
##
File path: sdks/java/extensions/sql/src/main/codegen/includes/parserImpls.ftl
##
@@ -1,106 +1,154 @@
-<#-- Licensed to the Apache Software Foundation (ASF) under one or more
contributor
- license agreements. See the NOTICE file distributed with this work for
additional
- information regarding copyright ownership. The ASF licenses this file to
- You under the Apache License, Version 2.0 (the "License"); you may not use
- this file except in compliance with the License. You may obtain a copy of
- the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required
- by applicable law or agreed to in writing, software distributed under the
- License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS
- OF ANY KIND, either express or implied. See the License for the specific
- language governing permissions and limitations under the License. -->
+<#--
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to you under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+-->
+boolean IfNotExistsOpt() :
+{
+}
+{
+ { return true; }
+|
+{ return false; }
+}
-private void ColumnDef(List list) :
+boolean IfExistsOpt() :
{
-SqlParserPos pos;
-SqlIdentifier name;
-SqlDataTypeSpec type;
-ColumnConstraint constraint = null;
-SqlNode comment = null;
}
{
-name = SimpleIdentifier() { pos = getPos(); }
-type = DataType()
-[
-
- { constraint = new ColumnConstraint.PrimaryKey(getPos()); }
-]
+ { return true; }
+|
+{ return false; }
+}
+
+SqlNodeList Options() :
+{
+final Span s;
+final List list = Lists.newArrayList();
+}
+{
+ { s = span(); }
[
- comment = StringLiteral()
+Option(list)
+(
+
+Option(list)
+)*
]
-{
-list.add(new ColumnDefinition(name, type, constraint, comment, pos));
+ {
+return new SqlNodeList(list, s.end(this));
}
}
-SqlNodeList ColumnDefinitionList() :
+void Option(List list) :
{
-SqlParserPos pos;
-List list = Lists.newArrayList();
+final SqlIdentifier id;
+final SqlNode value;
}
{
- { pos = getPos(); }
-ColumnDef(list)
-( ColumnDef(list) )*
+id = SimpleIdentifier()
+value = Literal() {
+list.add(id);
+list.add(value);
+}
+}
+
+SqlNodeList TableElementList() :
+{
+final Span s;
+final List list = Lists.newArrayList();
+}
+{
+ { s = span(); }
+TableElement(list)
+(
+ TableElement(list)
+)*
{
-return new SqlNodeList(list, pos.plus(getPos()));
+return new SqlNodeList(list, s.end(this));
+}
+}
+
+void TableElement(List list) :
+{
+final SqlIdentifier id;
+final SqlDataTypeSpec type;
+final boolean nullable;
+SqlNode comment = null;
+final Span s = Span.of();
+}
+{
+id = SimpleIdentifier()
+(
+type = DataType()
+(
+ { nullable = true; }
+|
+ { nullable = false; }
+|
+{ nullable = true; }
+)
+[ comment = StringLiteral() ]
+{
+list.add(
+SqlDdlNodes.column(s.add(id).end(this), id,
+type.withNullable(nullable), comment));
+}
+|
+{ list.add(id); }
+)
+|
+id = SimpleIdentifier() {
+list.add(id);
}
}
-/**
- * CREATE TABLE ( IF NOT EXISTS )?
- * ( database_name '.' )? table_name ( '(' column_def ( ',' column_def )* ')'
- * ( STORED AS INPUTFORMAT input_format_classname OUTPUTFORMAT
output_format_classname )?
- * LOCATION location_uri
- * ( TBLPROPERTIES tbl_properties )?
- * ( AS select_stmt )
