svn commit: r33783 - /dev/beam/2.12.0/ /release/beam/2.12.0/

[takidau]

Author: takidau
Date: Thu Apr 25 20:00:07 2019
New Revision: 33783

Log:
Promote 2.12.0 from dev to release.

Added:
release/beam/2.12.0/
  - copied from r33782, dev/beam/2.12.0/
Removed:
dev/beam/2.12.0/

[GitHub] beam pull request #3721: [BEAM-2770] Add @Experimental and ImmutableList.cop...

[takidau]

GitHub user takidau opened a pull request:

https://github.com/apache/beam/pull/3721

[BEAM-2770] Add @Experimental and ImmutableList.copyOf

Some final API polish, making sure \@Experimental annotations are on all 
the public SQL API classes, and immutable objects always return ImmutableLists 
for private members instead of the mutable members themselves.

You can merge this pull request into a Git repository by running:

$ git pull https://github.com/takidau/beam misc-api-cleanups

Alternatively you can review and apply these changes as the patch at:

https://github.com/apache/beam/pull/3721.patch

To close this pull request, make a commit to your master/trunk branch
with (at least) the following in the commit message:

This closes #3721


commit 670c2ee3f3ab03df60b89eb7f4d16760d43e0bd5
Author: Tyler Akidau <taki...@apache.org>
Date:   2017-08-15T23:05:11Z

Add @Experimental and ImmutableList.copyOf




---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

[3/5] beam git commit: [BEAM-2740] Hide BeamSqlEnv.

[takidau]

http://git-wip-us.apache.org/repos/asf/beam/blob/49aad927/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/schema/kafka/BeamKafkaCSVTable.java
--
diff --git 
a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/schema/kafka/BeamKafkaCSVTable.java
 
b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/schema/kafka/BeamKafkaCSVTable.java
new file mode 100644
index 000..4bedec1
--- /dev/null
+++ 
b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/schema/kafka/BeamKafkaCSVTable.java
@@ -0,0 +1,109 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.extensions.sql.impl.schema.kafka;
+
+import java.util.List;
+import org.apache.beam.sdk.extensions.sql.BeamRecordSqlType;
+import org.apache.beam.sdk.extensions.sql.impl.schema.BeamTableUtils;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.values.BeamRecord;
+import org.apache.beam.sdk.values.KV;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.commons.csv.CSVFormat;
+
+/**
+ * A Kafka topic that saves records as CSV format.
+ *
+ */
+public class BeamKafkaCSVTable extends BeamKafkaTable {
+  private CSVFormat csvFormat;
+  public BeamKafkaCSVTable(BeamRecordSqlType beamSqlRowType, String 
bootstrapServers,
+  List topics) {
+this(beamSqlRowType, bootstrapServers, topics, CSVFormat.DEFAULT);
+  }
+
+  public BeamKafkaCSVTable(BeamRecordSqlType beamSqlRowType, String 
bootstrapServers,
+  List topics, CSVFormat format) {
+super(beamSqlRowType, bootstrapServers, topics);
+this.csvFormat = format;
+  }
+
+  @Override
+  public PTransform>, PCollection>
+  getPTransformForInput() {
+return new CsvRecorderDecoder(beamSqlRowType, csvFormat);
+  }
+
+  @Override
+  public PTransform>>
+  getPTransformForOutput() {
+return new CsvRecorderEncoder(beamSqlRowType, csvFormat);
+  }
+
+  /**
+   * A PTransform to convert {@code KV} to {@link BeamRecord}.
+   *
+   */
+  public static class CsvRecorderDecoder
+  extends PTransform>, 
PCollection> {
+private BeamRecordSqlType rowType;
+private CSVFormat format;
+public CsvRecorderDecoder(BeamRecordSqlType rowType, CSVFormat format) {
+  this.rowType = rowType;
+  this.format = format;
+}
+
+@Override
+public PCollection expand(PCollection> 
input) {
+  return input.apply("decodeRecord", ParDo.of(new DoFn, 
BeamRecord>() {
+@ProcessElement
+public void processElement(ProcessContext c) {
+  String rowInString = new String(c.element().getValue());
+  c.output(BeamTableUtils.csvLine2BeamSqlRow(format, rowInString, 
rowType));
+}
+  }));
+}
+  }
+
+  /**
+   * A PTransform to convert {@link BeamRecord} to {@code KV}.
+   *
+   */
+  public static class CsvRecorderEncoder
+  extends PTransform>> {
+private BeamRecordSqlType rowType;
+private CSVFormat format;
+public CsvRecorderEncoder(BeamRecordSqlType rowType, CSVFormat format) {
+  this.rowType = rowType;
+  this.format = format;
+}
+
+@Override
+public PCollection> expand(PCollection 
input) {
+  return input.apply("encodeRecord", ParDo.of(new DoFn>() {
+@ProcessElement
+public void processElement(ProcessContext c) {
+  BeamRecord in = c.element();
+  c.output(KV.of(new byte[] {}, BeamTableUtils.beamSqlRow2CsvLine(in, 
format).getBytes()));
+}
+  }));
+}
+  }
+}

http://git-wip-us.apache.org/repos/asf/beam/blob/49aad927/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/schema/kafka/BeamKafkaTable.java

[4/5] beam git commit: [BEAM-2740] Hide BeamSqlEnv.

[takidau]

[BEAM-2740] Hide BeamSqlEnv.


Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/49aad927
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/49aad927
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/49aad927

Branch: refs/heads/DSL_SQL
Commit: 49aad927d4d9cf58c30c04641c766a62d44f44b7
Parents: 9eec6a0
Author: James Xu 
Authored: Wed Aug 9 18:54:54 2017 +0800
Committer: Tyler Akidau 
Committed: Tue Aug 15 11:40:39 2017 -0700

--
 .../sdk/extensions/sql/BeamRecordSqlType.java   | 185 
 .../apache/beam/sdk/extensions/sql/BeamSql.java | 113 ++---
 .../beam/sdk/extensions/sql/BeamSqlCli.java |  65 ---
 .../beam/sdk/extensions/sql/BeamSqlEnv.java | 127 --
 .../sdk/extensions/sql/BeamSqlRecordHelper.java | 217 +
 .../beam/sdk/extensions/sql/BeamSqlUdf.java |  41 ++
 .../extensions/sql/example/BeamSqlExample.java  |   2 +-
 .../sdk/extensions/sql/impl/BeamSqlCli.java |  65 +++
 .../sdk/extensions/sql/impl/BeamSqlEnv.java | 135 ++
 .../sdk/extensions/sql/impl/package-info.java   |  22 +
 .../sql/impl/planner/BeamQueryPlanner.java  |   9 +-
 .../sql/impl/rel/BeamAggregationRel.java|   4 +-
 .../extensions/sql/impl/rel/BeamFilterRel.java  |   2 +-
 .../extensions/sql/impl/rel/BeamIOSinkRel.java  |   6 +-
 .../sql/impl/rel/BeamIOSourceRel.java   |   6 +-
 .../sql/impl/rel/BeamIntersectRel.java  |   2 +-
 .../extensions/sql/impl/rel/BeamJoinRel.java|   4 +-
 .../extensions/sql/impl/rel/BeamMinusRel.java   |   2 +-
 .../extensions/sql/impl/rel/BeamProjectRel.java |   2 +-
 .../extensions/sql/impl/rel/BeamRelNode.java|   5 +-
 .../sql/impl/rel/BeamSetOperatorRelBase.java|   2 +-
 .../extensions/sql/impl/rel/BeamSortRel.java|   4 +-
 .../extensions/sql/impl/rel/BeamUnionRel.java   |   2 +-
 .../extensions/sql/impl/rel/BeamValuesRel.java  |   6 +-
 .../sql/impl/schema/BaseBeamTable.java  |  35 ++
 .../extensions/sql/impl/schema/BeamIOType.java  |  28 ++
 .../sql/impl/schema/BeamPCollectionTable.java   |  63 +++
 .../sql/impl/schema/BeamSqlTable.java   |  54 +++
 .../sql/impl/schema/BeamTableUtils.java | 118 +
 .../impl/schema/kafka/BeamKafkaCSVTable.java| 109 +
 .../sql/impl/schema/kafka/BeamKafkaTable.java   | 109 +
 .../sql/impl/schema/kafka/package-info.java |  22 +
 .../sql/impl/schema/package-info.java   |  22 +
 .../sql/impl/schema/text/BeamTextCSVTable.java  |  70 +++
 .../schema/text/BeamTextCSVTableIOReader.java   |  58 +++
 .../schema/text/BeamTextCSVTableIOWriter.java   |  58 +++
 .../sql/impl/schema/text/BeamTextTable.java |  41 ++
 .../sql/impl/schema/text/package-info.java  |  22 +
 .../transform/BeamAggregationTransforms.java|   4 +-
 .../sql/impl/transform/BeamJoinTransforms.java  |   4 +-
 .../sql/impl/transform/BeamSqlProjectFn.java|   4 +-
 .../extensions/sql/impl/utils/CalciteUtils.java |   2 +-
 .../extensions/sql/schema/BaseBeamTable.java|  34 --
 .../sdk/extensions/sql/schema/BeamIOType.java   |  28 --
 .../sql/schema/BeamPCollectionTable.java|  62 ---
 .../sql/schema/BeamRecordSqlType.java   | 185 
 .../sql/schema/BeamSqlRecordHelper.java | 217 -
 .../sdk/extensions/sql/schema/BeamSqlTable.java |  53 ---
 .../sdk/extensions/sql/schema/BeamSqlUdf.java   |  41 --
 .../extensions/sql/schema/BeamTableUtils.java   | 117 -
 .../sql/schema/kafka/BeamKafkaCSVTable.java | 109 -
 .../sql/schema/kafka/BeamKafkaTable.java| 109 -
 .../sql/schema/kafka/package-info.java  |  22 -
 .../sdk/extensions/sql/schema/package-info.java |  22 -
 .../sql/schema/text/BeamTextCSVTable.java   |  70 ---
 .../schema/text/BeamTextCSVTableIOReader.java   |  58 ---
 .../schema/text/BeamTextCSVTableIOWriter.java   |  58 ---
 .../sql/schema/text/BeamTextTable.java  |  41 --
 .../sql/schema/text/package-info.java   |  22 -
 .../extensions/sql/BeamSqlApiSurfaceTest.java   |  12 +-
 .../sql/BeamSqlDslAggregationTest.java  |   1 -
 .../beam/sdk/extensions/sql/BeamSqlDslBase.java |   1 -
 .../sdk/extensions/sql/BeamSqlDslJoinTest.java  |   1 -
 .../extensions/sql/BeamSqlDslProjectTest.java   |   1 -
 .../extensions/sql/BeamSqlDslUdfUdafTest.java   |   2 -
 .../beam/sdk/extensions/sql/TestUtils.java  |   1 -
 .../interpreter/BeamSqlFnExecutorTestBase.java  |   2 +-
 .../extensions/sql/impl/rel/BaseRelTest.java|  34 ++
 .../sql/impl/rel/BeamIntersectRelTest.java  |   9 +-
 .../rel/BeamJoinRelBoundedVsBoundedTest.java|  23 +-
 .../rel/BeamJoinRelUnboundedVsBoundedTest.java  |  25 +-
 .../BeamJoinRelUnboundedVsUnboundedTest.java|  19 +-
 .../sql/impl/rel/BeamMinusRelTest.java  |   9 +-
 .../impl/rel/BeamSetOperatorRelBaseTest.java|   9 +-
 

[2/5] beam git commit: [BEAM-2740] Hide BeamSqlEnv.

[takidau]

http://git-wip-us.apache.org/repos/asf/beam/blob/49aad927/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/BeamSqlApiSurfaceTest.java
--
diff --git 
a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/BeamSqlApiSurfaceTest.java
 
b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/BeamSqlApiSurfaceTest.java
index 08678d1..456662f 100644
--- 
a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/BeamSqlApiSurfaceTest.java
+++ 
b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/BeamSqlApiSurfaceTest.java
@@ -40,15 +40,13 @@ public class BeamSqlApiSurfaceTest {
 final Set allowed =
 ImmutableSet.of(
 "org.apache.beam",
-"org.joda.time",
-"org.apache.commons.csv");
+"org.joda.time");
 
 ApiSurface surface = ApiSurface
-.ofClass(BeamSqlCli.class)
-.includingClass(BeamSql.class)
-.includingClass(BeamSqlEnv.class)
-.includingPackage("org.apache.beam.sdk.extensions.sql.schema",
-getClass().getClassLoader())
+.ofClass(BeamSql.class)
+.includingClass(BeamSqlUdf.class)
+.includingClass(BeamRecordSqlType.class)
+.includingClass(BeamSqlRecordHelper.class)
 .pruningPrefix("java")
 
.pruningPattern("org[.]apache[.]beam[.]sdk[.]extensions[.]sql[.].*Test")
 
.pruningPattern("org[.]apache[.]beam[.]sdk[.]extensions[.]sql[.].*TestBase");

http://git-wip-us.apache.org/repos/asf/beam/blob/49aad927/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/BeamSqlDslAggregationTest.java
--
diff --git 
a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/BeamSqlDslAggregationTest.java
 
b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/BeamSqlDslAggregationTest.java
index db562da..d99ec20 100644
--- 
a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/BeamSqlDslAggregationTest.java
+++ 
b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/BeamSqlDslAggregationTest.java
@@ -19,7 +19,6 @@ package org.apache.beam.sdk.extensions.sql;
 
 import java.sql.Types;
 import java.util.Arrays;
-import org.apache.beam.sdk.extensions.sql.schema.BeamRecordSqlType;
 import org.apache.beam.sdk.testing.PAssert;
 import org.apache.beam.sdk.values.BeamRecord;
 import org.apache.beam.sdk.values.PCollection;

http://git-wip-us.apache.org/repos/asf/beam/blob/49aad927/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/BeamSqlDslBase.java
--
diff --git 
a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/BeamSqlDslBase.java
 
b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/BeamSqlDslBase.java
index ef75ee2..b27435c 100644
--- 
a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/BeamSqlDslBase.java
+++ 
b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/BeamSqlDslBase.java
@@ -25,7 +25,6 @@ import java.text.SimpleDateFormat;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
-import org.apache.beam.sdk.extensions.sql.schema.BeamRecordSqlType;
 import org.apache.beam.sdk.testing.TestPipeline;
 import org.apache.beam.sdk.testing.TestStream;
 import org.apache.beam.sdk.transforms.Create;

http://git-wip-us.apache.org/repos/asf/beam/blob/49aad927/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/BeamSqlDslJoinTest.java
--
diff --git 
a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/BeamSqlDslJoinTest.java
 
b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/BeamSqlDslJoinTest.java
index 0876dd9..47109e0 100644
--- 
a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/BeamSqlDslJoinTest.java
+++ 
b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/BeamSqlDslJoinTest.java
@@ -24,7 +24,6 @@ import static 
org.apache.beam.sdk.extensions.sql.impl.rel.BeamJoinRelBoundedVsBo
 import java.sql.Types;
 import java.util.Arrays;
 import org.apache.beam.sdk.coders.BeamRecordCoder;
-import org.apache.beam.sdk.extensions.sql.schema.BeamRecordSqlType;
 import org.apache.beam.sdk.testing.PAssert;
 import org.apache.beam.sdk.testing.TestPipeline;
 import org.apache.beam.sdk.values.BeamRecord;

http://git-wip-us.apache.org/repos/asf/beam/blob/49aad927/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/BeamSqlDslProjectTest.java
--
diff --git 

[1/5] beam git commit: [BEAM-2740] Hide BeamSqlEnv.

[takidau]

Repository: beam
Updated Branches:
  refs/heads/DSL_SQL 9eec6a030 -> a1cc5518e


http://git-wip-us.apache.org/repos/asf/beam/blob/49aad927/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/schema/BeamSqlRowCoderTest.java
--
diff --git 
a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/schema/BeamSqlRowCoderTest.java
 
b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/schema/BeamSqlRowCoderTest.java
deleted file mode 100644
index 8751bbb..000
--- 
a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/schema/BeamSqlRowCoderTest.java
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.beam.sdk.extensions.sql.schema;
-
-import java.math.BigDecimal;
-import java.util.Date;
-import java.util.GregorianCalendar;
-import org.apache.beam.sdk.coders.BeamRecordCoder;
-import org.apache.beam.sdk.extensions.sql.impl.utils.CalciteUtils;
-import org.apache.beam.sdk.testing.CoderProperties;
-import org.apache.beam.sdk.values.BeamRecord;
-import org.apache.calcite.jdbc.JavaTypeFactoryImpl;
-import org.apache.calcite.rel.type.RelDataType;
-import org.apache.calcite.rel.type.RelDataTypeFactory;
-import org.apache.calcite.rel.type.RelDataTypeSystem;
-import org.apache.calcite.rel.type.RelProtoDataType;
-import org.apache.calcite.sql.type.SqlTypeName;
-import org.junit.Test;
-
-/**
- * Tests for BeamSqlRowCoder.
- */
-public class BeamSqlRowCoderTest {
-
-  @Test
-  public void encodeAndDecode() throws Exception {
-final RelProtoDataType protoRowType = new RelProtoDataType() {
-  @Override
-  public RelDataType apply(RelDataTypeFactory a0) {
-return a0.builder()
-.add("col_tinyint", SqlTypeName.TINYINT)
-.add("col_smallint", SqlTypeName.SMALLINT)
-.add("col_integer", SqlTypeName.INTEGER)
-.add("col_bigint", SqlTypeName.BIGINT)
-.add("col_float", SqlTypeName.FLOAT)
-.add("col_double", SqlTypeName.DOUBLE)
-.add("col_decimal", SqlTypeName.DECIMAL)
-.add("col_string_varchar", SqlTypeName.VARCHAR)
-.add("col_time", SqlTypeName.TIME)
-.add("col_timestamp", SqlTypeName.TIMESTAMP)
-.add("col_boolean", SqlTypeName.BOOLEAN)
-.build();
-  }
-};
-
-BeamRecordSqlType beamSQLRowType = CalciteUtils.toBeamRowType(
-protoRowType.apply(new JavaTypeFactoryImpl(
-RelDataTypeSystem.DEFAULT)));
-
-GregorianCalendar calendar = new GregorianCalendar();
-calendar.setTime(new Date());
-BeamRecord row = new BeamRecord(beamSQLRowType
-, Byte.valueOf("1"), Short.valueOf("1"), 1, 1L, 1.1F, 1.1
-, BigDecimal.ZERO, "hello", calendar, new Date(), true);
-
-
-BeamRecordCoder coder = beamSQLRowType.getRecordCoder();
-CoderProperties.coderDecodeEncodeEqual(coder, row);
-  }
-}

http://git-wip-us.apache.org/repos/asf/beam/blob/49aad927/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/schema/kafka/BeamKafkaCSVTableTest.java
--
diff --git 
a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/schema/kafka/BeamKafkaCSVTableTest.java
 
b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/schema/kafka/BeamKafkaCSVTableTest.java
deleted file mode 100644
index e5d81fa..000
--- 
a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/schema/kafka/BeamKafkaCSVTableTest.java
+++ /dev/null
@@ -1,107 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by 

[5/5] beam git commit: [BEAM-2740] This closes #3708

[takidau]

[BEAM-2740] This closes #3708


Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/a1cc5518
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/a1cc5518
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/a1cc5518

Branch: refs/heads/DSL_SQL
Commit: a1cc5518e0c1cc83c33c3e2e87128fc59935a314
Parents: 9eec6a0 49aad92
Author: Tyler Akidau 
Authored: Tue Aug 15 11:40:59 2017 -0700
Committer: Tyler Akidau 
Committed: Tue Aug 15 11:40:59 2017 -0700

--
 .../sdk/extensions/sql/BeamRecordSqlType.java   | 185 
 .../apache/beam/sdk/extensions/sql/BeamSql.java | 113 ++---
 .../beam/sdk/extensions/sql/BeamSqlCli.java |  65 ---
 .../beam/sdk/extensions/sql/BeamSqlEnv.java | 127 --
 .../sdk/extensions/sql/BeamSqlRecordHelper.java | 217 +
 .../beam/sdk/extensions/sql/BeamSqlUdf.java |  41 ++
 .../extensions/sql/example/BeamSqlExample.java  |   2 +-
 .../sdk/extensions/sql/impl/BeamSqlCli.java |  65 +++
 .../sdk/extensions/sql/impl/BeamSqlEnv.java | 135 ++
 .../sdk/extensions/sql/impl/package-info.java   |  22 +
 .../sql/impl/planner/BeamQueryPlanner.java  |   9 +-
 .../sql/impl/rel/BeamAggregationRel.java|   4 +-
 .../extensions/sql/impl/rel/BeamFilterRel.java  |   2 +-
 .../extensions/sql/impl/rel/BeamIOSinkRel.java  |   6 +-
 .../sql/impl/rel/BeamIOSourceRel.java   |   6 +-
 .../sql/impl/rel/BeamIntersectRel.java  |   2 +-
 .../extensions/sql/impl/rel/BeamJoinRel.java|   4 +-
 .../extensions/sql/impl/rel/BeamMinusRel.java   |   2 +-
 .../extensions/sql/impl/rel/BeamProjectRel.java |   2 +-
 .../extensions/sql/impl/rel/BeamRelNode.java|   5 +-
 .../sql/impl/rel/BeamSetOperatorRelBase.java|   2 +-
 .../extensions/sql/impl/rel/BeamSortRel.java|   4 +-
 .../extensions/sql/impl/rel/BeamUnionRel.java   |   2 +-
 .../extensions/sql/impl/rel/BeamValuesRel.java  |   6 +-
 .../sql/impl/schema/BaseBeamTable.java  |  35 ++
 .../extensions/sql/impl/schema/BeamIOType.java  |  28 ++
 .../sql/impl/schema/BeamPCollectionTable.java   |  63 +++
 .../sql/impl/schema/BeamSqlTable.java   |  54 +++
 .../sql/impl/schema/BeamTableUtils.java | 118 +
 .../impl/schema/kafka/BeamKafkaCSVTable.java| 109 +
 .../sql/impl/schema/kafka/BeamKafkaTable.java   | 109 +
 .../sql/impl/schema/kafka/package-info.java |  22 +
 .../sql/impl/schema/package-info.java   |  22 +
 .../sql/impl/schema/text/BeamTextCSVTable.java  |  70 +++
 .../schema/text/BeamTextCSVTableIOReader.java   |  58 +++
 .../schema/text/BeamTextCSVTableIOWriter.java   |  58 +++
 .../sql/impl/schema/text/BeamTextTable.java |  41 ++
 .../sql/impl/schema/text/package-info.java  |  22 +
 .../transform/BeamAggregationTransforms.java|   4 +-
 .../sql/impl/transform/BeamJoinTransforms.java  |   4 +-
 .../sql/impl/transform/BeamSqlProjectFn.java|   4 +-
 .../extensions/sql/impl/utils/CalciteUtils.java |   2 +-
 .../extensions/sql/schema/BaseBeamTable.java|  34 --
 .../sdk/extensions/sql/schema/BeamIOType.java   |  28 --
 .../sql/schema/BeamPCollectionTable.java|  62 ---
 .../sql/schema/BeamRecordSqlType.java   | 185 
 .../sql/schema/BeamSqlRecordHelper.java | 217 -
 .../sdk/extensions/sql/schema/BeamSqlTable.java |  53 ---
 .../sdk/extensions/sql/schema/BeamSqlUdf.java   |  41 --
 .../extensions/sql/schema/BeamTableUtils.java   | 117 -
 .../sql/schema/kafka/BeamKafkaCSVTable.java | 109 -
 .../sql/schema/kafka/BeamKafkaTable.java| 109 -
 .../sql/schema/kafka/package-info.java  |  22 -
 .../sdk/extensions/sql/schema/package-info.java |  22 -
 .../sql/schema/text/BeamTextCSVTable.java   |  70 ---
 .../schema/text/BeamTextCSVTableIOReader.java   |  58 ---
 .../schema/text/BeamTextCSVTableIOWriter.java   |  58 ---
 .../sql/schema/text/BeamTextTable.java  |  41 --
 .../sql/schema/text/package-info.java   |  22 -
 .../extensions/sql/BeamSqlApiSurfaceTest.java   |  12 +-
 .../sql/BeamSqlDslAggregationTest.java  |   1 -
 .../beam/sdk/extensions/sql/BeamSqlDslBase.java |   1 -
 .../sdk/extensions/sql/BeamSqlDslJoinTest.java  |   1 -
 .../extensions/sql/BeamSqlDslProjectTest.java   |   1 -
 .../extensions/sql/BeamSqlDslUdfUdafTest.java   |   2 -
 .../beam/sdk/extensions/sql/TestUtils.java  |   1 -
 .../interpreter/BeamSqlFnExecutorTestBase.java  |   2 +-
 .../extensions/sql/impl/rel/BaseRelTest.java|  34 ++
 .../sql/impl/rel/BeamIntersectRelTest.java  |   9 +-
 .../rel/BeamJoinRelBoundedVsBoundedTest.java|  23 +-
 .../rel/BeamJoinRelUnboundedVsBoundedTest.java  |  25 +-
 .../BeamJoinRelUnboundedVsUnboundedTest.java|  19 +-
 .../sql/impl/rel/BeamMinusRelTest.java  |   9 +-
 .../impl/rel/BeamSetOperatorRelBaseTest.java|   9 +-
 

[2/2] beam git commit: [BEAM-2747] This closes #3716

[takidau]

[BEAM-2747] This closes #3716


Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/9eec6a03
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/9eec6a03
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/9eec6a03

Branch: refs/heads/DSL_SQL
Commit: 9eec6a030a4d14b504730b57a8d863a5bd97468e
Parents: f37a7a1 1770c86
Author: Tyler Akidau 
Authored: Fri Aug 11 17:12:29 2017 -0700
Committer: Tyler Akidau 
Committed: Fri Aug 11 17:12:29 2017 -0700

--
 .../apache/beam/sdk/coders/BeamRecordCoder.java |  16 +-
 .../apache/beam/sdk/extensions/sql/BeamSql.java |  22 +-
 .../beam/sdk/extensions/sql/BeamSqlEnv.java |  11 +-
 .../operator/BeamSqlInputRefExpression.java |   4 +
 .../sql/impl/interpreter/operator/UdafImpl.java |  87 
 .../transform/BeamAggregationTransforms.java|  44 +-
 .../impl/transform/BeamBuiltinAggregations.java | 504 +++
 .../sdk/extensions/sql/schema/BeamSqlUdaf.java  |  72 ---
 .../extensions/sql/BeamSqlDslUdfUdafTest.java   |  22 +-
 9 files changed, 344 insertions(+), 438 deletions(-)
--

[1/2] beam git commit: take CombineFn as UDAF.

[takidau]

Repository: beam
Updated Branches:
  refs/heads/DSL_SQL f37a7a19c -> 9eec6a030


take CombineFn as UDAF.


Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/1770c861
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/1770c861
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/1770c861

Branch: refs/heads/DSL_SQL
Commit: 1770c86121d7edc388cadc0e2791c19b027cc50f
Parents: f37a7a1
Author: mingmxu 
Authored: Thu Aug 10 17:42:29 2017 -0700
Committer: Tyler Akidau 
Committed: Fri Aug 11 17:11:23 2017 -0700

--
 .../apache/beam/sdk/coders/BeamRecordCoder.java |  16 +-
 .../apache/beam/sdk/extensions/sql/BeamSql.java |  22 +-
 .../beam/sdk/extensions/sql/BeamSqlEnv.java |  11 +-
 .../operator/BeamSqlInputRefExpression.java |   4 +
 .../sql/impl/interpreter/operator/UdafImpl.java |  87 
 .../transform/BeamAggregationTransforms.java|  44 +-
 .../impl/transform/BeamBuiltinAggregations.java | 504 +++
 .../sdk/extensions/sql/schema/BeamSqlUdaf.java  |  72 ---
 .../extensions/sql/BeamSqlDslUdfUdafTest.java   |  22 +-
 9 files changed, 344 insertions(+), 438 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/beam/blob/1770c861/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/BeamRecordCoder.java
--
diff --git 
a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/BeamRecordCoder.java 
b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/BeamRecordCoder.java
index cbed87d..7b1b681 100644
--- 
a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/BeamRecordCoder.java
+++ 
b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/BeamRecordCoder.java
@@ -35,11 +35,11 @@ public class BeamRecordCoder extends 
CustomCoder {
   private static final BitSetCoder nullListCoder = BitSetCoder.of();
 
   private BeamRecordType recordType;
-  private List coderArray;
+  private List coders;
 
-  private BeamRecordCoder(BeamRecordType recordType, List coderArray) {
+  private BeamRecordCoder(BeamRecordType recordType, List coders) {
 this.recordType = recordType;
-this.coderArray = coderArray;
+this.coders = coders;
   }
 
   public static BeamRecordCoder of(BeamRecordType recordType, List 
coderArray){
@@ -62,7 +62,7 @@ public class BeamRecordCoder extends CustomCoder {
 continue;
   }
 
-  coderArray.get(idx).encode(value.getFieldValue(idx), outStream);
+  coders.get(idx).encode(value.getFieldValue(idx), outStream);
 }
   }
 
@@ -75,7 +75,7 @@ public class BeamRecordCoder extends CustomCoder {
   if (nullFields.get(idx)) {
 fieldValues.add(null);
   } else {
-fieldValues.add(coderArray.get(idx).decode(inStream));
+fieldValues.add(coders.get(idx).decode(inStream));
   }
 }
 BeamRecord record = new BeamRecord(recordType, fieldValues);
@@ -99,8 +99,12 @@ public class BeamRecordCoder extends CustomCoder 
{
   @Override
   public void verifyDeterministic()
   throws org.apache.beam.sdk.coders.Coder.NonDeterministicException {
-for (Coder c : coderArray) {
+for (Coder c : coders) {
   c.verifyDeterministic();
 }
   }
+
+  public List getCoders() {
+return coders;
+  }
 }

http://git-wip-us.apache.org/repos/asf/beam/blob/1770c861/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/BeamSql.java
--
diff --git 
a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/BeamSql.java
 
b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/BeamSql.java
index a1e9877..bf6a9c0 100644
--- 
a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/BeamSql.java
+++ 
b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/BeamSql.java
@@ -23,8 +23,8 @@ import org.apache.beam.sdk.coders.BeamRecordCoder;
 import org.apache.beam.sdk.extensions.sql.impl.rel.BeamRelNode;
 import org.apache.beam.sdk.extensions.sql.schema.BeamPCollectionTable;
 import org.apache.beam.sdk.extensions.sql.schema.BeamRecordSqlType;
-import org.apache.beam.sdk.extensions.sql.schema.BeamSqlUdaf;
 import org.apache.beam.sdk.extensions.sql.schema.BeamSqlUdf;
+import org.apache.beam.sdk.transforms.Combine.CombineFn;
 import org.apache.beam.sdk.transforms.PTransform;
 import org.apache.beam.sdk.transforms.SerializableFunction;
 import org.apache.beam.sdk.values.BeamRecord;
@@ -155,10 +155,10 @@ public class BeamSql {
   }
 
  /**
-  * register a UDAF function used in this query.
+  * register a {@link CombineFn} as UDAF function used in this query.
   */
- public QueryTransform withUdaf(String functionName, Class clazz){
-   

[2/2] beam git commit: [BEAM-2741] This closes #3710

[takidau]

[BEAM-2741] This closes #3710


Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/f37a7a19
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/f37a7a19
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/f37a7a19

Branch: refs/heads/DSL_SQL
Commit: f37a7a19c35cba03225694af7491a89dbfe06de3
Parents: 2a1377e cd27036
Author: Tyler Akidau 
Authored: Thu Aug 10 13:59:45 2017 -0700
Committer: Tyler Akidau 
Committed: Thu Aug 10 13:59:45 2017 -0700

--
 .../org/apache/beam/sdk/values/BeamRecord.java  | 118 ++-
 .../apache/beam/sdk/values/BeamRecordType.java  |  60 ++
 .../operator/BeamSqlUdfExpression.java  |   5 +-
 3 files changed, 159 insertions(+), 24 deletions(-)
--

[1/2] beam git commit: update JavaDoc for BeamRecord, BeamRecordType. Also only create new UDF class instances for SerializableFunction UDFs.

[takidau]

Repository: beam
Updated Branches:
  refs/heads/DSL_SQL 2a1377e1c -> f37a7a19c


update JavaDoc for BeamRecord, BeamRecordType.
Also only create new UDF class instances for SerializableFunction UDFs.


Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/cd27036a
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/cd27036a
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/cd27036a

Branch: refs/heads/DSL_SQL
Commit: cd27036a1c537a9059f13bbcdaec0264481ebd0b
Parents: 2a1377e
Author: mingmxu 
Authored: Wed Aug 9 13:30:16 2017 -0700
Committer: Tyler Akidau 
Committed: Thu Aug 10 13:58:59 2017 -0700

--
 .../org/apache/beam/sdk/values/BeamRecord.java  | 118 ++-
 .../apache/beam/sdk/values/BeamRecordType.java  |  60 ++
 .../operator/BeamSqlUdfExpression.java  |   5 +-
 3 files changed, 159 insertions(+), 24 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/beam/blob/cd27036a/sdks/java/core/src/main/java/org/apache/beam/sdk/values/BeamRecord.java
--
diff --git 
a/sdks/java/core/src/main/java/org/apache/beam/sdk/values/BeamRecord.java 
b/sdks/java/core/src/main/java/org/apache/beam/sdk/values/BeamRecord.java
index fa3b574..fd26f46 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/values/BeamRecord.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/values/BeamRecord.java
@@ -25,11 +25,17 @@ import java.util.Date;
 import java.util.GregorianCalendar;
 import java.util.List;
 import org.apache.beam.sdk.annotations.Experimental;
+import org.apache.beam.sdk.coders.BeamRecordCoder;
 
 /**
- * {@link org.apache.beam.sdk.values.BeamRecord}, self-described with
- * {@link BeamRecordType}, represents one element in a
- * {@link org.apache.beam.sdk.values.PCollection}.
+ * {@link BeamRecord} is an immutable tuple-like type to represent one element 
in a
+ * {@link PCollection}. The fields are described with a {@link BeamRecordType}.
+ *
+ * By default, {@link BeamRecordType} only contains the name for each 
field. It
+ * can be extended to support more sophisticated validation by overwriting
+ * {@link BeamRecordType#validateValueType(int, Object)}.
+ *
+ * A Coder {@link BeamRecordCoder} is provided, which wraps the Coder for 
each data field.
  */
 @Experimental
 public class BeamRecord implements Serializable {
@@ -63,6 +69,9 @@ public class BeamRecord implements Serializable {
 }
   }
 
+  /**
+   * see {@link #BeamRecord(BeamRecordType, List)}.
+   */
   public BeamRecord(BeamRecordType dataType, Object... rawdataValues) {
 this(dataType, Arrays.asList(rawdataValues));
   }
@@ -72,110 +81,213 @@ public class BeamRecord implements Serializable {
 dataValues.set(index, fieldValue);
   }
 
+  /**
+   * Get value by field name.
+   */
   public Object getFieldValue(String fieldName) {
 return getFieldValue(dataType.getFieldNames().indexOf(fieldName));
   }
 
+  /**
+   * Get a {@link Byte} value by field name, {@link ClassCastException} is 
thrown
+   * if type doesn't match.
+   */
   public Byte getByte(String fieldName) {
 return (Byte) getFieldValue(fieldName);
   }
 
+  /**
+   * Get a {@link Short} value by field name, {@link ClassCastException} is 
thrown
+   * if type doesn't match.
+   */
   public Short getShort(String fieldName) {
 return (Short) getFieldValue(fieldName);
   }
 
+  /**
+   * Get a {@link Integer} value by field name, {@link ClassCastException} is 
thrown
+   * if type doesn't match.
+   */
   public Integer getInteger(String fieldName) {
 return (Integer) getFieldValue(fieldName);
   }
 
+  /**
+   * Get a {@link Float} value by field name, {@link ClassCastException} is 
thrown
+   * if type doesn't match.
+   */
   public Float getFloat(String fieldName) {
 return (Float) getFieldValue(fieldName);
   }
 
+  /**
+   * Get a {@link Double} value by field name, {@link ClassCastException} is 
thrown
+   * if type doesn't match.
+   */
   public Double getDouble(String fieldName) {
 return (Double) getFieldValue(fieldName);
   }
 
+  /**
+   * Get a {@link Long} value by field name, {@link ClassCastException} is 
thrown
+   * if type doesn't match.
+   */
   public Long getLong(String fieldName) {
 return (Long) getFieldValue(fieldName);
   }
 
+  /**
+   * Get a {@link String} value by field name, {@link ClassCastException} is 
thrown
+   * if type doesn't match.
+   */
   public String getString(String fieldName) {
 return (String) getFieldValue(fieldName);
   }
 
+  /**
+   * Get a {@link Date} value by field name, {@link ClassCastException} is 
thrown
+   * if type doesn't match.
+   */
   public Date getDate(String fieldName) {
 return (Date) getFieldValue(fieldName);
   }
 
+  

[2/2] beam git commit: [BEAM-2744] This closes #3702

[takidau]

[BEAM-2744] This closes #3702


Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/2a1377e1
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/2a1377e1
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/2a1377e1

Branch: refs/heads/DSL_SQL
Commit: 2a1377e1c3aae2234f20b38c2203a2d8d7c78f6b
Parents: be01be5 affb8f6
Author: Tyler Akidau 
Authored: Thu Aug 10 13:23:37 2017 -0700
Committer: Tyler Akidau 
Committed: Thu Aug 10 13:23:37 2017 -0700

--
 .../apache/beam/sdk/coders/BeamRecordCoder.java | 12 +--
 .../org/apache/beam/sdk/values/BeamRecord.java  | 22 +---
 .../apache/beam/sdk/values/BeamRecordType.java  |  7 ++-
 .../extensions/sql/impl/rel/BeamJoinRel.java|  2 +-
 .../extensions/sql/impl/rel/BeamValuesRel.java  |  2 +-
 .../sql/impl/transform/BeamJoinTransforms.java  |  4 ++--
 .../extensions/sql/schema/BeamTableUtils.java   | 10 -
 .../sql/BeamSqlDslAggregationTest.java  | 16 +++---
 .../beam/sdk/extensions/sql/TestUtils.java  |  6 +++---
 9 files changed, 38 insertions(+), 43 deletions(-)
--

[1/2] beam git commit: [BEAM-2744] rename BeamRecordType#size()

[takidau]

Repository: beam
Updated Branches:
  refs/heads/DSL_SQL be01be5cb -> 2a1377e1c


[BEAM-2744] rename BeamRecordType#size()


Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/affb8f6e
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/affb8f6e
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/affb8f6e

Branch: refs/heads/DSL_SQL
Commit: affb8f6e9f1fb3e4df79c787528b7af726100d29
Parents: be01be5
Author: James Xu 
Authored: Tue Aug 8 15:15:59 2017 +0800
Committer: Tyler Akidau 
Committed: Thu Aug 10 13:06:15 2017 -0700

--
 .../apache/beam/sdk/coders/BeamRecordCoder.java | 12 +--
 .../org/apache/beam/sdk/values/BeamRecord.java  | 22 +---
 .../apache/beam/sdk/values/BeamRecordType.java  |  7 ++-
 .../extensions/sql/impl/rel/BeamJoinRel.java|  2 +-
 .../extensions/sql/impl/rel/BeamValuesRel.java  |  2 +-
 .../sql/impl/transform/BeamJoinTransforms.java  |  4 ++--
 .../extensions/sql/schema/BeamTableUtils.java   | 10 -
 .../sql/BeamSqlDslAggregationTest.java  | 16 +++---
 .../beam/sdk/extensions/sql/TestUtils.java  |  6 +++---
 9 files changed, 38 insertions(+), 43 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/beam/blob/affb8f6e/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/BeamRecordCoder.java
--
diff --git 
a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/BeamRecordCoder.java 
b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/BeamRecordCoder.java
index 4e24b82..cbed87d 100644
--- 
a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/BeamRecordCoder.java
+++ 
b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/BeamRecordCoder.java
@@ -43,7 +43,7 @@ public class BeamRecordCoder extends CustomCoder {
   }
 
   public static BeamRecordCoder of(BeamRecordType recordType, List 
coderArray){
-if (recordType.size() != coderArray.size()) {
+if (recordType.getFieldCount() != coderArray.size()) {
   throw new IllegalArgumentException("Coder size doesn't match with field 
size");
 }
 return new BeamRecordCoder(recordType, coderArray);
@@ -57,7 +57,7 @@ public class BeamRecordCoder extends CustomCoder {
   public void encode(BeamRecord value, OutputStream outStream)
   throws CoderException, IOException {
 nullListCoder.encode(scanNullFields(value), outStream);
-for (int idx = 0; idx < value.size(); ++idx) {
+for (int idx = 0; idx < value.getFieldCount(); ++idx) {
   if (value.getFieldValue(idx) == null) {
 continue;
   }
@@ -70,8 +70,8 @@ public class BeamRecordCoder extends CustomCoder {
   public BeamRecord decode(InputStream inStream) throws CoderException, 
IOException {
 BitSet nullFields = nullListCoder.decode(inStream);
 
-List fieldValues = new ArrayList<>(recordType.size());
-for (int idx = 0; idx < recordType.size(); ++idx) {
+List fieldValues = new ArrayList<>(recordType.getFieldCount());
+for (int idx = 0; idx < recordType.getFieldCount(); ++idx) {
   if (nullFields.get(idx)) {
 fieldValues.add(null);
   } else {
@@ -87,8 +87,8 @@ public class BeamRecordCoder extends CustomCoder {
* Scan {@link BeamRecord} to find fields with a NULL value.
*/
   private BitSet scanNullFields(BeamRecord record){
-BitSet nullFields = new BitSet(record.size());
-for (int idx = 0; idx < record.size(); ++idx) {
+BitSet nullFields = new BitSet(record.getFieldCount());
+for (int idx = 0; idx < record.getFieldCount(); ++idx) {
   if (record.getFieldValue(idx) == null) {
 nullFields.set(idx);
   }

http://git-wip-us.apache.org/repos/asf/beam/blob/affb8f6e/sdks/java/core/src/main/java/org/apache/beam/sdk/values/BeamRecord.java
--
diff --git 
a/sdks/java/core/src/main/java/org/apache/beam/sdk/values/BeamRecord.java 
b/sdks/java/core/src/main/java/org/apache/beam/sdk/values/BeamRecord.java
index a3ede3c..fa3b574 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/values/BeamRecord.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/values/BeamRecord.java
@@ -52,13 +52,13 @@ public class BeamRecord implements Serializable {
 }
 
 this.dataType = dataType;
-this.dataValues = new ArrayList<>(dataType.size());
+this.dataValues = new ArrayList<>(dataType.getFieldCount());
 
-for (int idx = 0; idx < dataType.size(); ++idx) {
+for (int idx = 0; idx < dataType.getFieldCount(); ++idx) {
   dataValues.add(null);
 }
 
-for (int idx = 0; idx < dataType.size(); ++idx) {
+for (int idx = 0; idx < dataType.getFieldCount(); ++idx) {
   addField(idx, rawDataValues.get(idx));
 

[1/2] beam git commit: take SerializableFunction as UDF.

[takidau]

Repository: beam
Updated Branches:
  refs/heads/DSL_SQL 3e25ffb04 -> be01be5cb


take SerializableFunction as UDF.


Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/ffbd4c2f
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/ffbd4c2f
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/ffbd4c2f

Branch: refs/heads/DSL_SQL
Commit: ffbd4c2f7ae0a608f061e718aefa23d3349e34a5
Parents: 3e25ffb
Author: mingmxu 
Authored: Tue Aug 8 23:02:32 2017 -0700
Committer: Tyler Akidau 
Committed: Wed Aug 9 13:27:20 2017 -0700

--
 .../apache/beam/sdk/extensions/sql/BeamSql.java| 17 +
 .../apache/beam/sdk/extensions/sql/BeamSqlEnv.java |  9 +
 .../interpreter/operator/BeamSqlUdfExpression.java |  4 +++-
 .../sdk/extensions/sql/BeamSqlDslUdfUdafTest.java  | 13 -
 4 files changed, 41 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/beam/blob/ffbd4c2f/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/BeamSql.java
--
diff --git 
a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/BeamSql.java
 
b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/BeamSql.java
index ac617ad..a1e9877 100644
--- 
a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/BeamSql.java
+++ 
b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/BeamSql.java
@@ -26,6 +26,7 @@ import 
org.apache.beam.sdk.extensions.sql.schema.BeamRecordSqlType;
 import org.apache.beam.sdk.extensions.sql.schema.BeamSqlUdaf;
 import org.apache.beam.sdk.extensions.sql.schema.BeamSqlUdf;
 import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.transforms.SerializableFunction;
 import org.apache.beam.sdk.values.BeamRecord;
 import org.apache.beam.sdk.values.PCollection;
 import org.apache.beam.sdk.values.PCollectionTuple;
@@ -144,6 +145,14 @@ public class BeamSql {
getSqlEnv().registerUdf(functionName, clazz);
return this;
  }
+ /**
+  * register {@link SerializableFunction} as a UDF function used in this 
query.
+  * Note, {@link SerializableFunction} must have a constructor without 
arguments.
+  */
+  public QueryTransform withUdf(String functionName, SerializableFunction 
sfn){
+getSqlEnv().registerUdf(functionName, sfn);
+return this;
+  }
 
  /**
   * register a UDAF function used in this query.
@@ -213,6 +222,14 @@ public class BeamSql {
getSqlEnv().registerUdf(functionName, clazz);
return this;
  }
+ /**
+  * register {@link SerializableFunction} as a UDF function used in this 
query.
+  * Note, {@link SerializableFunction} must have a constructor without 
arguments.
+  */
+  public SimpleQueryTransform withUdf(String functionName, 
SerializableFunction sfn){
+getSqlEnv().registerUdf(functionName, sfn);
+return this;
+  }
 
  /**
   * register a UDAF function used in this query.

http://git-wip-us.apache.org/repos/asf/beam/blob/ffbd4c2f/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/BeamSqlEnv.java
--
diff --git 
a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/BeamSqlEnv.java
 
b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/BeamSqlEnv.java
index 4d21425..0737c49 100644
--- 
a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/BeamSqlEnv.java
+++ 
b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/BeamSqlEnv.java
@@ -24,6 +24,7 @@ import 
org.apache.beam.sdk.extensions.sql.schema.BaseBeamTable;
 import org.apache.beam.sdk.extensions.sql.schema.BeamRecordSqlType;
 import org.apache.beam.sdk.extensions.sql.schema.BeamSqlUdaf;
 import org.apache.beam.sdk.extensions.sql.schema.BeamSqlUdf;
+import org.apache.beam.sdk.transforms.SerializableFunction;
 import org.apache.calcite.DataContext;
 import org.apache.calcite.linq4j.Enumerable;
 import org.apache.calcite.rel.type.RelDataType;
@@ -60,6 +61,14 @@ public class BeamSqlEnv implements Serializable{
   }
 
   /**
+   * register {@link SerializableFunction} as a UDF function which can be used 
in SQL expression.
+   * Note, {@link SerializableFunction} must have a constructor without 
arguments.
+   */
+  public void registerUdf(String functionName, SerializableFunction sfn) {
+schema.add(functionName, ScalarFunctionImpl.create(sfn.getClass(), 
"apply"));
+  }
+
+  /**
* Register a UDAF function which can be used in GROUP-BY expression.
* See {@link BeamSqlUdaf} on how to 

[2/2] beam git commit: [BEAM-2748] This closes #3707

[takidau]

[BEAM-2748] This closes #3707


Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/be01be5c
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/be01be5c
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/be01be5c

Branch: refs/heads/DSL_SQL
Commit: be01be5cb5ebf494718c09d0518ac77fdc2b4efd
Parents: 3e25ffb ffbd4c2
Author: Tyler Akidau 
Authored: Wed Aug 9 13:27:52 2017 -0700
Committer: Tyler Akidau 
Committed: Wed Aug 9 13:27:52 2017 -0700

--
 .../apache/beam/sdk/extensions/sql/BeamSql.java| 17 +
 .../apache/beam/sdk/extensions/sql/BeamSqlEnv.java |  9 +
 .../interpreter/operator/BeamSqlUdfExpression.java |  4 +++-
 .../sdk/extensions/sql/BeamSqlDslUdfUdafTest.java  | 13 -
 4 files changed, 41 insertions(+), 2 deletions(-)
--

[2/2] beam git commit: [BEAM-2749] This closes #3706

[takidau]

[BEAM-2749] This closes #3706


Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/3e25ffb0
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/3e25ffb0
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/3e25ffb0

Branch: refs/heads/DSL_SQL
Commit: 3e25ffb04e3190a67174bbe3883c3ca982840663
Parents: f59dccc ee1f97b
Author: Tyler Akidau 
Authored: Wed Aug 9 09:52:51 2017 -0700
Committer: Tyler Akidau 
Committed: Wed Aug 9 09:52:51 2017 -0700

--
 .../extensions/sql/example/BeamSqlExample.java  | 24 
 1 file changed, 15 insertions(+), 9 deletions(-)
--

[1/2] beam git commit: Update BeamSqlExample: - Fix mvn example - Add aggregation/group by

[takidau]

Repository: beam
Updated Branches:
  refs/heads/DSL_SQL f59dccc51 -> 3e25ffb04


Update BeamSqlExample:
- Fix mvn example
- Add aggregation/group by


Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/ee1f97bb
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/ee1f97bb
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/ee1f97bb

Branch: refs/heads/DSL_SQL
Commit: ee1f97bb4bf5baa1810a787d0b0e68f314e7c110
Parents: f59dccc
Author: mingmxu 
Authored: Tue Aug 8 21:52:54 2017 -0700
Committer: Tyler Akidau 
Committed: Wed Aug 9 09:50:37 2017 -0700

--
 .../extensions/sql/example/BeamSqlExample.java  | 24 
 1 file changed, 15 insertions(+), 9 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/beam/blob/ee1f97bb/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/example/BeamSqlExample.java
--
diff --git 
a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/example/BeamSqlExample.java
 
b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/example/BeamSqlExample.java
index 3a46acc..91251cf 100644
--- 
a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/example/BeamSqlExample.java
+++ 
b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/example/BeamSqlExample.java
@@ -39,11 +39,10 @@ import org.apache.beam.sdk.values.TupleTag;
  *
  * Run the example with
  * 
- * mvn -pl dsls/sql compile exec:java \
- *  -Dexec.mainClass=BeamSqlExample \
+ * mvn -pl sdks/java/extensions/sql \
+ *   compile exec:java 
-Dexec.mainClass=org.apache.beam.sdk.extensions.sql.example.BeamSqlExample \
  *   -Dexec.args="--runner=DirectRunner" -Pdirect-runner
  * 
- *
  */
 class BeamSqlExample {
   public static void main(String[] args) throws Exception {
@@ -54,21 +53,26 @@ class BeamSqlExample {
 List fieldNames = Arrays.asList("c1", "c2", "c3");
 List fieldTypes = Arrays.asList(Types.INTEGER, Types.VARCHAR, 
Types.DOUBLE);
 BeamRecordSqlType type = BeamRecordSqlType.create(fieldNames, fieldTypes);
-BeamRecord row = new BeamRecord(type, 1, "row", 1.0);
+BeamRecord row1 = new BeamRecord(type, 1, "row", 1.0);
+BeamRecord row2 = new BeamRecord(type, 2, "row", 2.0);
+BeamRecord row3 = new BeamRecord(type, 3, "row", 3.0);
 
 //create a source PCollection with Create.of();
-PCollection inputTable = PBegin.in(p).apply(Create.of(row)
+PCollection inputTable = PBegin.in(p).apply(Create.of(row1, 
row2, row3)
 .withCoder(type.getRecordCoder()));
 
 //Case 1. run a simple SQL query over input PCollection with 
BeamSql.simpleQuery;
 PCollection outputStream = inputTable.apply(
-BeamSql.simpleQuery("select c1, c2, c3 from PCOLLECTION where c1=1"));
+BeamSql.simpleQuery("select c1, c2, c3 from PCOLLECTION where c1 > 
1"));
 
 //print the output record of case 1;
 outputStream.apply("log_result",
 MapElements.via(new SimpleFunction() {
   public Void apply(BeamRecord input) {
-System.out.println("PCOLLECTION: " + input);
+//expect output:
+//  PCOLLECTION: [3, row, 3.0]
+//  PCOLLECTION: [2, row, 2.0]
+System.out.println("PCOLLECTION: " + input.getDataValues());
 return null;
   }
 }));
@@ -76,14 +80,16 @@ class BeamSqlExample {
 //Case 2. run the query with BeamSql.query over result PCollection of case 
1.
 PCollection outputStream2 =
 PCollectionTuple.of(new TupleTag("CASE1_RESULT"), 
outputStream)
-.apply(BeamSql.query("select c2, c3 from CASE1_RESULT where c1=1"));
+.apply(BeamSql.query("select c2, sum(c3) from CASE1_RESULT group by 
c2"));
 
 //print the output record of case 2;
 outputStream2.apply("log_result",
 MapElements.via(new SimpleFunction() {
   @Override
   public Void apply(BeamRecord input) {
-System.out.println("TABLE_B: " + input);
+//expect output:
+//  CASE1_RESULT: [row, 5.0]
+System.out.println("CASE1_RESULT: " + input.getDataValues());
 return null;
   }
 }));

[3/3] beam git commit: [BEAM-2745] This closes #3700

[takidau]

[BEAM-2745] This closes #3700


Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/f59dccc5
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/f59dccc5
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/f59dccc5

Branch: refs/heads/DSL_SQL
Commit: f59dccc51abc8326bea3918766c62add93306fda
Parents: 6628674 7c76129
Author: Tyler Akidau 
Authored: Wed Aug 9 09:38:45 2017 -0700
Committer: Tyler Akidau 
Committed: Wed Aug 9 09:38:45 2017 -0700

--
 .../org/apache/beam/sdk/values/BeamRecord.java  |  21 ++-
 .../apache/beam/sdk/values/BeamRecordType.java  |  25 +--
 .../apache/beam/sdk/extensions/sql/BeamSql.java |   4 +-
 .../beam/sdk/extensions/sql/BeamSqlEnv.java |   6 +-
 .../extensions/sql/example/BeamSqlExample.java  |   4 +-
 .../sql/impl/rel/BeamAggregationRel.java|  16 +-
 .../extensions/sql/impl/rel/BeamJoinRel.java|  10 +-
 .../extensions/sql/impl/rel/BeamValuesRel.java  |   6 +-
 .../transform/BeamAggregationTransforms.java|  25 +--
 .../sql/impl/transform/BeamJoinTransforms.java  |  22 +--
 .../sql/impl/transform/BeamSqlProjectFn.java|   8 +-
 .../extensions/sql/impl/utils/CalciteUtils.java |  16 +-
 .../extensions/sql/schema/BaseBeamTable.java|   6 +-
 .../sql/schema/BeamPCollectionTable.java|   4 +-
 .../sql/schema/BeamRecordSqlType.java   | 185 +++
 .../sql/schema/BeamSqlRecordHelper.java |   4 +-
 .../sql/schema/BeamSqlRecordType.java   | 175 --
 .../sdk/extensions/sql/schema/BeamSqlTable.java |   2 +-
 .../extensions/sql/schema/BeamTableUtils.java   |  14 +-
 .../sql/schema/kafka/BeamKafkaCSVTable.java |  14 +-
 .../sql/schema/kafka/BeamKafkaTable.java|   6 +-
 .../sql/schema/text/BeamTextCSVTable.java   |   6 +-
 .../schema/text/BeamTextCSVTableIOReader.java   |   6 +-
 .../schema/text/BeamTextCSVTableIOWriter.java   |   6 +-
 .../sql/schema/text/BeamTextTable.java  |   4 +-
 .../sql/BeamSqlDslAggregationTest.java  |  14 +-
 .../beam/sdk/extensions/sql/BeamSqlDslBase.java |   6 +-
 .../sdk/extensions/sql/BeamSqlDslJoinTest.java  |  10 +-
 .../extensions/sql/BeamSqlDslProjectTest.java   |  10 +-
 .../extensions/sql/BeamSqlDslUdfUdafTest.java   |   6 +-
 .../beam/sdk/extensions/sql/TestUtils.java  |  14 +-
 .../interpreter/BeamSqlFnExecutorTestBase.java  |   4 +-
 ...mSqlBuiltinFunctionsIntegrationTestBase.java |   6 +-
 ...amSqlComparisonOperatorsIntegrationTest.java |   4 +-
 .../extensions/sql/mock/MockedBoundedTable.java |   6 +-
 .../sdk/extensions/sql/mock/MockedTable.java|   4 +-
 .../sql/mock/MockedUnboundedTable.java  |   4 +-
 .../sql/schema/BeamSqlRowCoderTest.java |   2 +-
 .../sql/schema/kafka/BeamKafkaCSVTableTest.java |   4 +-
 .../sql/schema/text/BeamTextCSVTableTest.java   |   4 +-
 .../transform/BeamAggregationTransformTest.java |  10 +-
 .../schema/transform/BeamTransformBaseTest.java |   8 +-
 42 files changed, 368 insertions(+), 343 deletions(-)
--

[2/3] beam git commit: [BEAM-2745] add BeamRecordSqlType.getFieldTypeByIndex()

[takidau]

[BEAM-2745] add BeamRecordSqlType.getFieldTypeByIndex()


Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/7c76129a
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/7c76129a
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/7c76129a

Branch: refs/heads/DSL_SQL
Commit: 7c76129a02c19595b257cff6efdc1fd0e637815d
Parents: 6628674
Author: James Xu 
Authored: Tue Aug 8 14:53:18 2017 +0800
Committer: Tyler Akidau 
Committed: Wed Aug 9 09:34:10 2017 -0700

--
 .../org/apache/beam/sdk/values/BeamRecord.java  |  21 ++-
 .../apache/beam/sdk/values/BeamRecordType.java  |  25 +--
 .../apache/beam/sdk/extensions/sql/BeamSql.java |   4 +-
 .../beam/sdk/extensions/sql/BeamSqlEnv.java |   6 +-
 .../extensions/sql/example/BeamSqlExample.java  |   4 +-
 .../sql/impl/rel/BeamAggregationRel.java|  16 +-
 .../extensions/sql/impl/rel/BeamJoinRel.java|  10 +-
 .../extensions/sql/impl/rel/BeamValuesRel.java  |   6 +-
 .../transform/BeamAggregationTransforms.java|  25 +--
 .../sql/impl/transform/BeamJoinTransforms.java  |  22 +--
 .../sql/impl/transform/BeamSqlProjectFn.java|   8 +-
 .../extensions/sql/impl/utils/CalciteUtils.java |  16 +-
 .../extensions/sql/schema/BaseBeamTable.java|   6 +-
 .../sql/schema/BeamPCollectionTable.java|   4 +-
 .../sql/schema/BeamRecordSqlType.java   | 185 +++
 .../sql/schema/BeamSqlRecordHelper.java |   4 +-
 .../sql/schema/BeamSqlRecordType.java   | 175 --
 .../sdk/extensions/sql/schema/BeamSqlTable.java |   2 +-
 .../extensions/sql/schema/BeamTableUtils.java   |  14 +-
 .../sql/schema/kafka/BeamKafkaCSVTable.java |  14 +-
 .../sql/schema/kafka/BeamKafkaTable.java|   6 +-
 .../sql/schema/text/BeamTextCSVTable.java   |   6 +-
 .../schema/text/BeamTextCSVTableIOReader.java   |   6 +-
 .../schema/text/BeamTextCSVTableIOWriter.java   |   6 +-
 .../sql/schema/text/BeamTextTable.java  |   4 +-
 .../sql/BeamSqlDslAggregationTest.java  |  14 +-
 .../beam/sdk/extensions/sql/BeamSqlDslBase.java |   6 +-
 .../sdk/extensions/sql/BeamSqlDslJoinTest.java  |  10 +-
 .../extensions/sql/BeamSqlDslProjectTest.java   |  10 +-
 .../extensions/sql/BeamSqlDslUdfUdafTest.java   |   6 +-
 .../beam/sdk/extensions/sql/TestUtils.java  |  14 +-
 .../interpreter/BeamSqlFnExecutorTestBase.java  |   4 +-
 ...mSqlBuiltinFunctionsIntegrationTestBase.java |   6 +-
 ...amSqlComparisonOperatorsIntegrationTest.java |   4 +-
 .../extensions/sql/mock/MockedBoundedTable.java |   6 +-
 .../sdk/extensions/sql/mock/MockedTable.java|   4 +-
 .../sql/mock/MockedUnboundedTable.java  |   4 +-
 .../sql/schema/BeamSqlRowCoderTest.java |   2 +-
 .../sql/schema/kafka/BeamKafkaCSVTableTest.java |   4 +-
 .../sql/schema/text/BeamTextCSVTableTest.java   |   4 +-
 .../transform/BeamAggregationTransformTest.java |  10 +-
 .../schema/transform/BeamTransformBaseTest.java |   8 +-
 42 files changed, 368 insertions(+), 343 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/beam/blob/7c76129a/sdks/java/core/src/main/java/org/apache/beam/sdk/values/BeamRecord.java
--
diff --git 
a/sdks/java/core/src/main/java/org/apache/beam/sdk/values/BeamRecord.java 
b/sdks/java/core/src/main/java/org/apache/beam/sdk/values/BeamRecord.java
index 6e4bd4c..a3ede3c 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/values/BeamRecord.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/values/BeamRecord.java
@@ -37,7 +37,20 @@ public class BeamRecord implements Serializable {
   private List dataValues;
   private BeamRecordType dataType;
 
-  public BeamRecord(BeamRecordType dataType, List rawdataValues) {
+  /**
+   * Creates a BeamRecord.
+   * @param dataType type of the record
+   * @param rawDataValues values of the record, record's size must match size 
of
+   *  the {@code BeamRecordType}, or can be null, if it is 
null
+   *  then every field is null.
+   */
+  public BeamRecord(BeamRecordType dataType, List rawDataValues) {
+if (dataType.getFieldNames().size() != rawDataValues.size()) {
+  throw new IllegalArgumentException(
+  "Field count in BeamRecordType(" + dataType.getFieldNames().size()
+  + ") and rawDataValues(" + rawDataValues.size() + ") must 
match!");
+}
+
 this.dataType = dataType;
 this.dataValues = new ArrayList<>(dataType.size());
 
@@ -46,7 +59,7 @@ public class BeamRecord implements Serializable {
 }
 
 for (int idx = 0; idx < dataType.size(); ++idx) {
-  addField(idx, rawdataValues.get(idx));
+  addField(idx, rawDataValues.get(idx));
 }
   }
 
@@ -60,7 +73,7 @@ 

[1/3] beam git commit: [BEAM-2745] add BeamRecordSqlType.getFieldTypeByIndex()

[takidau]

Repository: beam
Updated Branches:
  refs/heads/DSL_SQL 66286749f -> f59dccc51


http://git-wip-us.apache.org/repos/asf/beam/blob/7c76129a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/mock/MockedBoundedTable.java
--
diff --git 
a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/mock/MockedBoundedTable.java
 
b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/mock/MockedBoundedTable.java
index 073ca52..60e8211 100644
--- 
a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/mock/MockedBoundedTable.java
+++ 
b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/mock/MockedBoundedTable.java
@@ -26,7 +26,7 @@ import java.util.List;
 import java.util.concurrent.ConcurrentLinkedQueue;
 import org.apache.beam.sdk.Pipeline;
 import org.apache.beam.sdk.extensions.sql.schema.BeamIOType;
-import org.apache.beam.sdk.extensions.sql.schema.BeamSqlRecordType;
+import org.apache.beam.sdk.extensions.sql.schema.BeamRecordSqlType;
 import org.apache.beam.sdk.transforms.Create;
 import org.apache.beam.sdk.transforms.DoFn;
 import org.apache.beam.sdk.transforms.PTransform;
@@ -45,7 +45,7 @@ public class MockedBoundedTable extends MockedTable {
   /** rows flow out from this table. */
   private final List rows = new ArrayList<>();
 
-  public MockedBoundedTable(BeamSqlRecordType beamSqlRowType) {
+  public MockedBoundedTable(BeamRecordSqlType beamSqlRowType) {
 super(beamSqlRowType);
   }
 
@@ -69,7 +69,7 @@ public class MockedBoundedTable extends MockedTable {
   /**
* Build a mocked bounded table with the specified type.
*/
-  public static MockedBoundedTable of(final BeamSqlRecordType type) {
+  public static MockedBoundedTable of(final BeamRecordSqlType type) {
 return new MockedBoundedTable(type);
   }
 

http://git-wip-us.apache.org/repos/asf/beam/blob/7c76129a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/mock/MockedTable.java
--
diff --git 
a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/mock/MockedTable.java
 
b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/mock/MockedTable.java
index 59fc6e1..426789c 100644
--- 
a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/mock/MockedTable.java
+++ 
b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/mock/MockedTable.java
@@ -20,7 +20,7 @@ package org.apache.beam.sdk.extensions.sql.mock;
 
 import java.util.concurrent.atomic.AtomicInteger;
 import org.apache.beam.sdk.extensions.sql.schema.BaseBeamTable;
-import org.apache.beam.sdk.extensions.sql.schema.BeamSqlRecordType;
+import org.apache.beam.sdk.extensions.sql.schema.BeamRecordSqlType;
 import org.apache.beam.sdk.transforms.PTransform;
 import org.apache.beam.sdk.values.BeamRecord;
 import org.apache.beam.sdk.values.PCollection;
@@ -31,7 +31,7 @@ import org.apache.beam.sdk.values.PDone;
  */
 public abstract class MockedTable extends BaseBeamTable {
   public static final AtomicInteger COUNTER = new AtomicInteger();
-  public MockedTable(BeamSqlRecordType beamSqlRowType) {
+  public MockedTable(BeamRecordSqlType beamSqlRowType) {
 super(beamSqlRowType);
   }
 

http://git-wip-us.apache.org/repos/asf/beam/blob/7c76129a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/mock/MockedUnboundedTable.java
--
diff --git 
a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/mock/MockedUnboundedTable.java
 
b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/mock/MockedUnboundedTable.java
index 6194264..465705d 100644
--- 
a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/mock/MockedUnboundedTable.java
+++ 
b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/mock/MockedUnboundedTable.java
@@ -24,7 +24,7 @@ import java.util.List;
 import org.apache.beam.sdk.Pipeline;
 import org.apache.beam.sdk.extensions.sql.TestUtils;
 import org.apache.beam.sdk.extensions.sql.schema.BeamIOType;
-import org.apache.beam.sdk.extensions.sql.schema.BeamSqlRecordType;
+import org.apache.beam.sdk.extensions.sql.schema.BeamRecordSqlType;
 import org.apache.beam.sdk.testing.TestStream;
 import org.apache.beam.sdk.values.BeamRecord;
 import org.apache.beam.sdk.values.PCollection;
@@ -41,7 +41,7 @@ public class MockedUnboundedTable extends MockedTable {
   private final List> timestampedRows = new 
ArrayList<>();
   /** specify the index of column in the row which stands for the event time 
field. */
   private int timestampField;
-  private MockedUnboundedTable(BeamSqlRecordType beamSqlRowType) {
+  private MockedUnboundedTable(BeamRecordSqlType 

[1/2] beam git commit: [BEAM-2730] make BeamRecord an immutable type

[takidau]

Repository: beam
Updated Branches:
  refs/heads/DSL_SQL 926c70a34 -> 66286749f


[BEAM-2730] make BeamRecord an immutable type


Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/fa36b128
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/fa36b128
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/fa36b128

Branch: refs/heads/DSL_SQL
Commit: fa36b1285a4d8fc967b6302456d0f7fcf7943894
Parents: 926c70a
Author: mingmxu 
Authored: Mon Aug 7 16:12:21 2017 -0700
Committer: Tyler Akidau 
Committed: Tue Aug 8 19:07:03 2017 -0700

--
 .../apache/beam/sdk/coders/BeamRecordCoder.java |  10 +-
 .../org/apache/beam/sdk/values/BeamRecord.java  |  24 ++---
 .../extensions/sql/example/BeamSqlExample.java  |   5 +-
 .../extensions/sql/impl/rel/BeamJoinRel.java|   7 +-
 .../extensions/sql/impl/rel/BeamValuesRel.java  |   7 +-
 .../transform/BeamAggregationTransforms.java|  26 +++--
 .../sql/impl/transform/BeamJoinTransforms.java  |  20 ++--
 .../sql/impl/transform/BeamSqlProjectFn.java|   9 +-
 .../sql/schema/BeamSqlRecordType.java   |   5 +-
 .../extensions/sql/schema/BeamTableUtils.java   |  41 +++
 .../sql/BeamSqlDslAggregationTest.java  | 107 ---
 .../beam/sdk/extensions/sql/BeamSqlDslBase.java |  56 +++---
 .../extensions/sql/BeamSqlDslProjectTest.java   |  48 -
 .../extensions/sql/BeamSqlDslUdfUdafTest.java   |   8 +-
 .../beam/sdk/extensions/sql/TestUtils.java  |   6 +-
 .../interpreter/BeamSqlFnExecutorTestBase.java  |   8 +-
 .../sql/schema/BeamSqlRowCoderTest.java |  16 +--
 .../sql/schema/kafka/BeamKafkaCSVTableTest.java |  12 +--
 18 files changed, 132 insertions(+), 283 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/beam/blob/fa36b128/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/BeamRecordCoder.java
--
diff --git 
a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/BeamRecordCoder.java 
b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/BeamRecordCoder.java
index a6200f6..4e24b82 100644
--- 
a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/BeamRecordCoder.java
+++ 
b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/BeamRecordCoder.java
@@ -20,6 +20,7 @@ package org.apache.beam.sdk.coders;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
+import java.util.ArrayList;
 import java.util.BitSet;
 import java.util.List;
 import org.apache.beam.sdk.annotations.Experimental;
@@ -69,14 +70,15 @@ public class BeamRecordCoder extends 
CustomCoder {
   public BeamRecord decode(InputStream inStream) throws CoderException, 
IOException {
 BitSet nullFields = nullListCoder.decode(inStream);
 
-BeamRecord record = new BeamRecord(recordType);
+List fieldValues = new ArrayList<>(recordType.size());
 for (int idx = 0; idx < recordType.size(); ++idx) {
   if (nullFields.get(idx)) {
-continue;
+fieldValues.add(null);
+  } else {
+fieldValues.add(coderArray.get(idx).decode(inStream));
   }
-
-  record.addField(idx, coderArray.get(idx).decode(inStream));
 }
+BeamRecord record = new BeamRecord(recordType, fieldValues);
 
 return record;
   }

http://git-wip-us.apache.org/repos/asf/beam/blob/fa36b128/sdks/java/core/src/main/java/org/apache/beam/sdk/values/BeamRecord.java
--
diff --git 
a/sdks/java/core/src/main/java/org/apache/beam/sdk/values/BeamRecord.java 
b/sdks/java/core/src/main/java/org/apache/beam/sdk/values/BeamRecord.java
index 35a96f6..6e4bd4c 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/values/BeamRecord.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/values/BeamRecord.java
@@ -20,6 +20,7 @@ package org.apache.beam.sdk.values;
 import java.io.Serializable;
 import java.math.BigDecimal;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Date;
 import java.util.GregorianCalendar;
 import java.util.List;
@@ -32,29 +33,28 @@ import org.apache.beam.sdk.annotations.Experimental;
  */
 @Experimental
 public class BeamRecord implements Serializable {
+  //immutable list of field values.
   private List dataValues;
   private BeamRecordType dataType;
 
-  public BeamRecord(BeamRecordType dataType) {
+  public BeamRecord(BeamRecordType dataType, List rawdataValues) {
 this.dataType = dataType;
-this.dataValues = new ArrayList<>();
+this.dataValues = new ArrayList<>(dataType.size());
+
 for (int idx = 0; idx < dataType.size(); ++idx) {
   dataValues.add(null);
 }
-  }
 
-  public BeamRecord(BeamRecordType dataType, List dataValues) {
-this(dataType);
-

[2/2] beam git commit: [BEAM-2730] This closes #3692

[takidau]

[BEAM-2730] This closes #3692


Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/66286749
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/66286749
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/66286749

Branch: refs/heads/DSL_SQL
Commit: 66286749f2d64eb3f25d99b4d9bf56c49d9f62b6
Parents: 926c70a fa36b12
Author: Tyler Akidau 
Authored: Tue Aug 8 19:07:56 2017 -0700
Committer: Tyler Akidau 
Committed: Tue Aug 8 19:07:56 2017 -0700

--
 .../apache/beam/sdk/coders/BeamRecordCoder.java |  10 +-
 .../org/apache/beam/sdk/values/BeamRecord.java  |  24 ++---
 .../extensions/sql/example/BeamSqlExample.java  |   5 +-
 .../extensions/sql/impl/rel/BeamJoinRel.java|   7 +-
 .../extensions/sql/impl/rel/BeamValuesRel.java  |   7 +-
 .../transform/BeamAggregationTransforms.java|  26 +++--
 .../sql/impl/transform/BeamJoinTransforms.java  |  20 ++--
 .../sql/impl/transform/BeamSqlProjectFn.java|   9 +-
 .../sql/schema/BeamSqlRecordType.java   |   5 +-
 .../extensions/sql/schema/BeamTableUtils.java   |  41 +++
 .../sql/BeamSqlDslAggregationTest.java  | 107 ---
 .../beam/sdk/extensions/sql/BeamSqlDslBase.java |  56 +++---
 .../extensions/sql/BeamSqlDslProjectTest.java   |  48 -
 .../extensions/sql/BeamSqlDslUdfUdafTest.java   |   8 +-
 .../beam/sdk/extensions/sql/TestUtils.java  |   6 +-
 .../interpreter/BeamSqlFnExecutorTestBase.java  |   8 +-
 .../sql/schema/BeamSqlRowCoderTest.java |  16 +--
 .../sql/schema/kafka/BeamKafkaCSVTableTest.java |  12 +--
 18 files changed, 132 insertions(+), 283 deletions(-)
--

[2/2] beam git commit: [BEAM-2742] This closes #3699

[takidau]

[BEAM-2742] This closes #3699


Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/926c70a3
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/926c70a3
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/926c70a3

Branch: refs/heads/DSL_SQL
Commit: 926c70a345912b084fbb9366e9ecf2ecc4450739
Parents: d7120f0 990fedc
Author: Tyler Akidau 
Authored: Tue Aug 8 11:18:49 2017 -0700
Committer: Tyler Akidau 
Committed: Tue Aug 8 11:18:49 2017 -0700

--
 .../apache/beam/sdk/coders/BeamRecordCoder.java |  4 +-
 .../org/apache/beam/sdk/values/BeamRecord.java  | 39 
 .../extensions/sql/impl/rel/BeamSortRel.java|  8 ++--
 3 files changed, 23 insertions(+), 28 deletions(-)
--

[1/2] beam git commit: [BEAM-2742] change Field type from primitives to boxed types

[takidau]

Repository: beam
Updated Branches:
  refs/heads/DSL_SQL d7120f077 -> 926c70a34


[BEAM-2742] change Field type from primitives to boxed types


Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/990fedc9
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/990fedc9
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/990fedc9

Branch: refs/heads/DSL_SQL
Commit: 990fedc934da4eee2ec34c6a9ed5bd5a2dacbded
Parents: d7120f0
Author: James Xu 
Authored: Tue Aug 8 14:26:09 2017 +0800
Committer: Tyler Akidau 
Committed: Tue Aug 8 11:17:46 2017 -0700

--
 .../apache/beam/sdk/coders/BeamRecordCoder.java |  4 +-
 .../org/apache/beam/sdk/values/BeamRecord.java  | 39 
 .../extensions/sql/impl/rel/BeamSortRel.java|  8 ++--
 3 files changed, 23 insertions(+), 28 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/beam/blob/990fedc9/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/BeamRecordCoder.java
--
diff --git 
a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/BeamRecordCoder.java 
b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/BeamRecordCoder.java
index 40b9f3f..a6200f6 100644
--- 
a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/BeamRecordCoder.java
+++ 
b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/BeamRecordCoder.java
@@ -57,7 +57,7 @@ public class BeamRecordCoder extends CustomCoder {
   throws CoderException, IOException {
 nullListCoder.encode(scanNullFields(value), outStream);
 for (int idx = 0; idx < value.size(); ++idx) {
-  if (value.isNull(idx)) {
+  if (value.getFieldValue(idx) == null) {
 continue;
   }
 
@@ -87,7 +87,7 @@ public class BeamRecordCoder extends CustomCoder {
   private BitSet scanNullFields(BeamRecord record){
 BitSet nullFields = new BitSet(record.size());
 for (int idx = 0; idx < record.size(); ++idx) {
-  if (record.isNull(idx)) {
+  if (record.getFieldValue(idx) == null) {
 nullFields.set(idx);
   }
 }

http://git-wip-us.apache.org/repos/asf/beam/blob/990fedc9/sdks/java/core/src/main/java/org/apache/beam/sdk/values/BeamRecord.java
--
diff --git 
a/sdks/java/core/src/main/java/org/apache/beam/sdk/values/BeamRecord.java 
b/sdks/java/core/src/main/java/org/apache/beam/sdk/values/BeamRecord.java
index 6cbd11b..35a96f6 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/values/BeamRecord.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/values/BeamRecord.java
@@ -63,27 +63,27 @@ public class BeamRecord implements Serializable {
 return getFieldValue(dataType.getFieldsName().indexOf(fieldName));
   }
 
-  public byte getByte(String fieldName) {
+  public Byte getByte(String fieldName) {
 return (Byte) getFieldValue(fieldName);
   }
 
-  public short getShort(String fieldName) {
+  public Short getShort(String fieldName) {
 return (Short) getFieldValue(fieldName);
   }
 
-  public int getInteger(String fieldName) {
+  public Integer getInteger(String fieldName) {
 return (Integer) getFieldValue(fieldName);
   }
 
-  public float getFloat(String fieldName) {
+  public Float getFloat(String fieldName) {
 return (Float) getFieldValue(fieldName);
   }
 
-  public double getDouble(String fieldName) {
+  public Double getDouble(String fieldName) {
 return (Double) getFieldValue(fieldName);
   }
 
-  public long getLong(String fieldName) {
+  public Long getLong(String fieldName) {
 return (Long) getFieldValue(fieldName);
   }
 
@@ -103,35 +103,35 @@ public class BeamRecord implements Serializable {
 return (BigDecimal) getFieldValue(fieldName);
   }
 
-  public boolean getBoolean(String fieldName) {
-return (boolean) getFieldValue(fieldName);
+  public Boolean getBoolean(String fieldName) {
+return (Boolean) getFieldValue(fieldName);
   }
 
   public Object getFieldValue(int fieldIdx) {
 return dataValues.get(fieldIdx);
   }
 
-  public byte getByte(int idx) {
+  public Byte getByte(int idx) {
 return (Byte) getFieldValue(idx);
   }
 
-  public short getShort(int idx) {
+  public Short getShort(int idx) {
 return (Short) getFieldValue(idx);
   }
 
-  public int getInteger(int idx) {
+  public Integer getInteger(int idx) {
 return (Integer) getFieldValue(idx);
   }
 
-  public float getFloat(int idx) {
+  public Float getFloat(int idx) {
 return (Float) getFieldValue(idx);
   }
 
-  public double getDouble(int idx) {
+  public Double getDouble(int idx) {
 return (Double) getFieldValue(idx);
   }
 
-  public long getLong(int idx) {
+  public Long getLong(int idx) {
 return (Long) getFieldValue(idx);

[2/2] beam git commit: [BEAM-2733] This closes #3704

[takidau]

[BEAM-2733] This closes #3704


Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/d7120f07
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/d7120f07
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/d7120f07

Branch: refs/heads/DSL_SQL
Commit: d7120f07795973b9c782d180a2a9cc3c1affe658
Parents: 880531a 2a1fde3
Author: Tyler Akidau 
Authored: Tue Aug 8 11:13:05 2017 -0700
Committer: Tyler Akidau 
Committed: Tue Aug 8 11:13:05 2017 -0700

--
 .../org/apache/beam/sdk/extensions/sql/BeamSql.java | 16 +---
 1 file changed, 13 insertions(+), 3 deletions(-)
--

[1/2] beam git commit: [BEAM-2733] update javadoc for BeamSql

[takidau]

Repository: beam
Updated Branches:
  refs/heads/DSL_SQL 880531aa2 -> d7120f077


[BEAM-2733] update javadoc for BeamSql


Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/2a1fde3a
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/2a1fde3a
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/2a1fde3a

Branch: refs/heads/DSL_SQL
Commit: 2a1fde3aaf2bc54783fca18138bd7effba8c667e
Parents: 880531a
Author: James Xu 
Authored: Tue Aug 8 19:52:53 2017 +0800
Committer: Tyler Akidau 
Committed: Tue Aug 8 11:12:49 2017 -0700

--
 .../org/apache/beam/sdk/extensions/sql/BeamSql.java | 16 +---
 1 file changed, 13 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/beam/blob/2a1fde3a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/BeamSql.java
--
diff --git 
a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/BeamSql.java
 
b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/BeamSql.java
index 86e4d8d..d0a6360 100644
--- 
a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/BeamSql.java
+++ 
b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/BeamSql.java
@@ -71,6 +71,7 @@ 
outputTableB.apply(...).apply(TextIO.write().to("/my/output/path"));
 p.run().waitUntilFinish();
  * }
  * 
+ *
  */
 @Experimental
 public class BeamSql {
@@ -82,8 +83,14 @@ public class BeamSql {
* table. The {@link PCollectionTuple} contains the mapping from {@code 
table names} to
* {@code PCollection}, each representing an input table.
*
-   * It is an error to apply a {@link PCollectionTuple} missing any {@code 
table names}
-   * referenced within the query.
+   * 
+   * If the sql query only uses a subset of tables from the upstream 
{@link PCollectionTuple},
+   * this is valid;
+   * If the sql query references a table not included in the upstream 
{@link PCollectionTuple},
+   * an {@code IllegalStateException} is thrown during query 
validation;
+   * Always, tables from the upstream {@link PCollectionTuple} are only 
valid in the scope
+   * of the current query call.
+   * 
*/
   public static QueryTransform query(String sqlQuery) {
 return QueryTransform.builder()
@@ -100,7 +107,7 @@ public class BeamSql {
*
* Make sure to query it from a static table name PCOLLECTION.
*/
-  public static SimpleQueryTransform simpleQuery(String sqlQuery) throws 
Exception {
+  public static SimpleQueryTransform simpleQuery(String sqlQuery) {
 return SimpleQueryTransform.builder()
 .setSqlEnv(new BeamSqlEnv())
 .setSqlQuery(sqlQuery)
@@ -109,6 +116,9 @@ public class BeamSql {
 
   /**
* A {@link PTransform} representing an execution plan for a SQL query.
+   *
+   * The table names in the input {@code PCollectionTuple} are only valid 
during the current
+   * query.
*/
   @AutoValue
   public abstract static class QueryTransform extends

[2/2] beam git commit: [BEAM-2725] This closes #3691

[takidau]

[BEAM-2725] This closes #3691


Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/880531aa
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/880531aa
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/880531aa

Branch: refs/heads/DSL_SQL
Commit: 880531aa291d92cd275829acb043621c2bb100b8
Parents: 79880b6 4871edb
Author: Tyler Akidau 
Authored: Mon Aug 7 15:17:11 2017 -0700
Committer: Tyler Akidau 
Committed: Mon Aug 7 15:17:11 2017 -0700

--
 .../apache/beam/sdk/coders/BeamRecordCoder.java | 15 -
 .../org/apache/beam/sdk/values/BeamRecord.java  | 23 +++-
 .../extensions/sql/impl/rel/BeamSortRel.java|  4 
 .../sql/schema/BeamSqlRecordType.java   |  4 
 4 files changed, 21 insertions(+), 25 deletions(-)
--

[1/2] beam git commit: [rebased] remove nullFields in BeamRecord

[takidau]

Repository: beam
Updated Branches:
  refs/heads/DSL_SQL 79880b6ab -> 880531aa2


[rebased] remove nullFields in BeamRecord


Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/4871edbc
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/4871edbc
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/4871edbc

Branch: refs/heads/DSL_SQL
Commit: 4871edbc430a2e360ba59f10eeafbe2205d47ec1
Parents: 79880b6
Author: mingmxu 
Authored: Mon Aug 7 14:36:36 2017 -0700
Committer: mingmxu 
Committed: Mon Aug 7 14:36:36 2017 -0700

--
 .../apache/beam/sdk/coders/BeamRecordCoder.java | 15 -
 .../org/apache/beam/sdk/values/BeamRecord.java  | 23 +++-
 .../extensions/sql/impl/rel/BeamSortRel.java|  4 
 .../sql/schema/BeamSqlRecordType.java   |  4 
 4 files changed, 21 insertions(+), 25 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/beam/blob/4871edbc/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/BeamRecordCoder.java
--
diff --git 
a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/BeamRecordCoder.java 
b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/BeamRecordCoder.java
index fe9c295..40b9f3f 100644
--- 
a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/BeamRecordCoder.java
+++ 
b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/BeamRecordCoder.java
@@ -55,7 +55,7 @@ public class BeamRecordCoder extends CustomCoder {
   @Override
   public void encode(BeamRecord value, OutputStream outStream)
   throws CoderException, IOException {
-nullListCoder.encode(value.getNullFields(), outStream);
+nullListCoder.encode(scanNullFields(value), outStream);
 for (int idx = 0; idx < value.size(); ++idx) {
   if (value.isNull(idx)) {
 continue;
@@ -81,6 +81,19 @@ public class BeamRecordCoder extends CustomCoder 
{
 return record;
   }
 
+  /**
+   * Scan {@link BeamRecord} to find fields with a NULL value.
+   */
+  private BitSet scanNullFields(BeamRecord record){
+BitSet nullFields = new BitSet(record.size());
+for (int idx = 0; idx < record.size(); ++idx) {
+  if (record.isNull(idx)) {
+nullFields.set(idx);
+  }
+}
+return nullFields;
+  }
+
   @Override
   public void verifyDeterministic()
   throws org.apache.beam.sdk.coders.Coder.NonDeterministicException {

http://git-wip-us.apache.org/repos/asf/beam/blob/4871edbc/sdks/java/core/src/main/java/org/apache/beam/sdk/values/BeamRecord.java
--
diff --git 
a/sdks/java/core/src/main/java/org/apache/beam/sdk/values/BeamRecord.java 
b/sdks/java/core/src/main/java/org/apache/beam/sdk/values/BeamRecord.java
index 8d0aa42..6cbd11b 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/values/BeamRecord.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/values/BeamRecord.java
@@ -20,7 +20,6 @@ package org.apache.beam.sdk.values;
 import java.io.Serializable;
 import java.math.BigDecimal;
 import java.util.ArrayList;
-import java.util.BitSet;
 import java.util.Date;
 import java.util.GregorianCalendar;
 import java.util.List;
@@ -34,17 +33,13 @@ import org.apache.beam.sdk.annotations.Experimental;
 @Experimental
 public class BeamRecord implements Serializable {
   private List dataValues;
-  //null values are indexed here, to handle properly in Coder.
-  private BitSet nullFields;
   private BeamRecordType dataType;
 
   public BeamRecord(BeamRecordType dataType) {
 this.dataType = dataType;
-this.nullFields = new BitSet(dataType.size());
 this.dataValues = new ArrayList<>();
 for (int idx = 0; idx < dataType.size(); ++idx) {
   dataValues.add(null);
-  nullFields.set(idx);
 }
   }
 
@@ -60,12 +55,6 @@ public class BeamRecord implements Serializable {
   }
 
   public void addField(int index, Object fieldValue) {
-if (fieldValue == null) {
-  return;
-} else {
-  nullFields.clear(index);
-}
-
 dataType.validateValueType(index, fieldValue);
 dataValues.set(index, fieldValue);
   }
@@ -182,21 +171,16 @@ public class BeamRecord implements Serializable {
 return dataType;
   }
 
-  public BitSet getNullFields() {
-return nullFields;
-  }
-
   /**
* is the specified field NULL?
*/
   public boolean isNull(int idx) {
-return nullFields.get(idx);
+return null ==  getFieldValue(idx);
   }
 
   @Override
   public String toString() {
-return "BeamSqlRow [nullFields=" + nullFields + ", dataValues=" + 
dataValues + ", dataType="
-+ dataType + "]";
+return "BeamSqlRow [dataValues=" + dataValues + ", dataType=" + dataType + 
"]";
   }
 
   /**
@@ -227,7 +211,6 @@ 

[4/4] beam git commit: [BEAM-2722] This closes #3689

[takidau]

[BEAM-2722] This closes #3689


Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/79880b6a
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/79880b6a
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/79880b6a

Branch: refs/heads/DSL_SQL
Commit: 79880b6ab71936b6681c581128db87cc69ab6395
Parents: 8f922f7 c0b1fed
Author: Tyler Akidau 
Authored: Mon Aug 7 14:05:04 2017 -0700
Committer: Tyler Akidau 
Committed: Mon Aug 7 14:05:04 2017 -0700

--
 .../apache/beam/sdk/coders/BeamRecordCoder.java |  7 --
 .../org/apache/beam/sdk/values/BeamRecord.java  | 36 +---
 .../interpreter/BeamSqlExpressionExecutor.java  |  5 +-
 .../sql/impl/interpreter/BeamSqlFnExecutor.java |  5 +-
 .../operator/BeamSqlCaseExpression.java |  9 +-
 .../operator/BeamSqlCastExpression.java | 31 +++
 .../interpreter/operator/BeamSqlExpression.java |  9 +-
 .../operator/BeamSqlInputRefExpression.java |  3 +-
 .../interpreter/operator/BeamSqlPrimitive.java  |  5 +-
 .../operator/BeamSqlReinterpretExpression.java  |  7 +-
 .../operator/BeamSqlUdfExpression.java  |  5 +-
 .../operator/BeamSqlWindowEndExpression.java| 12 ++-
 .../operator/BeamSqlWindowExpression.java   |  5 +-
 .../operator/BeamSqlWindowStartExpression.java  | 12 ++-
 .../arithmetic/BeamSqlArithmeticExpression.java |  8 +-
 .../comparison/BeamSqlCompareExpression.java|  7 +-
 .../comparison/BeamSqlIsNotNullExpression.java  |  5 +-
 .../comparison/BeamSqlIsNullExpression.java |  5 +-
 .../date/BeamSqlCurrentDateExpression.java  |  3 +-
 .../date/BeamSqlCurrentTimeExpression.java  |  3 +-
 .../date/BeamSqlCurrentTimestampExpression.java |  3 +-
 .../date/BeamSqlDateCeilExpression.java |  5 +-
 .../date/BeamSqlDateFloorExpression.java|  5 +-
 .../operator/date/BeamSqlExtractExpression.java |  5 +-
 .../operator/logical/BeamSqlAndExpression.java  |  5 +-
 .../operator/logical/BeamSqlNotExpression.java  |  7 +-
 .../operator/logical/BeamSqlOrExpression.java   |  5 +-
 .../math/BeamSqlMathBinaryExpression.java   |  6 +-
 .../math/BeamSqlMathUnaryExpression.java|  6 +-
 .../operator/math/BeamSqlPiExpression.java  |  3 +-
 .../operator/math/BeamSqlRandExpression.java|  5 +-
 .../math/BeamSqlRandIntegerExpression.java  |  7 +-
 .../string/BeamSqlCharLengthExpression.java |  5 +-
 .../string/BeamSqlConcatExpression.java |  7 +-
 .../string/BeamSqlInitCapExpression.java|  5 +-
 .../operator/string/BeamSqlLowerExpression.java |  5 +-
 .../string/BeamSqlOverlayExpression.java| 11 +--
 .../string/BeamSqlPositionExpression.java   |  9 +-
 .../string/BeamSqlSubstringExpression.java  |  9 +-
 .../operator/string/BeamSqlTrimExpression.java  | 11 +--
 .../operator/string/BeamSqlUpperExpression.java |  5 +-
 .../transform/BeamAggregationTransforms.java|  7 +-
 .../sql/impl/transform/BeamSqlFilterFn.java |  5 +-
 .../sql/impl/transform/BeamSqlProjectFn.java|  3 +-
 .../sql/BeamSqlDslAggregationTest.java  | 17 
 .../operator/BeamNullExperssionTest.java|  8 +-
 .../operator/BeamSqlAndOrExpressionTest.java|  8 +-
 .../operator/BeamSqlCaseExpressionTest.java |  6 +-
 .../operator/BeamSqlCastExpressionTest.java | 24 +++---
 .../operator/BeamSqlCompareExpressionTest.java  | 24 +++---
 .../operator/BeamSqlInputRefExpressionTest.java | 12 +--
 .../operator/BeamSqlPrimitiveTest.java  | 10 +--
 .../BeamSqlReinterpretExpressionTest.java   |  2 +-
 .../operator/BeamSqlUdfExpressionTest.java  |  2 +-
 .../BeamSqlArithmeticExpressionTest.java| 46 +-
 .../date/BeamSqlCurrentDateExpressionTest.java  |  2 +-
 .../date/BeamSqlCurrentTimeExpressionTest.java  |  2 +-
 .../BeamSqlCurrentTimestampExpressionTest.java  |  2 +-
 .../date/BeamSqlDateCeilExpressionTest.java |  4 +-
 .../date/BeamSqlDateFloorExpressionTest.java|  4 +-
 .../date/BeamSqlExtractExpressionTest.java  | 14 +--
 .../logical/BeamSqlNotExpressionTest.java   |  6 +-
 .../math/BeamSqlMathBinaryExpressionTest.java   | 58 -
 .../math/BeamSqlMathUnaryExpressionTest.java| 91 ++--
 .../string/BeamSqlCharLengthExpressionTest.java |  2 +-
 .../string/BeamSqlConcatExpressionTest.java |  2 +-
 .../string/BeamSqlInitCapExpressionTest.java|  6 +-
 .../string/BeamSqlLowerExpressionTest.java  |  2 +-
 .../string/BeamSqlOverlayExpressionTest.java|  8 +-
 .../string/BeamSqlPositionExpressionTest.java   |  6 +-
 .../string/BeamSqlSubstringExpressionTest.java  | 14 +--
 .../string/BeamSqlTrimExpressionTest.java   |  8 +-
 .../string/BeamSqlUpperExpressionTest.java  |  2 +-
 73 files changed, 366 insertions(+), 352 deletions(-)
--

[3/4] beam git commit: Remove redundant windowing information from the BeamRecord itself element window information `BoundedWindow` is added in `BeamSqlExpression`.

[takidau]

Remove redundant windowing information from the BeamRecord itself
element window information `BoundedWindow` is added in `BeamSqlExpression`.


Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/c0b1fed1
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/c0b1fed1
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/c0b1fed1

Branch: refs/heads/DSL_SQL
Commit: c0b1fed1c7687a1e44e8ee6997be40aa1785ea51
Parents: 8f922f7
Author: mingmxu 
Authored: Fri Aug 4 11:12:45 2017 -0700
Committer: mingmxu 
Committed: Fri Aug 4 11:41:56 2017 -0700

--
 .../apache/beam/sdk/coders/BeamRecordCoder.java |  7 --
 .../org/apache/beam/sdk/values/BeamRecord.java  | 36 +---
 .../interpreter/BeamSqlExpressionExecutor.java  |  5 +-
 .../sql/impl/interpreter/BeamSqlFnExecutor.java |  5 +-
 .../operator/BeamSqlCaseExpression.java |  9 +-
 .../operator/BeamSqlCastExpression.java | 31 +++
 .../interpreter/operator/BeamSqlExpression.java |  9 +-
 .../operator/BeamSqlInputRefExpression.java |  3 +-
 .../interpreter/operator/BeamSqlPrimitive.java  |  5 +-
 .../operator/BeamSqlReinterpretExpression.java  |  7 +-
 .../operator/BeamSqlUdfExpression.java  |  5 +-
 .../operator/BeamSqlWindowEndExpression.java| 12 ++-
 .../operator/BeamSqlWindowExpression.java   |  5 +-
 .../operator/BeamSqlWindowStartExpression.java  | 12 ++-
 .../arithmetic/BeamSqlArithmeticExpression.java |  8 +-
 .../comparison/BeamSqlCompareExpression.java|  7 +-
 .../comparison/BeamSqlIsNotNullExpression.java  |  5 +-
 .../comparison/BeamSqlIsNullExpression.java |  5 +-
 .../date/BeamSqlCurrentDateExpression.java  |  3 +-
 .../date/BeamSqlCurrentTimeExpression.java  |  3 +-
 .../date/BeamSqlCurrentTimestampExpression.java |  3 +-
 .../date/BeamSqlDateCeilExpression.java |  5 +-
 .../date/BeamSqlDateFloorExpression.java|  5 +-
 .../operator/date/BeamSqlExtractExpression.java |  5 +-
 .../operator/logical/BeamSqlAndExpression.java  |  5 +-
 .../operator/logical/BeamSqlNotExpression.java  |  7 +-
 .../operator/logical/BeamSqlOrExpression.java   |  5 +-
 .../math/BeamSqlMathBinaryExpression.java   |  6 +-
 .../math/BeamSqlMathUnaryExpression.java|  6 +-
 .../operator/math/BeamSqlPiExpression.java  |  3 +-
 .../operator/math/BeamSqlRandExpression.java|  5 +-
 .../math/BeamSqlRandIntegerExpression.java  |  7 +-
 .../string/BeamSqlCharLengthExpression.java |  5 +-
 .../string/BeamSqlConcatExpression.java |  7 +-
 .../string/BeamSqlInitCapExpression.java|  5 +-
 .../operator/string/BeamSqlLowerExpression.java |  5 +-
 .../string/BeamSqlOverlayExpression.java| 11 +--
 .../string/BeamSqlPositionExpression.java   |  9 +-
 .../string/BeamSqlSubstringExpression.java  |  9 +-
 .../operator/string/BeamSqlTrimExpression.java  | 11 +--
 .../operator/string/BeamSqlUpperExpression.java |  5 +-
 .../transform/BeamAggregationTransforms.java|  7 +-
 .../sql/impl/transform/BeamSqlFilterFn.java |  5 +-
 .../sql/impl/transform/BeamSqlProjectFn.java|  3 +-
 .../sql/BeamSqlDslAggregationTest.java  | 17 
 .../operator/BeamNullExperssionTest.java|  8 +-
 .../operator/BeamSqlAndOrExpressionTest.java|  8 +-
 .../operator/BeamSqlCaseExpressionTest.java |  6 +-
 .../operator/BeamSqlCastExpressionTest.java | 24 +++---
 .../operator/BeamSqlCompareExpressionTest.java  | 24 +++---
 .../operator/BeamSqlInputRefExpressionTest.java | 12 +--
 .../operator/BeamSqlPrimitiveTest.java  | 10 +--
 .../BeamSqlReinterpretExpressionTest.java   |  2 +-
 .../operator/BeamSqlUdfExpressionTest.java  |  2 +-
 .../BeamSqlArithmeticExpressionTest.java| 46 +-
 .../date/BeamSqlCurrentDateExpressionTest.java  |  2 +-
 .../date/BeamSqlCurrentTimeExpressionTest.java  |  2 +-
 .../BeamSqlCurrentTimestampExpressionTest.java  |  2 +-
 .../date/BeamSqlDateCeilExpressionTest.java |  4 +-
 .../date/BeamSqlDateFloorExpressionTest.java|  4 +-
 .../date/BeamSqlExtractExpressionTest.java  | 14 +--
 .../logical/BeamSqlNotExpressionTest.java   |  6 +-
 .../math/BeamSqlMathBinaryExpressionTest.java   | 58 -
 .../math/BeamSqlMathUnaryExpressionTest.java| 91 ++--
 .../string/BeamSqlCharLengthExpressionTest.java |  2 +-
 .../string/BeamSqlConcatExpressionTest.java |  2 +-
 .../string/BeamSqlInitCapExpressionTest.java|  6 +-
 .../string/BeamSqlLowerExpressionTest.java  |  2 +-
 .../string/BeamSqlOverlayExpressionTest.java|  8 +-
 .../string/BeamSqlPositionExpressionTest.java   |  6 +-
 .../string/BeamSqlSubstringExpressionTest.java  | 14 +--
 .../string/BeamSqlTrimExpressionTest.java   |  8 +-
 .../string/BeamSqlUpperExpressionTest.java  |  2 +-
 73 files changed, 366 insertions(+), 352 

[2/4] beam git commit: Remove redundant windowing information from the BeamRecord itself element window information `BoundedWindow` is added in `BeamSqlExpression`.

[takidau]

http://git-wip-us.apache.org/repos/asf/beam/blob/c0b1fed1/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/transform/BeamSqlProjectFn.java
--
diff --git 
a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/transform/BeamSqlProjectFn.java
 
b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/transform/BeamSqlProjectFn.java
index a95c743..45dc621 100644
--- 
a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/transform/BeamSqlProjectFn.java
+++ 
b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/transform/BeamSqlProjectFn.java
@@ -52,10 +52,9 @@ public class BeamSqlProjectFn extends DoFn {
   @ProcessElement
   public void processElement(ProcessContext c, BoundedWindow window) {
 BeamRecord inputRow = c.element();
-List results = executor.execute(inputRow);
+List results = executor.execute(inputRow, window);
 
 BeamRecord outRow = new BeamRecord(outputRowType);
-outRow.updateWindowRange(inputRow, window);
 
 for (int idx = 0; idx < results.size(); ++idx) {
   BeamTableUtils.addFieldWithAutoTypeCasting(outRow, idx, 
results.get(idx));

http://git-wip-us.apache.org/repos/asf/beam/blob/c0b1fed1/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/BeamSqlDslAggregationTest.java
--
diff --git 
a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/BeamSqlDslAggregationTest.java
 
b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/BeamSqlDslAggregationTest.java
index 8501157..71278ec 100644
--- 
a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/BeamSqlDslAggregationTest.java
+++ 
b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/BeamSqlDslAggregationTest.java
@@ -25,7 +25,6 @@ import org.apache.beam.sdk.values.BeamRecord;
 import org.apache.beam.sdk.values.PCollection;
 import org.apache.beam.sdk.values.PCollectionTuple;
 import org.apache.beam.sdk.values.TupleTag;
-import org.joda.time.Instant;
 import org.junit.Test;
 
 /**
@@ -224,15 +223,11 @@ public class BeamSqlDslAggregationTest extends 
BeamSqlDslBase {
 record1.addField("f_int2", 0);
 record1.addField("size", 3L);
 record1.addField("window_start", FORMAT.parse("2017-01-01 01:00:00"));
-record1.setWindowStart(new Instant(FORMAT.parse("2017-01-01 
01:00:00").getTime()));
-record1.setWindowEnd(new Instant(FORMAT.parse("2017-01-01 
02:00:00").getTime()));
 
 BeamRecord record2 = new BeamRecord(resultType);
 record2.addField("f_int2", 0);
 record2.addField("size", 1L);
 record2.addField("window_start", FORMAT.parse("2017-01-01 02:00:00"));
-record2.setWindowStart(new Instant(FORMAT.parse("2017-01-01 
02:00:00").getTime()));
-record2.setWindowEnd(new Instant(FORMAT.parse("2017-01-01 
03:00:00").getTime()));
 
 PAssert.that(result).containsInAnyOrder(record1, record2);
 
@@ -271,29 +266,21 @@ public class BeamSqlDslAggregationTest extends 
BeamSqlDslBase {
 record1.addField("f_int2", 0);
 record1.addField("size", 3L);
 record1.addField("window_start", FORMAT.parse("2017-01-01 00:30:00"));
-record1.setWindowStart(new Instant(FORMAT.parse("2017-01-01 
00:30:00").getTime()));
-record1.setWindowEnd(new Instant(FORMAT.parse("2017-01-01 
01:30:00").getTime()));
 
 BeamRecord record2 = new BeamRecord(resultType);
 record2.addField("f_int2", 0);
 record2.addField("size", 3L);
 record2.addField("window_start", FORMAT.parse("2017-01-01 01:00:00"));
-record2.setWindowStart(new Instant(FORMAT.parse("2017-01-01 
01:00:00").getTime()));
-record2.setWindowEnd(new Instant(FORMAT.parse("2017-01-01 
02:00:00").getTime()));
 
 BeamRecord record3 = new BeamRecord(resultType);
 record3.addField("f_int2", 0);
 record3.addField("size", 1L);
 record3.addField("window_start", FORMAT.parse("2017-01-01 01:30:00"));
-record3.setWindowStart(new Instant(FORMAT.parse("2017-01-01 
01:30:00").getTime()));
-record3.setWindowEnd(new Instant(FORMAT.parse("2017-01-01 
02:30:00").getTime()));
 
 BeamRecord record4 = new BeamRecord(resultType);
 record4.addField("f_int2", 0);
 record4.addField("size", 1L);
 record4.addField("window_start", FORMAT.parse("2017-01-01 02:00:00"));
-record4.setWindowStart(new Instant(FORMAT.parse("2017-01-01 
02:00:00").getTime()));
-record4.setWindowEnd(new Instant(FORMAT.parse("2017-01-01 
03:00:00").getTime()));
 
 PAssert.that(result).containsInAnyOrder(record1, record2, record3, 
record4);
 
@@ -333,15 +320,11 @@ public class BeamSqlDslAggregationTest extends 
BeamSqlDslBase {
 record1.addField("f_int2", 0);
 record1.addField("size", 3L);
 

[1/4] beam git commit: Remove redundant windowing information from the BeamRecord itself element window information `BoundedWindow` is added in `BeamSqlExpression`.

[takidau]

Repository: beam
Updated Branches:
  refs/heads/DSL_SQL 8f922f74b -> 79880b6ab


http://git-wip-us.apache.org/repos/asf/beam/blob/c0b1fed1/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/string/BeamSqlPositionExpressionTest.java
--
diff --git 
a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/string/BeamSqlPositionExpressionTest.java
 
b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/string/BeamSqlPositionExpressionTest.java
index 4c21a71..3b477cc 100644
--- 
a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/string/BeamSqlPositionExpressionTest.java
+++ 
b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/string/BeamSqlPositionExpressionTest.java
@@ -66,19 +66,19 @@ public class BeamSqlPositionExpressionTest extends 
BeamSqlFnExecutorTestBase {
 
 operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "hello"));
 operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "worldhello"));
-assertEquals(5, new 
BeamSqlPositionExpression(operands).evaluate(record).getValue());
+assertEquals(5, new BeamSqlPositionExpression(operands).evaluate(record, 
null).getValue());
 
 operands.clear();
 operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "hello"));
 operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "worldhello"));
 operands.add(BeamSqlPrimitive.of(SqlTypeName.INTEGER, 1));
-assertEquals(5, new 
BeamSqlPositionExpression(operands).evaluate(record).getValue());
+assertEquals(5, new BeamSqlPositionExpression(operands).evaluate(record, 
null).getValue());
 
 operands.clear();
 operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "hello"));
 operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "world"));
 operands.add(BeamSqlPrimitive.of(SqlTypeName.INTEGER, 1));
-assertEquals(-1, new 
BeamSqlPositionExpression(operands).evaluate(record).getValue());
+assertEquals(-1, new BeamSqlPositionExpression(operands).evaluate(record, 
null).getValue());
   }
 
 }

http://git-wip-us.apache.org/repos/asf/beam/blob/c0b1fed1/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/string/BeamSqlSubstringExpressionTest.java
--
diff --git 
a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/string/BeamSqlSubstringExpressionTest.java
 
b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/string/BeamSqlSubstringExpressionTest.java
index 2fb451e..b48a8be 100644
--- 
a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/string/BeamSqlSubstringExpressionTest.java
+++ 
b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/string/BeamSqlSubstringExpressionTest.java
@@ -54,48 +54,48 @@ public class BeamSqlSubstringExpressionTest extends 
BeamSqlFnExecutorTestBase {
 operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "hello"));
 operands.add(BeamSqlPrimitive.of(SqlTypeName.INTEGER, 1));
 assertEquals("hello",
-new BeamSqlSubstringExpression(operands).evaluate(record).getValue());
+new BeamSqlSubstringExpression(operands).evaluate(record, 
null).getValue());
 
 operands.clear();
 operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "hello"));
 operands.add(BeamSqlPrimitive.of(SqlTypeName.INTEGER, 1));
 operands.add(BeamSqlPrimitive.of(SqlTypeName.INTEGER, 2));
 assertEquals("he",
-new BeamSqlSubstringExpression(operands).evaluate(record).getValue());
+new BeamSqlSubstringExpression(operands).evaluate(record, 
null).getValue());
 
 operands.clear();
 operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "hello"));
 operands.add(BeamSqlPrimitive.of(SqlTypeName.INTEGER, 1));
 operands.add(BeamSqlPrimitive.of(SqlTypeName.INTEGER, 5));
 assertEquals("hello",
-new BeamSqlSubstringExpression(operands).evaluate(record).getValue());
+new BeamSqlSubstringExpression(operands).evaluate(record, 
null).getValue());
 
 operands.clear();
 operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "hello"));
 operands.add(BeamSqlPrimitive.of(SqlTypeName.INTEGER, 1));
 operands.add(BeamSqlPrimitive.of(SqlTypeName.INTEGER, 100));
 assertEquals("hello",
-new BeamSqlSubstringExpression(operands).evaluate(record).getValue());
+new BeamSqlSubstringExpression(operands).evaluate(record, 
null).getValue());
 
 operands.clear();
 operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "hello"));
 

[23/59] beam git commit: rename package org.apache.beam.dsls.sql to org.apache.beam.sdk.extensions.sql

[takidau]

http://git-wip-us.apache.org/repos/asf/beam/blob/c1b5482d/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/interpreter/operator/string/BeamSqlOverlayExpressionTest.java
--
diff --git 
a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/interpreter/operator/string/BeamSqlOverlayExpressionTest.java
 
b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/interpreter/operator/string/BeamSqlOverlayExpressionTest.java
new file mode 100644
index 000..2ca0a98
--- /dev/null
+++ 
b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/interpreter/operator/string/BeamSqlOverlayExpressionTest.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.extensions.sql.interpreter.operator.string;
+
+import static org.junit.Assert.assertTrue;
+
+import java.util.ArrayList;
+import java.util.List;
+import 
org.apache.beam.sdk.extensions.sql.interpreter.BeamSqlFnExecutorTestBase;
+import 
org.apache.beam.sdk.extensions.sql.interpreter.operator.BeamSqlExpression;
+import 
org.apache.beam.sdk.extensions.sql.interpreter.operator.BeamSqlPrimitive;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * Test for BeamSqlOverlayExpression.
+ */
+public class BeamSqlOverlayExpressionTest extends BeamSqlFnExecutorTestBase {
+
+  @Test public void accept() throws Exception {
+List operands = new ArrayList<>();
+
+operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "hello"));
+operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "hello"));
+operands.add(BeamSqlPrimitive.of(SqlTypeName.INTEGER, 1));
+assertTrue(new BeamSqlOverlayExpression(operands).accept());
+
+operands.clear();
+operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "hello"));
+operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "hello"));
+operands.add(BeamSqlPrimitive.of(SqlTypeName.INTEGER, 1));
+operands.add(BeamSqlPrimitive.of(SqlTypeName.INTEGER, 2));
+assertTrue(new BeamSqlOverlayExpression(operands).accept());
+  }
+
+  @Test public void evaluate() throws Exception {
+List operands = new ArrayList<>();
+
+operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "w333rce"));
+operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "resou"));
+operands.add(BeamSqlPrimitive.of(SqlTypeName.INTEGER, 3));
+Assert.assertEquals("w3resou3rce",
+new BeamSqlOverlayExpression(operands).evaluate(record).getValue());
+
+operands.clear();
+operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "w333rce"));
+operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "resou"));
+operands.add(BeamSqlPrimitive.of(SqlTypeName.INTEGER, 3));
+operands.add(BeamSqlPrimitive.of(SqlTypeName.INTEGER, 4));
+Assert.assertEquals("w3resou33rce",
+new BeamSqlOverlayExpression(operands).evaluate(record).getValue());
+
+operands.clear();
+operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "w333rce"));
+operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "resou"));
+operands.add(BeamSqlPrimitive.of(SqlTypeName.INTEGER, 3));
+operands.add(BeamSqlPrimitive.of(SqlTypeName.INTEGER, 5));
+Assert.assertEquals("w3resou3rce",
+new BeamSqlOverlayExpression(operands).evaluate(record).getValue());
+
+operands.clear();
+operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "w333rce"));
+operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "resou"));
+operands.add(BeamSqlPrimitive.of(SqlTypeName.INTEGER, 3));
+operands.add(BeamSqlPrimitive.of(SqlTypeName.INTEGER, 7));
+Assert.assertEquals("w3resouce",
+new BeamSqlOverlayExpression(operands).evaluate(record).getValue());
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/beam/blob/c1b5482d/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/interpreter/operator/string/BeamSqlPositionExpressionTest.java
--
diff --git 

[08/59] beam git commit: move dsls/sql to sdks/java/extensions/sql

[takidau]

http://git-wip-us.apache.org/repos/asf/beam/blob/ba493f85/sdks/java/extensions/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/string/BeamSqlOverlayExpression.java
--
diff --git 
a/sdks/java/extensions/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/string/BeamSqlOverlayExpression.java
 
b/sdks/java/extensions/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/string/BeamSqlOverlayExpression.java
new file mode 100644
index 000..cb6a523
--- /dev/null
+++ 
b/sdks/java/extensions/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/string/BeamSqlOverlayExpression.java
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.dsls.sql.interpreter.operator.string;
+
+import java.util.List;
+
+import org.apache.beam.dsls.sql.interpreter.operator.BeamSqlExpression;
+import org.apache.beam.dsls.sql.interpreter.operator.BeamSqlPrimitive;
+import org.apache.beam.dsls.sql.schema.BeamSqlRow;
+import org.apache.calcite.sql.type.SqlTypeName;
+
+/**
+ * 'OVERLAY' operator.
+ *
+ * 
+ *   OVERLAY(string1 PLACING string2 FROM integer [ FOR integer2 ])
+ * 
+ */
+public class BeamSqlOverlayExpression extends BeamSqlExpression {
+  public BeamSqlOverlayExpression(List operands) {
+super(operands, SqlTypeName.VARCHAR);
+  }
+
+  @Override public boolean accept() {
+if (operands.size() < 3 || operands.size() > 4) {
+  return false;
+}
+
+if (!SqlTypeName.CHAR_TYPES.contains(opType(0))
+|| !SqlTypeName.CHAR_TYPES.contains(opType(1))
+|| !SqlTypeName.INT_TYPES.contains(opType(2))) {
+  return false;
+}
+
+if (operands.size() == 4 && !SqlTypeName.INT_TYPES.contains(opType(3))) {
+  return false;
+}
+
+return true;
+  }
+
+  @Override public BeamSqlPrimitive evaluate(BeamSqlRow inputRow) {
+String str = opValueEvaluated(0, inputRow);
+String replaceStr = opValueEvaluated(1, inputRow);
+int idx = opValueEvaluated(2, inputRow);
+// the index is 1 based.
+idx -= 1;
+int length = replaceStr.length();
+if (operands.size() == 4) {
+  length = opValueEvaluated(3, inputRow);
+}
+
+StringBuilder result = new StringBuilder(
+str.length() + replaceStr.length() - length);
+result.append(str.substring(0, idx))
+.append(replaceStr)
+.append(str.substring(idx + length));
+
+return BeamSqlPrimitive.of(SqlTypeName.VARCHAR, result.toString());
+  }
+}

http://git-wip-us.apache.org/repos/asf/beam/blob/ba493f85/sdks/java/extensions/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/string/BeamSqlPositionExpression.java
--
diff --git 
a/sdks/java/extensions/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/string/BeamSqlPositionExpression.java
 
b/sdks/java/extensions/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/string/BeamSqlPositionExpression.java
new file mode 100644
index 000..144acbf
--- /dev/null
+++ 
b/sdks/java/extensions/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/string/BeamSqlPositionExpression.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.dsls.sql.interpreter.operator.string;
+
+import java.util.List;
+
+import 

[50/59] beam git commit: move all implementation classes/packages into impl package

[takidau]

http://git-wip-us.apache.org/repos/asf/beam/blob/febd044a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/rel/BeamFilterRel.java
--
diff --git 
a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/rel/BeamFilterRel.java
 
b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/rel/BeamFilterRel.java
deleted file mode 100644
index f1da29f..000
--- 
a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/rel/BeamFilterRel.java
+++ /dev/null
@@ -1,70 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.beam.sdk.extensions.sql.rel;
-
-import org.apache.beam.sdk.extensions.sql.BeamSqlEnv;
-import 
org.apache.beam.sdk.extensions.sql.interpreter.BeamSqlExpressionExecutor;
-import org.apache.beam.sdk.extensions.sql.interpreter.BeamSqlFnExecutor;
-import org.apache.beam.sdk.extensions.sql.schema.BeamSqlRow;
-import org.apache.beam.sdk.extensions.sql.schema.BeamSqlRowCoder;
-import org.apache.beam.sdk.extensions.sql.transform.BeamSqlFilterFn;
-import org.apache.beam.sdk.extensions.sql.utils.CalciteUtils;
-import org.apache.beam.sdk.transforms.ParDo;
-import org.apache.beam.sdk.values.PCollection;
-import org.apache.beam.sdk.values.PCollectionTuple;
-import org.apache.calcite.plan.RelOptCluster;
-import org.apache.calcite.plan.RelTraitSet;
-import org.apache.calcite.rel.RelNode;
-import org.apache.calcite.rel.core.Filter;
-import org.apache.calcite.rex.RexNode;
-
-/**
- * BeamRelNode to replace a {@code Filter} node.
- *
- */
-public class BeamFilterRel extends Filter implements BeamRelNode {
-
-  public BeamFilterRel(RelOptCluster cluster, RelTraitSet traits, RelNode 
child,
-  RexNode condition) {
-super(cluster, traits, child, condition);
-  }
-
-  @Override
-  public Filter copy(RelTraitSet traitSet, RelNode input, RexNode condition) {
-return new BeamFilterRel(getCluster(), traitSet, input, condition);
-  }
-
-  @Override
-  public PCollection buildBeamPipeline(PCollectionTuple 
inputPCollections
-  , BeamSqlEnv sqlEnv) throws Exception {
-RelNode input = getInput();
-String stageName = BeamSqlRelUtils.getStageName(this);
-
-PCollection upstream =
-
BeamSqlRelUtils.getBeamRelInput(input).buildBeamPipeline(inputPCollections, 
sqlEnv);
-
-BeamSqlExpressionExecutor executor = new BeamSqlFnExecutor(this);
-
-PCollection filterStream = upstream.apply(stageName,
-ParDo.of(new BeamSqlFilterFn(getRelTypeName(), executor)));
-filterStream.setCoder(new 
BeamSqlRowCoder(CalciteUtils.toBeamRowType(getRowType(;
-
-return filterStream;
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/beam/blob/febd044a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/rel/BeamIOSinkRel.java
--
diff --git 
a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/rel/BeamIOSinkRel.java
 
b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/rel/BeamIOSinkRel.java
deleted file mode 100644
index ce941a0..000
--- 
a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/rel/BeamIOSinkRel.java
+++ /dev/null
@@ -1,75 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.beam.sdk.extensions.sql.rel;
-
-import 

[41/59] beam git commit: rename package org.apache.beam.dsls.sql to org.apache.beam.sdk.extensions.sql

[takidau]

http://git-wip-us.apache.org/repos/asf/beam/blob/c1b5482d/sdks/java/extensions/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/BeamSqlPrimitive.java
--
diff --git 
a/sdks/java/extensions/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/BeamSqlPrimitive.java
 
b/sdks/java/extensions/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/BeamSqlPrimitive.java
deleted file mode 100644
index 51724bb..000
--- 
a/sdks/java/extensions/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/BeamSqlPrimitive.java
+++ /dev/null
@@ -1,152 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.beam.dsls.sql.interpreter.operator;
-
-import java.math.BigDecimal;
-import java.util.Date;
-import java.util.GregorianCalendar;
-import java.util.List;
-import org.apache.beam.dsls.sql.schema.BeamSqlRow;
-import org.apache.calcite.sql.type.SqlTypeName;
-import org.apache.calcite.util.NlsString;
-
-/**
- * {@link BeamSqlPrimitive} is a special, self-reference {@link 
BeamSqlExpression}.
- * It holds the value, and return it directly during {@link 
#evaluate(BeamSqlRow)}.
- *
- */
-public class BeamSqlPrimitive extends BeamSqlExpression {
-  private T value;
-
-  private BeamSqlPrimitive() {
-  }
-
-  private BeamSqlPrimitive(List operands, SqlTypeName 
outputType) {
-super(operands, outputType);
-  }
-
-  /**
-   * A builder function to create from Type and value directly.
-   */
-  public static  BeamSqlPrimitive of(SqlTypeName outputType, T value){
-BeamSqlPrimitive exp = new BeamSqlPrimitive<>();
-exp.outputType = outputType;
-exp.value = value;
-if (!exp.accept()) {
-  throw new IllegalArgumentException(
-  String.format("value [%s] doesn't match type [%s].", value, 
outputType));
-}
-return exp;
-  }
-
-  public SqlTypeName getOutputType() {
-return outputType;
-  }
-
-  public T getValue() {
-return value;
-  }
-
-  public long getLong() {
-return (Long) getValue();
-  }
-
-  public double getDouble() {
-return (Double) getValue();
-  }
-
-  public float getFloat() {
-return (Float) getValue();
-  }
-
-  public int getInteger() {
-return (Integer) getValue();
-  }
-
-  public short getShort() {
-return (Short) getValue();
-  }
-
-  public byte getByte() {
-return (Byte) getValue();
-  }
-  public boolean getBoolean() {
-return (Boolean) getValue();
-  }
-
-  public String getString() {
-return (String) getValue();
-  }
-
-  public Date getDate() {
-return (Date) getValue();
-  }
-
-  public BigDecimal getDecimal() {
-return (BigDecimal) getValue();
-  }
-
-  @Override
-  public boolean accept() {
-if (value == null) {
-  return true;
-}
-
-switch (outputType) {
-case BIGINT:
-  return value instanceof Long;
-case DECIMAL:
-  return value instanceof BigDecimal;
-case DOUBLE:
-  return value instanceof Double;
-case FLOAT:
-  return value instanceof Float;
-case INTEGER:
-  return value instanceof Integer;
-case SMALLINT:
-  return value instanceof Short;
-case TINYINT:
-  return value instanceof Byte;
-case BOOLEAN:
-  return value instanceof Boolean;
-case CHAR:
-case VARCHAR:
-  return value instanceof String || value instanceof NlsString;
-case TIME:
-  return value instanceof GregorianCalendar;
-case TIMESTAMP:
-case DATE:
-  return value instanceof Date;
-case INTERVAL_HOUR:
-  return value instanceof BigDecimal;
-case INTERVAL_MINUTE:
-  return value instanceof BigDecimal;
-case SYMBOL:
-  // for SYMBOL, it supports anything...
-  return true;
-default:
-  throw new UnsupportedOperationException(outputType.name());
-}
-  }
-
-  @Override
-  public BeamSqlPrimitive evaluate(BeamSqlRow inputRow) {
-return this;
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/beam/blob/c1b5482d/sdks/java/extensions/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/BeamSqlReinterpretExpression.java
--
diff --git 

[11/59] beam git commit: move dsls/sql to sdks/java/extensions/sql

[takidau]

http://git-wip-us.apache.org/repos/asf/beam/blob/ba493f85/dsls/sql/src/test/java/org/apache/beam/dsls/sql/schema/transform/BeamAggregationTransformTest.java
--
diff --git 
a/dsls/sql/src/test/java/org/apache/beam/dsls/sql/schema/transform/BeamAggregationTransformTest.java
 
b/dsls/sql/src/test/java/org/apache/beam/dsls/sql/schema/transform/BeamAggregationTransformTest.java
deleted file mode 100644
index 5d5d4fc..000
--- 
a/dsls/sql/src/test/java/org/apache/beam/dsls/sql/schema/transform/BeamAggregationTransformTest.java
+++ /dev/null
@@ -1,453 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * 
- * http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.beam.dsls.sql.schema.transform;
-
-import java.text.ParseException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-import org.apache.beam.dsls.sql.planner.BeamQueryPlanner;
-import org.apache.beam.dsls.sql.schema.BeamSqlRow;
-import org.apache.beam.dsls.sql.schema.BeamSqlRowCoder;
-import org.apache.beam.dsls.sql.schema.BeamSqlRowType;
-import org.apache.beam.dsls.sql.transform.BeamAggregationTransforms;
-import org.apache.beam.dsls.sql.utils.CalciteUtils;
-import org.apache.beam.sdk.coders.IterableCoder;
-import org.apache.beam.sdk.coders.KvCoder;
-import org.apache.beam.sdk.testing.PAssert;
-import org.apache.beam.sdk.testing.TestPipeline;
-import org.apache.beam.sdk.transforms.Combine;
-import org.apache.beam.sdk.transforms.Create;
-import org.apache.beam.sdk.transforms.GroupByKey;
-import org.apache.beam.sdk.transforms.ParDo;
-import org.apache.beam.sdk.transforms.WithKeys;
-import org.apache.beam.sdk.values.KV;
-import org.apache.beam.sdk.values.PCollection;
-import org.apache.calcite.rel.core.AggregateCall;
-import org.apache.calcite.rel.type.RelDataTypeFactory.FieldInfoBuilder;
-import org.apache.calcite.rel.type.RelDataTypeSystem;
-import org.apache.calcite.sql.SqlKind;
-import org.apache.calcite.sql.fun.SqlAvgAggFunction;
-import org.apache.calcite.sql.fun.SqlCountAggFunction;
-import org.apache.calcite.sql.fun.SqlMinMaxAggFunction;
-import org.apache.calcite.sql.fun.SqlSumAggFunction;
-import org.apache.calcite.sql.type.BasicSqlType;
-import org.apache.calcite.sql.type.SqlTypeName;
-import org.apache.calcite.util.ImmutableBitSet;
-import org.junit.Rule;
-import org.junit.Test;
-
-/**
- * Unit tests for {@link BeamAggregationTransforms}.
- *
- */
-public class BeamAggregationTransformTest extends BeamTransformBaseTest{
-
-  @Rule
-  public TestPipeline p = TestPipeline.create();
-
-  private List aggCalls;
-
-  private BeamSqlRowType keyType;
-  private BeamSqlRowType aggPartType;
-  private BeamSqlRowType outputType;
-
-  private BeamSqlRowCoder inRecordCoder;
-  private BeamSqlRowCoder keyCoder;
-  private BeamSqlRowCoder aggCoder;
-  private BeamSqlRowCoder outRecordCoder;
-
-  /**
-   * This step equals to below query.
-   * 
-   * SELECT `f_int`
-   * , COUNT(*) AS `size`
-   * , SUM(`f_long`) AS `sum1`, AVG(`f_long`) AS `avg1`
-   * , MAX(`f_long`) AS `max1`, MIN(`f_long`) AS `min1`
-   * , SUM(`f_short`) AS `sum2`, AVG(`f_short`) AS `avg2`
-   * , MAX(`f_short`) AS `max2`, MIN(`f_short`) AS `min2`
-   * , SUM(`f_byte`) AS `sum3`, AVG(`f_byte`) AS `avg3`
-   * , MAX(`f_byte`) AS `max3`, MIN(`f_byte`) AS `min3`
-   * , SUM(`f_float`) AS `sum4`, AVG(`f_float`) AS `avg4`
-   * , MAX(`f_float`) AS `max4`, MIN(`f_float`) AS `min4`
-   * , SUM(`f_double`) AS `sum5`, AVG(`f_double`) AS `avg5`
-   * , MAX(`f_double`) AS `max5`, MIN(`f_double`) AS `min5`
-   * , MAX(`f_timestamp`) AS `max7`, MIN(`f_timestamp`) AS `min7`
-   * ,SUM(`f_int2`) AS `sum8`, AVG(`f_int2`) AS `avg8`
-   * , MAX(`f_int2`) AS `max8`, MIN(`f_int2`) AS `min8`
-   * FROM TABLE_NAME
-   * GROUP BY `f_int`
-   * 
-   * @throws ParseException
-   */
-  @Test
-  public void testCountPerElementBasic() throws ParseException {
-setupEnvironment();
-
-PCollection input = p.apply(Create.of(inputRows));
-
-//1. extract fields in group-by key part
-PCollection> exGroupByStream = 
input.apply("exGroupBy",
-WithKeys
-.of(new BeamAggregationTransforms.AggregationGroupByKeyFn(-1, 
ImmutableBitSet.of(0

[38/59] beam git commit: rename package org.apache.beam.dsls.sql to org.apache.beam.sdk.extensions.sql

[takidau]

http://git-wip-us.apache.org/repos/asf/beam/blob/c1b5482d/sdks/java/extensions/sql/src/main/java/org/apache/beam/dsls/sql/rel/BeamSortRel.java
--
diff --git 
a/sdks/java/extensions/sql/src/main/java/org/apache/beam/dsls/sql/rel/BeamSortRel.java
 
b/sdks/java/extensions/sql/src/main/java/org/apache/beam/dsls/sql/rel/BeamSortRel.java
deleted file mode 100644
index ba344df..000
--- 
a/sdks/java/extensions/sql/src/main/java/org/apache/beam/dsls/sql/rel/BeamSortRel.java
+++ /dev/null
@@ -1,247 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.beam.dsls.sql.rel;
-
-import java.io.Serializable;
-import java.lang.reflect.Type;
-import java.math.BigDecimal;
-import java.util.ArrayList;
-import java.util.Comparator;
-import java.util.List;
-import org.apache.beam.dsls.sql.BeamSqlEnv;
-import org.apache.beam.dsls.sql.schema.BeamSqlRow;
-import org.apache.beam.dsls.sql.schema.BeamSqlRowCoder;
-import org.apache.beam.dsls.sql.utils.CalciteUtils;
-import org.apache.beam.sdk.coders.ListCoder;
-import org.apache.beam.sdk.transforms.DoFn;
-import org.apache.beam.sdk.transforms.Flatten;
-import org.apache.beam.sdk.transforms.ParDo;
-import org.apache.beam.sdk.transforms.Top;
-import org.apache.beam.sdk.transforms.windowing.GlobalWindow;
-import org.apache.beam.sdk.values.PCollection;
-import org.apache.beam.sdk.values.PCollectionTuple;
-import org.apache.calcite.plan.RelOptCluster;
-import org.apache.calcite.plan.RelTraitSet;
-import org.apache.calcite.rel.RelCollation;
-import org.apache.calcite.rel.RelCollationImpl;
-import org.apache.calcite.rel.RelFieldCollation;
-import org.apache.calcite.rel.RelNode;
-import org.apache.calcite.rel.core.Sort;
-import org.apache.calcite.rex.RexInputRef;
-import org.apache.calcite.rex.RexLiteral;
-import org.apache.calcite.rex.RexNode;
-import org.apache.calcite.sql.type.SqlTypeName;
-
-/**
- * {@code BeamRelNode} to replace a {@code Sort} node.
- *
- * Since Beam does not fully supported global sort we are using {@link Top} 
to implement
- * the {@code Sort} algebra. The following types of ORDER BY are supported:
-
- * {@code
- * select * from t order by id desc limit 10;
- * select * from t order by id desc limit 10, 5;
- * }
- *
- * but Order BY without a limit is NOT supported:
- *
- * {@code
- *   select * from t order by id desc
- * }
- *
- * Constraints
- * 
- *   Due to the constraints of {@link Top}, the result of a `ORDER BY 
LIMIT`
- *   must fit into the memory of a single machine.
- *   Since `WINDOW`(HOP, TUMBLE, SESSION etc) is always associated with 
`GroupBy`,
- *   it does not make much sense to use `ORDER BY` with `WINDOW`.
- *   
- * 
- */
-public class BeamSortRel extends Sort implements BeamRelNode {
-  private List fieldIndices = new ArrayList<>();
-  private List orientation = new ArrayList<>();
-  private List nullsFirst = new ArrayList<>();
-
-  private int startIndex = 0;
-  private int count;
-
-  public BeamSortRel(
-  RelOptCluster cluster,
-  RelTraitSet traits,
-  RelNode child,
-  RelCollation collation,
-  RexNode offset,
-  RexNode fetch) {
-super(cluster, traits, child, collation, offset, fetch);
-
-List fieldExps = getChildExps();
-RelCollationImpl collationImpl = (RelCollationImpl) collation;
-List collations = collationImpl.getFieldCollations();
-for (int i = 0; i < fieldExps.size(); i++) {
-  RexNode fieldExp = fieldExps.get(i);
-  RexInputRef inputRef = (RexInputRef) fieldExp;
-  fieldIndices.add(inputRef.getIndex());
-  orientation.add(collations.get(i).getDirection() == 
RelFieldCollation.Direction.ASCENDING);
-
-  RelFieldCollation.NullDirection rawNullDirection = 
collations.get(i).nullDirection;
-  if (rawNullDirection == RelFieldCollation.NullDirection.UNSPECIFIED) {
-rawNullDirection = 
collations.get(i).getDirection().defaultNullDirection();
-  }
-  nullsFirst.add(rawNullDirection == 
RelFieldCollation.NullDirection.FIRST);
-}
-
-if (fetch == null) {
-  throw new UnsupportedOperationException("ORDER BY without a LIMIT is not 
supported!");
-}
-
-RexLiteral fetchLiteral = 

[18/59] beam git commit: move dsls/sql to sdks/java/extensions/sql

[takidau]

http://git-wip-us.apache.org/repos/asf/beam/blob/ba493f85/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/string/BeamSqlConcatExpression.java
--
diff --git 
a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/string/BeamSqlConcatExpression.java
 
b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/string/BeamSqlConcatExpression.java
deleted file mode 100644
index 93e1f71..000
--- 
a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/string/BeamSqlConcatExpression.java
+++ /dev/null
@@ -1,63 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.beam.dsls.sql.interpreter.operator.string;
-
-import java.util.List;
-
-import org.apache.beam.dsls.sql.interpreter.operator.BeamSqlExpression;
-import org.apache.beam.dsls.sql.interpreter.operator.BeamSqlPrimitive;
-import org.apache.beam.dsls.sql.schema.BeamSqlRow;
-import org.apache.calcite.sql.type.SqlTypeName;
-
-/**
- * String concat operator.
- */
-public class BeamSqlConcatExpression extends BeamSqlExpression {
-
-  protected BeamSqlConcatExpression(List operands, 
SqlTypeName outputType) {
-super(operands, outputType);
-  }
-
-  public BeamSqlConcatExpression(List operands) {
-super(operands, SqlTypeName.VARCHAR);
-  }
-
-  @Override public boolean accept() {
-if (operands.size() != 2) {
-  return false;
-}
-
-for (BeamSqlExpression exp : getOperands()) {
-  if (!SqlTypeName.CHAR_TYPES.contains(exp.getOutputType())) {
-return false;
-  }
-}
-
-return true;
-  }
-
-  @Override public BeamSqlPrimitive evaluate(BeamSqlRow inputRow) {
-String left = opValueEvaluated(0, inputRow);
-String right = opValueEvaluated(1, inputRow);
-
-return BeamSqlPrimitive.of(SqlTypeName.VARCHAR,
-new StringBuilder(left.length() + right.length())
-.append(left).append(right).toString());
-  }
-}

http://git-wip-us.apache.org/repos/asf/beam/blob/ba493f85/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/string/BeamSqlInitCapExpression.java
--
diff --git 
a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/string/BeamSqlInitCapExpression.java
 
b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/string/BeamSqlInitCapExpression.java
deleted file mode 100644
index 7726e27..000
--- 
a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/string/BeamSqlInitCapExpression.java
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.beam.dsls.sql.interpreter.operator.string;
-
-import java.util.List;
-
-import org.apache.beam.dsls.sql.interpreter.operator.BeamSqlExpression;
-import org.apache.beam.dsls.sql.interpreter.operator.BeamSqlPrimitive;
-import org.apache.beam.dsls.sql.schema.BeamSqlRow;
-import org.apache.calcite.sql.type.SqlTypeName;
-
-/**
- * 'INITCAP' operator.
- */
-public class BeamSqlInitCapExpression extends BeamSqlStringUnaryExpression {
-  public BeamSqlInitCapExpression(List operands) {
-super(operands, SqlTypeName.VARCHAR);
-  }
-
-  @Override public BeamSqlPrimitive evaluate(BeamSqlRow inputRow) {
-String str = opValueEvaluated(0, inputRow);
-
-StringBuilder ret = new StringBuilder(str);
-boolean 

[10/59] beam git commit: move dsls/sql to sdks/java/extensions/sql

[takidau]

http://git-wip-us.apache.org/repos/asf/beam/blob/ba493f85/sdks/java/extensions/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/BeamSqlExpression.java
--
diff --git 
a/sdks/java/extensions/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/BeamSqlExpression.java
 
b/sdks/java/extensions/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/BeamSqlExpression.java
new file mode 100644
index 000..9d2815c
--- /dev/null
+++ 
b/sdks/java/extensions/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/BeamSqlExpression.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.dsls.sql.interpreter.operator;
+
+import java.io.Serializable;
+import java.util.List;
+import org.apache.beam.dsls.sql.schema.BeamSqlRow;
+import org.apache.calcite.rex.RexNode;
+import org.apache.calcite.sql.type.SqlTypeName;
+
+/**
+ * {@code BeamSqlExpression} is an equivalent expression in BeamSQL, of {@link 
RexNode} in Calcite.
+ *
+ * An implementation of {@link BeamSqlExpression} takes one or more {@code 
BeamSqlExpression}
+ * as its operands, and return a value with type {@link SqlTypeName}.
+ *
+ */
+public abstract class BeamSqlExpression implements Serializable {
+  protected List operands;
+  protected SqlTypeName outputType;
+
+  protected BeamSqlExpression(){}
+
+  public BeamSqlExpression(List operands, SqlTypeName 
outputType) {
+this.operands = operands;
+this.outputType = outputType;
+  }
+
+  public BeamSqlExpression op(int idx) {
+return operands.get(idx);
+  }
+
+  public SqlTypeName opType(int idx) {
+return op(idx).getOutputType();
+  }
+
+  public  T opValueEvaluated(int idx, BeamSqlRow row) {
+return (T) op(idx).evaluate(row).getValue();
+  }
+
+  /**
+   * assertion to make sure the input and output are supported in this 
expression.
+   */
+  public abstract boolean accept();
+
+  /**
+   * Apply input record {@link BeamSqlRow} to this expression,
+   * the output value is wrapped with {@link BeamSqlPrimitive}.
+   */
+  public abstract BeamSqlPrimitive evaluate(BeamSqlRow inputRow);
+
+  public List getOperands() {
+return operands;
+  }
+
+  public SqlTypeName getOutputType() {
+return outputType;
+  }
+
+  public int numberOfOperands() {
+return operands.size();
+  }
+}

http://git-wip-us.apache.org/repos/asf/beam/blob/ba493f85/sdks/java/extensions/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/BeamSqlInputRefExpression.java
--
diff --git 
a/sdks/java/extensions/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/BeamSqlInputRefExpression.java
 
b/sdks/java/extensions/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/BeamSqlInputRefExpression.java
new file mode 100644
index 000..710460b
--- /dev/null
+++ 
b/sdks/java/extensions/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/BeamSqlInputRefExpression.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.dsls.sql.interpreter.operator;
+
+import org.apache.beam.dsls.sql.schema.BeamSqlRow;
+import org.apache.calcite.sql.type.SqlTypeName;
+
+/**
+ * An primitive operation for direct field extraction.
+ */
+public class BeamSqlInputRefExpression extends BeamSqlExpression {
+  private int 

[04/59] beam git commit: move dsls/sql to sdks/java/extensions/sql

[takidau]

http://git-wip-us.apache.org/repos/asf/beam/blob/ba493f85/sdks/java/extensions/sql/src/test/java/org/apache/beam/dsls/sql/integrationtest/BeamSqlDateFunctionsIntegrationTest.java
--
diff --git 
a/sdks/java/extensions/sql/src/test/java/org/apache/beam/dsls/sql/integrationtest/BeamSqlDateFunctionsIntegrationTest.java
 
b/sdks/java/extensions/sql/src/test/java/org/apache/beam/dsls/sql/integrationtest/BeamSqlDateFunctionsIntegrationTest.java
new file mode 100644
index 000..bd0d3ba
--- /dev/null
+++ 
b/sdks/java/extensions/sql/src/test/java/org/apache/beam/dsls/sql/integrationtest/BeamSqlDateFunctionsIntegrationTest.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.dsls.sql.integrationtest;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import java.util.Date;
+import java.util.Iterator;
+import org.apache.beam.dsls.sql.BeamSql;
+import org.apache.beam.dsls.sql.schema.BeamSqlRow;
+import org.apache.beam.sdk.testing.PAssert;
+import org.apache.beam.sdk.transforms.SerializableFunction;
+import org.apache.beam.sdk.values.PCollection;
+import org.junit.Test;
+
+/**
+ * Integration test for date functions.
+ */
+public class BeamSqlDateFunctionsIntegrationTest
+extends BeamSqlBuiltinFunctionsIntegrationTestBase {
+  @Test public void testDateTimeFunctions() throws Exception {
+ExpressionChecker checker = new ExpressionChecker()
+.addExpr("EXTRACT(YEAR FROM ts)", 1986L)
+.addExpr("YEAR(ts)", 1986L)
+.addExpr("QUARTER(ts)", 1L)
+.addExpr("MONTH(ts)", 2L)
+.addExpr("WEEK(ts)", 7L)
+.addExpr("DAYOFMONTH(ts)", 15L)
+.addExpr("DAYOFYEAR(ts)", 46L)
+.addExpr("DAYOFWEEK(ts)", 7L)
+.addExpr("HOUR(ts)", 11L)
+.addExpr("MINUTE(ts)", 35L)
+.addExpr("SECOND(ts)", 26L)
+.addExpr("FLOOR(ts TO YEAR)", parseDate("1986-01-01 00:00:00"))
+.addExpr("CEIL(ts TO YEAR)", parseDate("1987-01-01 00:00:00"))
+;
+checker.buildRunAndCheck();
+  }
+
+  @Test public void testDateTimeFunctions_currentTime() throws Exception {
+String sql = "SELECT "
++ "LOCALTIME as l,"
++ "LOCALTIMESTAMP as l1,"
++ "CURRENT_DATE as c1,"
++ "CURRENT_TIME as c2,"
++ "CURRENT_TIMESTAMP as c3"
++ " FROM PCOLLECTION"
+;
+PCollection rows = getTestPCollection().apply(
+BeamSql.simpleQuery(sql));
+PAssert.that(rows).satisfies(new Checker());
+pipeline.run();
+  }
+
+  private static class Checker implements 
SerializableFunction {
+@Override public Void apply(Iterable input) {
+  Iterator iter = input.iterator();
+  assertTrue(iter.hasNext());
+  BeamSqlRow row = iter.next();
+// LOCALTIME
+  Date date = new Date();
+  assertTrue(date.getTime() - 
row.getGregorianCalendar(0).getTime().getTime() < 1000);
+  assertTrue(date.getTime() - row.getDate(1).getTime() < 1000);
+  assertTrue(date.getTime() - row.getDate(2).getTime() < 1000);
+  assertTrue(date.getTime() - 
row.getGregorianCalendar(3).getTime().getTime() < 1000);
+  assertTrue(date.getTime() - row.getDate(4).getTime() < 1000);
+  assertFalse(iter.hasNext());
+  return null;
+}
+  }
+}

http://git-wip-us.apache.org/repos/asf/beam/blob/ba493f85/sdks/java/extensions/sql/src/test/java/org/apache/beam/dsls/sql/integrationtest/BeamSqlLogicalFunctionsIntegrationTest.java
--
diff --git 
a/sdks/java/extensions/sql/src/test/java/org/apache/beam/dsls/sql/integrationtest/BeamSqlLogicalFunctionsIntegrationTest.java
 
b/sdks/java/extensions/sql/src/test/java/org/apache/beam/dsls/sql/integrationtest/BeamSqlLogicalFunctionsIntegrationTest.java
new file mode 100644
index 000..4ed1f86
--- /dev/null
+++ 
b/sdks/java/extensions/sql/src/test/java/org/apache/beam/dsls/sql/integrationtest/BeamSqlLogicalFunctionsIntegrationTest.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the 

[05/59] beam git commit: move dsls/sql to sdks/java/extensions/sql

[takidau]

http://git-wip-us.apache.org/repos/asf/beam/blob/ba493f85/sdks/java/extensions/sql/src/test/java/org/apache/beam/dsls/sql/BeamSqlApiSurfaceTest.java
--
diff --git 
a/sdks/java/extensions/sql/src/test/java/org/apache/beam/dsls/sql/BeamSqlApiSurfaceTest.java
 
b/sdks/java/extensions/sql/src/test/java/org/apache/beam/dsls/sql/BeamSqlApiSurfaceTest.java
new file mode 100644
index 000..922931c
--- /dev/null
+++ 
b/sdks/java/extensions/sql/src/test/java/org/apache/beam/dsls/sql/BeamSqlApiSurfaceTest.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.dsls.sql;
+
+import static org.apache.beam.sdk.util.ApiSurface.containsOnlyPackages;
+import static org.junit.Assert.assertThat;
+
+import com.google.common.collect.ImmutableSet;
+import java.util.Set;
+import org.apache.beam.sdk.util.ApiSurface;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+/**
+ * Surface test for BeamSql api.
+ */
+@RunWith(JUnit4.class)
+public class BeamSqlApiSurfaceTest {
+  @Test
+  public void testSdkApiSurface() throws Exception {
+
+@SuppressWarnings("unchecked")
+final Set allowed =
+ImmutableSet.of(
+"org.apache.beam",
+"org.joda.time",
+"org.apache.commons.csv");
+
+ApiSurface surface = ApiSurface
+.ofClass(BeamSqlCli.class)
+.includingClass(BeamSql.class)
+.includingClass(BeamSqlEnv.class)
+.includingPackage("org.apache.beam.dsls.sql.schema",
+getClass().getClassLoader())
+.pruningPrefix("java")
+.pruningPattern("org[.]apache[.]beam[.]dsls[.]sql[.].*Test")
+.pruningPattern("org[.]apache[.]beam[.]dsls[.]sql[.].*TestBase");
+
+assertThat(surface, containsOnlyPackages(allowed));
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/beam/blob/ba493f85/sdks/java/extensions/sql/src/test/java/org/apache/beam/dsls/sql/BeamSqlDslAggregationTest.java
--
diff --git 
a/sdks/java/extensions/sql/src/test/java/org/apache/beam/dsls/sql/BeamSqlDslAggregationTest.java
 
b/sdks/java/extensions/sql/src/test/java/org/apache/beam/dsls/sql/BeamSqlDslAggregationTest.java
new file mode 100644
index 000..a142514
--- /dev/null
+++ 
b/sdks/java/extensions/sql/src/test/java/org/apache/beam/dsls/sql/BeamSqlDslAggregationTest.java
@@ -0,0 +1,380 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.dsls.sql;
+
+import java.sql.Types;
+import java.util.Arrays;
+import org.apache.beam.dsls.sql.schema.BeamSqlRow;
+import org.apache.beam.dsls.sql.schema.BeamSqlRowType;
+import org.apache.beam.sdk.testing.PAssert;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PCollectionTuple;
+import org.apache.beam.sdk.values.TupleTag;
+import org.joda.time.Instant;
+import org.junit.Test;
+
+/**
+ * Tests for GROUP-BY/aggregation, with 
global_window/fix_time_window/sliding_window/session_window
+ * with BOUNDED PCollection.
+ */
+public class BeamSqlDslAggregationTest extends BeamSqlDslBase {
+  /**
+   * GROUP-BY with single aggregation function with bounded PCollection.
+   */
+  @Test
+  public void testAggregationWithoutWindowWithBounded() throws Exception {
+runAggregationWithoutWindow(boundedInput1);
+  }
+
+  /**
+   * GROUP-BY with single 

[57/59] beam git commit: move all implementation classes/packages into impl package

[takidau]

http://git-wip-us.apache.org/repos/asf/beam/blob/febd044a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/arithmetic/BeamSqlMinusExpression.java
--
diff --git 
a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/arithmetic/BeamSqlMinusExpression.java
 
b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/arithmetic/BeamSqlMinusExpression.java
new file mode 100644
index 000..4fc6a4b
--- /dev/null
+++ 
b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/arithmetic/BeamSqlMinusExpression.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package 
org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.arithmetic;
+
+import java.math.BigDecimal;
+import java.util.List;
+import 
org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlExpression;
+
+/**
+ * '-' operator.
+ */
+public class BeamSqlMinusExpression extends BeamSqlArithmeticExpression {
+  public BeamSqlMinusExpression(List operands) {
+super(operands);
+  }
+
+  @Override protected BigDecimal calc(BigDecimal left, BigDecimal right) {
+return left.subtract(right);
+  }
+}

http://git-wip-us.apache.org/repos/asf/beam/blob/febd044a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/arithmetic/BeamSqlModExpression.java
--
diff --git 
a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/arithmetic/BeamSqlModExpression.java
 
b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/arithmetic/BeamSqlModExpression.java
new file mode 100644
index 000..5c55bf4
--- /dev/null
+++ 
b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/arithmetic/BeamSqlModExpression.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package 
org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.arithmetic;
+
+import java.math.BigDecimal;
+import java.util.List;
+import 
org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlExpression;
+
+/**
+ * '%' operator.
+ */
+public class BeamSqlModExpression extends BeamSqlArithmeticExpression {
+  public BeamSqlModExpression(List operands) {
+super(operands, operands.get(1).getOutputType());
+  }
+
+  @Override protected BigDecimal calc(BigDecimal left, BigDecimal right) {
+return BigDecimal.valueOf(left.doubleValue() % right.doubleValue());
+  }
+}

http://git-wip-us.apache.org/repos/asf/beam/blob/febd044a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/arithmetic/BeamSqlMultiplyExpression.java
--
diff --git 
a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/arithmetic/BeamSqlMultiplyExpression.java
 
b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/arithmetic/BeamSqlMultiplyExpression.java
new file mode 100644
index 000..e6cd35d
--- /dev/null
+++ 

[27/59] beam git commit: rename package org.apache.beam.dsls.sql to org.apache.beam.sdk.extensions.sql

[takidau]

http://git-wip-us.apache.org/repos/asf/beam/blob/c1b5482d/sdks/java/extensions/sql/src/test/java/org/apache/beam/dsls/sql/mock/MockedUnboundedTable.java
--
diff --git 
a/sdks/java/extensions/sql/src/test/java/org/apache/beam/dsls/sql/mock/MockedUnboundedTable.java
 
b/sdks/java/extensions/sql/src/test/java/org/apache/beam/dsls/sql/mock/MockedUnboundedTable.java
deleted file mode 100644
index ee6eb22..000
--- 
a/sdks/java/extensions/sql/src/test/java/org/apache/beam/dsls/sql/mock/MockedUnboundedTable.java
+++ /dev/null
@@ -1,114 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.beam.dsls.sql.mock;
-
-import static org.apache.beam.dsls.sql.TestUtils.buildBeamSqlRowType;
-import static org.apache.beam.dsls.sql.TestUtils.buildRows;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-import org.apache.beam.dsls.sql.schema.BeamIOType;
-import org.apache.beam.dsls.sql.schema.BeamSqlRow;
-import org.apache.beam.dsls.sql.schema.BeamSqlRowCoder;
-import org.apache.beam.dsls.sql.schema.BeamSqlRowType;
-import org.apache.beam.sdk.Pipeline;
-import org.apache.beam.sdk.testing.TestStream;
-import org.apache.beam.sdk.values.PCollection;
-import org.apache.beam.sdk.values.TimestampedValue;
-import org.apache.calcite.util.Pair;
-import org.joda.time.Duration;
-import org.joda.time.Instant;
-
-/**
- * A mocked unbounded table.
- */
-public class MockedUnboundedTable extends MockedTable {
-  /** rows flow out from this table with the specified watermark instant. */
-  private final List> timestampedRows = new 
ArrayList<>();
-  /** specify the index of column in the row which stands for the event time 
field. */
-  private int timestampField;
-  private MockedUnboundedTable(BeamSqlRowType beamSqlRowType) {
-super(beamSqlRowType);
-  }
-
-  /**
-   * Convenient way to build a mocked unbounded table.
-   *
-   * e.g.
-   *
-   * {@code
-   * MockedUnboundedTable
-   *   .of(Types.BIGINT, "order_id",
-   *   Types.INTEGER, "site_id",
-   *   Types.DOUBLE, "price",
-   *   Types.TIMESTAMP, "order_time")
-   * }
-   */
-  public static MockedUnboundedTable of(final Object... args){
-return new MockedUnboundedTable(buildBeamSqlRowType(args));
-  }
-
-  public MockedUnboundedTable timestampColumnIndex(int idx) {
-this.timestampField = idx;
-return this;
-  }
-
-  /**
-   * Add rows to the builder.
-   *
-   * Sample usage:
-   *
-   * {@code
-   * addRows(
-   *   duration,  -- duration which stands for the corresponding watermark 
instant
-   *   1, 3, "james", -- first row
-   *   2, 5, "bond"   -- second row
-   *   ...
-   * )
-   * }
-   */
-  public MockedUnboundedTable addRows(Duration duration, Object... args) {
-List rows = buildRows(getRowType(), Arrays.asList(args));
-// record the watermark + rows
-this.timestampedRows.add(Pair.of(duration, rows));
-return this;
-  }
-
-  @Override public BeamIOType getSourceType() {
-return BeamIOType.UNBOUNDED;
-  }
-
-  @Override public PCollection buildIOReader(Pipeline pipeline) {
-TestStream.Builder values = TestStream.create(
-new BeamSqlRowCoder(beamSqlRowType));
-
-for (Pair pair : timestampedRows) {
-  values = values.advanceWatermarkTo(new Instant(0).plus(pair.getKey()));
-  for (int i = 0; i < pair.getValue().size(); i++) {
-values = values.addElements(TimestampedValue.of(pair.getValue().get(i),
-new Instant(pair.getValue().get(i).getDate(timestampField;
-  }
-}
-
-return pipeline.begin().apply(
-"MockedUnboundedTable_" + COUNTER.incrementAndGet(),
-values.advanceWatermarkToInfinity());
-  }
-}

http://git-wip-us.apache.org/repos/asf/beam/blob/c1b5482d/sdks/java/extensions/sql/src/test/java/org/apache/beam/dsls/sql/rel/BeamIntersectRelTest.java
--
diff --git 
a/sdks/java/extensions/sql/src/test/java/org/apache/beam/dsls/sql/rel/BeamIntersectRelTest.java
 
b/sdks/java/extensions/sql/src/test/java/org/apache/beam/dsls/sql/rel/BeamIntersectRelTest.java
deleted 

[06/59] beam git commit: move dsls/sql to sdks/java/extensions/sql

[takidau]

http://git-wip-us.apache.org/repos/asf/beam/blob/ba493f85/sdks/java/extensions/sql/src/main/java/org/apache/beam/dsls/sql/schema/BeamSqlUdf.java
--
diff --git 
a/sdks/java/extensions/sql/src/main/java/org/apache/beam/dsls/sql/schema/BeamSqlUdf.java
 
b/sdks/java/extensions/sql/src/main/java/org/apache/beam/dsls/sql/schema/BeamSqlUdf.java
new file mode 100644
index 000..2066353
--- /dev/null
+++ 
b/sdks/java/extensions/sql/src/main/java/org/apache/beam/dsls/sql/schema/BeamSqlUdf.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.dsls.sql.schema;
+
+import java.io.Serializable;
+
+/**
+ * Interface to create a UDF in Beam SQL.
+ *
+ * A static method {@code eval} is required. Here is an example:
+ *
+ * 
+ * public static class MyLeftFunction {
+ *   public String eval(
+ *   Parameter(name = "s") String s,
+ *   Parameter(name = "n", optional = true) Integer n) {
+ * return s.substring(0, n == null ? 1 : n);
+ *   }
+ * }
+ *
+ * The first parameter is named "s" and is mandatory,
+ * and the second parameter is named "n" and is optional.
+ */
+public interface BeamSqlUdf extends Serializable {
+  String UDF_METHOD = "eval";
+}

http://git-wip-us.apache.org/repos/asf/beam/blob/ba493f85/sdks/java/extensions/sql/src/main/java/org/apache/beam/dsls/sql/schema/BeamTableUtils.java
--
diff --git 
a/sdks/java/extensions/sql/src/main/java/org/apache/beam/dsls/sql/schema/BeamTableUtils.java
 
b/sdks/java/extensions/sql/src/main/java/org/apache/beam/dsls/sql/schema/BeamTableUtils.java
new file mode 100644
index 000..4b7e76b
--- /dev/null
+++ 
b/sdks/java/extensions/sql/src/main/java/org/apache/beam/dsls/sql/schema/BeamTableUtils.java
@@ -0,0 +1,122 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.dsls.sql.schema;
+
+import java.io.IOException;
+import java.io.StringReader;
+import java.io.StringWriter;
+import java.math.BigDecimal;
+import org.apache.beam.dsls.sql.utils.CalciteUtils;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.apache.calcite.util.NlsString;
+import org.apache.commons.csv.CSVFormat;
+import org.apache.commons.csv.CSVParser;
+import org.apache.commons.csv.CSVPrinter;
+import org.apache.commons.csv.CSVRecord;
+
+/**
+ * Utility methods for working with {@code BeamTable}.
+ */
+public final class BeamTableUtils {
+  public static BeamSqlRow csvLine2BeamSqlRow(
+  CSVFormat csvFormat,
+  String line,
+  BeamSqlRowType beamSqlRowType) {
+BeamSqlRow row = new BeamSqlRow(beamSqlRowType);
+try (StringReader reader = new StringReader(line)) {
+  CSVParser parser = csvFormat.parse(reader);
+  CSVRecord rawRecord = parser.getRecords().get(0);
+
+  if (rawRecord.size() != beamSqlRowType.size()) {
+throw new IllegalArgumentException(String.format(
+"Expect %d fields, but actually %d",
+beamSqlRowType.size(), rawRecord.size()
+));
+  } else {
+for (int idx = 0; idx < beamSqlRowType.size(); idx++) {
+  String raw = rawRecord.get(idx);
+  addFieldWithAutoTypeCasting(row, idx, raw);
+}
+  }
+} catch (IOException e) {
+  throw new IllegalArgumentException("decodeRecord failed!", e);
+}
+return row;
+  }
+
+  public static String 

[19/59] beam git commit: move dsls/sql to sdks/java/extensions/sql

[takidau]

http://git-wip-us.apache.org/repos/asf/beam/blob/ba493f85/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/date/BeamSqlExtractExpression.java
--
diff --git 
a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/date/BeamSqlExtractExpression.java
 
b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/date/BeamSqlExtractExpression.java
deleted file mode 100644
index d41a249..000
--- 
a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/date/BeamSqlExtractExpression.java
+++ /dev/null
@@ -1,101 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.beam.dsls.sql.interpreter.operator.date;
-
-import java.util.Calendar;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import org.apache.beam.dsls.sql.interpreter.operator.BeamSqlExpression;
-import org.apache.beam.dsls.sql.interpreter.operator.BeamSqlPrimitive;
-import org.apache.beam.dsls.sql.schema.BeamSqlRow;
-import org.apache.calcite.avatica.util.DateTimeUtils;
-import org.apache.calcite.avatica.util.TimeUnitRange;
-import org.apache.calcite.sql.type.SqlTypeName;
-
-/**
- * {@code BeamSqlExpression} for EXTRACT.
- *
- * The following date functions also implicitly converted to {@code 
EXTRACT}:
- * 
- *   YEAR(date) = EXTRACT(YEAR FROM date)
- *   MONTH(date) = EXTRACT(MONTH FROM date)
- *   DAY(date) = EXTRACT(DAY FROM date)
- *   QUARTER(date) = EXTRACT(QUARTER FROM date)
- *   WEEK(date) = EXTRACT(WEEK FROM date)
- *   DAYOFYEAR(date) = EXTRACT(DOY FROM date)
- *   DAYOFMONTH(date) = EXTRACT(DAY FROM date)
- *   DAYOFWEEK(date) = EXTRACT(DOW FROM date)
- * 
- */
-public class BeamSqlExtractExpression extends BeamSqlExpression {
-  private static final Map typeMapping = new 
HashMap<>();
-  static {
-typeMapping.put(TimeUnitRange.DOW, Calendar.DAY_OF_WEEK);
-typeMapping.put(TimeUnitRange.DOY, Calendar.DAY_OF_YEAR);
-typeMapping.put(TimeUnitRange.WEEK, Calendar.WEEK_OF_YEAR);
-  }
-
-  public BeamSqlExtractExpression(List operands) {
-super(operands, SqlTypeName.BIGINT);
-  }
-  @Override public boolean accept() {
-return operands.size() == 2
-&& opType(1) == SqlTypeName.BIGINT;
-  }
-
-  @Override public BeamSqlPrimitive evaluate(BeamSqlRow inputRow) {
-Long time = opValueEvaluated(1, inputRow);
-
-TimeUnitRange unit = ((BeamSqlPrimitive) op(0)).getValue();
-
-switch (unit) {
-  case YEAR:
-  case MONTH:
-  case DAY:
-Long timeByDay = time / 1000 / 3600 / 24;
-Long extracted = DateTimeUtils.unixDateExtract(
-unit,
-timeByDay
-);
-return BeamSqlPrimitive.of(outputType, extracted);
-
-  case DOY:
-  case DOW:
-  case WEEK:
-Calendar calendar = Calendar.getInstance();
-calendar.setTime(new Date(time));
-return BeamSqlPrimitive.of(outputType, (long) 
calendar.get(typeMapping.get(unit)));
-
-  case QUARTER:
-calendar = Calendar.getInstance();
-calendar.setTime(new Date(time));
-long ret = calendar.get(Calendar.MONTH) / 3;
-if (ret * 3 < calendar.get(Calendar.MONTH)) {
-  ret += 1;
-}
-return BeamSqlPrimitive.of(outputType, ret);
-
-  default:
-throw new UnsupportedOperationException(
-"Extract for time unit: " + unit + " not supported!");
-}
-  }
-}

http://git-wip-us.apache.org/repos/asf/beam/blob/ba493f85/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/date/package-info.java
--
diff --git 
a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/date/package-info.java
 
b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/date/package-info.java
deleted file mode 100644
index d3cc98f..000
--- 
a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/date/package-info.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor 

[43/59] beam git commit: move all implementation classes/packages into impl package

[takidau]

http://git-wip-us.apache.org/repos/asf/beam/blob/febd044a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/schema/transform/BeamTransformBaseTest.java
--
diff --git 
a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/schema/transform/BeamTransformBaseTest.java
 
b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/schema/transform/BeamTransformBaseTest.java
index b2aa6c4..af7ec23 100644
--- 
a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/schema/transform/BeamTransformBaseTest.java
+++ 
b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/schema/transform/BeamTransformBaseTest.java
@@ -22,10 +22,10 @@ import java.text.ParseException;
 import java.text.SimpleDateFormat;
 import java.util.Arrays;
 import java.util.List;
-import org.apache.beam.sdk.extensions.sql.planner.BeamQueryPlanner;
+import org.apache.beam.sdk.extensions.sql.impl.planner.BeamQueryPlanner;
+import org.apache.beam.sdk.extensions.sql.impl.utils.CalciteUtils;
 import org.apache.beam.sdk.extensions.sql.schema.BeamSqlRow;
 import org.apache.beam.sdk.extensions.sql.schema.BeamSqlRowType;
-import org.apache.beam.sdk.extensions.sql.utils.CalciteUtils;
 import org.apache.beam.sdk.values.KV;
 import org.apache.calcite.rel.type.RelDataTypeFactory.FieldInfoBuilder;
 import org.apache.calcite.sql.type.SqlTypeName;

[58/59] beam git commit: move all implementation classes/packages into impl package

[takidau]

move all implementation classes/packages into impl package


Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/febd044a
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/febd044a
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/febd044a

Branch: refs/heads/DSL_SQL
Commit: febd044ae306a28fa3797a1663e54c1d7fbe43ce
Parents: c1b5482
Author: James Xu 
Authored: Mon Jul 31 17:11:53 2017 +0800
Committer: James Xu 
Committed: Mon Jul 31 17:11:53 2017 +0800

--
 .../apache/beam/sdk/extensions/sql/BeamSql.java |   2 +-
 .../beam/sdk/extensions/sql/BeamSqlCli.java |   2 +-
 .../beam/sdk/extensions/sql/BeamSqlEnv.java |   4 +-
 .../interpreter/BeamSqlExpressionExecutor.java  |  43 ++
 .../sql/impl/interpreter/BeamSqlFnExecutor.java | 442 +++
 .../operator/BeamSqlCaseExpression.java |  63 +++
 .../operator/BeamSqlCastExpression.java | 131 ++
 .../interpreter/operator/BeamSqlExpression.java |  78 
 .../operator/BeamSqlInputRefExpression.java |  43 ++
 .../interpreter/operator/BeamSqlPrimitive.java  | 152 +++
 .../operator/BeamSqlReinterpretExpression.java  |  54 +++
 .../operator/BeamSqlUdfExpression.java  |  86 
 .../operator/BeamSqlWindowEndExpression.java|  42 ++
 .../operator/BeamSqlWindowExpression.java   |  50 +++
 .../operator/BeamSqlWindowStartExpression.java  |  43 ++
 .../arithmetic/BeamSqlArithmeticExpression.java | 122 +
 .../arithmetic/BeamSqlDivideExpression.java |  37 ++
 .../arithmetic/BeamSqlMinusExpression.java  |  36 ++
 .../arithmetic/BeamSqlModExpression.java|  36 ++
 .../arithmetic/BeamSqlMultiplyExpression.java   |  36 ++
 .../arithmetic/BeamSqlPlusExpression.java   |  36 ++
 .../operator/arithmetic/package-info.java   |  22 +
 .../comparison/BeamSqlCompareExpression.java|  96 
 .../comparison/BeamSqlEqualsExpression.java |  49 ++
 .../BeamSqlGreaterThanExpression.java   |  49 ++
 .../BeamSqlGreaterThanOrEqualsExpression.java   |  49 ++
 .../comparison/BeamSqlIsNotNullExpression.java  |  53 +++
 .../comparison/BeamSqlIsNullExpression.java |  53 +++
 .../comparison/BeamSqlLessThanExpression.java   |  49 ++
 .../BeamSqlLessThanOrEqualsExpression.java  |  49 ++
 .../comparison/BeamSqlNotEqualsExpression.java  |  49 ++
 .../operator/comparison/package-info.java   |  22 +
 .../date/BeamSqlCurrentDateExpression.java  |  44 ++
 .../date/BeamSqlCurrentTimeExpression.java  |  52 +++
 .../date/BeamSqlCurrentTimestampExpression.java |  48 ++
 .../date/BeamSqlDateCeilExpression.java |  54 +++
 .../date/BeamSqlDateFloorExpression.java|  54 +++
 .../operator/date/BeamSqlExtractExpression.java | 101 +
 .../interpreter/operator/date/package-info.java |  22 +
 .../operator/logical/BeamSqlAndExpression.java  |  47 ++
 .../logical/BeamSqlLogicalExpression.java   |  46 ++
 .../operator/logical/BeamSqlNotExpression.java  |  53 +++
 .../operator/logical/BeamSqlOrExpression.java   |  47 ++
 .../operator/logical/package-info.java  |  22 +
 .../operator/math/BeamSqlAbsExpression.java |  74 
 .../operator/math/BeamSqlAcosExpression.java|  40 ++
 .../operator/math/BeamSqlAsinExpression.java|  40 ++
 .../operator/math/BeamSqlAtan2Expression.java   |  42 ++
 .../operator/math/BeamSqlAtanExpression.java|  40 ++
 .../operator/math/BeamSqlCeilExpression.java|  45 ++
 .../operator/math/BeamSqlCosExpression.java |  40 ++
 .../operator/math/BeamSqlCotExpression.java |  40 ++
 .../operator/math/BeamSqlDegreesExpression.java |  40 ++
 .../operator/math/BeamSqlExpExpression.java |  40 ++
 .../operator/math/BeamSqlFloorExpression.java   |  45 ++
 .../operator/math/BeamSqlLnExpression.java  |  40 ++
 .../operator/math/BeamSqlLogExpression.java |  40 ++
 .../math/BeamSqlMathBinaryExpression.java   |  63 +++
 .../math/BeamSqlMathUnaryExpression.java|  58 +++
 .../operator/math/BeamSqlPiExpression.java  |  42 ++
 .../operator/math/BeamSqlPowerExpression.java   |  44 ++
 .../operator/math/BeamSqlRadiansExpression.java |  40 ++
 .../operator/math/BeamSqlRandExpression.java|  54 +++
 .../math/BeamSqlRandIntegerExpression.java  |  58 +++
 .../operator/math/BeamSqlRoundExpression.java   | 107 +
 .../operator/math/BeamSqlSignExpression.java|  72 +++
 .../operator/math/BeamSqlSinExpression.java |  40 ++
 .../operator/math/BeamSqlTanExpression.java |  40 ++
 .../math/BeamSqlTruncateExpression.java |  75 
 .../interpreter/operator/math/package-info.java |  22 +
 .../impl/interpreter/operator/package-info.java |  22 +
 .../string/BeamSqlCharLengthExpression.java |  39 ++
 .../string/BeamSqlConcatExpression.java |  62 +++
 .../string/BeamSqlInitCapExpression.java|  55 +++
 

[52/59] beam git commit: move all implementation classes/packages into impl package

[takidau]

http://git-wip-us.apache.org/repos/asf/beam/blob/febd044a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/interpreter/operator/comparison/BeamSqlGreaterThanOrEqualsExpression.java
--
diff --git 
a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/interpreter/operator/comparison/BeamSqlGreaterThanOrEqualsExpression.java
 
b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/interpreter/operator/comparison/BeamSqlGreaterThanOrEqualsExpression.java
deleted file mode 100644
index ae22054..000
--- 
a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/interpreter/operator/comparison/BeamSqlGreaterThanOrEqualsExpression.java
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.beam.sdk.extensions.sql.interpreter.operator.comparison;
-
-import java.util.List;
-import 
org.apache.beam.sdk.extensions.sql.interpreter.operator.BeamSqlExpression;
-
-/**
- * {@code BeamSqlExpression} for {@code >=} operation.
- */
-public class BeamSqlGreaterThanOrEqualsExpression extends 
BeamSqlCompareExpression {
-
-  public BeamSqlGreaterThanOrEqualsExpression(List 
operands) {
-super(operands);
-  }
-
-  @Override
-  public Boolean compare(CharSequence leftValue, CharSequence rightValue) {
-return String.valueOf(leftValue).compareTo(String.valueOf(rightValue)) >= 
0;
-  }
-
-  @Override
-  public Boolean compare(Boolean leftValue, Boolean rightValue) {
-throw new IllegalArgumentException(">= is not supported for Boolean.");
-  }
-
-  @Override
-  public Boolean compare(Number leftValue, Number rightValue) {
-return (leftValue == null && rightValue == null)
-|| (leftValue != null && rightValue != null
-  && leftValue.floatValue() >= (rightValue).floatValue());
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/beam/blob/febd044a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/interpreter/operator/comparison/BeamSqlIsNotNullExpression.java
--
diff --git 
a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/interpreter/operator/comparison/BeamSqlIsNotNullExpression.java
 
b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/interpreter/operator/comparison/BeamSqlIsNotNullExpression.java
deleted file mode 100644
index 78660cb..000
--- 
a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/interpreter/operator/comparison/BeamSqlIsNotNullExpression.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.beam.sdk.extensions.sql.interpreter.operator.comparison;
-
-import java.util.Arrays;
-import java.util.List;
-import 
org.apache.beam.sdk.extensions.sql.interpreter.operator.BeamSqlExpression;
-import 
org.apache.beam.sdk.extensions.sql.interpreter.operator.BeamSqlPrimitive;
-import org.apache.beam.sdk.extensions.sql.schema.BeamSqlRow;
-import org.apache.calcite.sql.type.SqlTypeName;
-
-/**
- * {@code BeamSqlExpression} for 'IS NOT NULL' operation.
- */
-public class BeamSqlIsNotNullExpression extends BeamSqlExpression {
-
-  private BeamSqlIsNotNullExpression(List operands, 
SqlTypeName outputType) {
-super(operands, outputType);
-  }
-
-  public 

[45/59] beam git commit: move all implementation classes/packages into impl package

[takidau]

http://git-wip-us.apache.org/repos/asf/beam/blob/febd044a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/interpreter/operator/BeamSqlCompareExpressionTest.java
--
diff --git 
a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/interpreter/operator/BeamSqlCompareExpressionTest.java
 
b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/interpreter/operator/BeamSqlCompareExpressionTest.java
deleted file mode 100644
index ed77ffb..000
--- 
a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/interpreter/operator/BeamSqlCompareExpressionTest.java
+++ /dev/null
@@ -1,115 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.beam.sdk.extensions.sql.interpreter.operator;
-
-import java.util.Arrays;
-import 
org.apache.beam.sdk.extensions.sql.interpreter.BeamSqlFnExecutorTestBase;
-import 
org.apache.beam.sdk.extensions.sql.interpreter.operator.comparison.BeamSqlCompareExpression;
-import 
org.apache.beam.sdk.extensions.sql.interpreter.operator.comparison.BeamSqlEqualsExpression;
-import 
org.apache.beam.sdk.extensions.sql.interpreter.operator.comparison.BeamSqlGreaterThanExpression;
-import 
org.apache.beam.sdk.extensions.sql.interpreter.operator.comparison.BeamSqlGreaterThanOrEqualsExpression;
-import 
org.apache.beam.sdk.extensions.sql.interpreter.operator.comparison.BeamSqlLessThanExpression;
-import 
org.apache.beam.sdk.extensions.sql.interpreter.operator.comparison.BeamSqlLessThanOrEqualsExpression;
-import 
org.apache.beam.sdk.extensions.sql.interpreter.operator.comparison.BeamSqlNotEqualsExpression;
-import org.apache.calcite.sql.type.SqlTypeName;
-import org.junit.Assert;
-import org.junit.Test;
-
-/**
- * Test cases for the collections of {@link BeamSqlCompareExpression}.
- */
-public class BeamSqlCompareExpressionTest extends BeamSqlFnExecutorTestBase {
-
-  @Test
-  public void testEqual() {
-BeamSqlEqualsExpression exp1 = new BeamSqlEqualsExpression(
-Arrays.asList(new BeamSqlInputRefExpression(SqlTypeName.BIGINT, 0),
-BeamSqlPrimitive.of(SqlTypeName.BIGINT, 100L)));
-Assert.assertEquals(false, exp1.evaluate(record).getValue());
-
-BeamSqlEqualsExpression exp2 = new BeamSqlEqualsExpression(
-Arrays.asList(new BeamSqlInputRefExpression(SqlTypeName.BIGINT, 0),
-BeamSqlPrimitive.of(SqlTypeName.BIGINT, 1234567L)));
-Assert.assertEquals(true, exp2.evaluate(record).getValue());
-  }
-
-  @Test
-  public void testLargerThan(){
-BeamSqlGreaterThanExpression exp1 = new BeamSqlGreaterThanExpression(
-Arrays.asList(new BeamSqlInputRefExpression(SqlTypeName.BIGINT, 0),
-BeamSqlPrimitive.of(SqlTypeName.BIGINT, 1234567L)));
-Assert.assertEquals(false, exp1.evaluate(record).getValue());
-
-BeamSqlGreaterThanExpression exp2 = new BeamSqlGreaterThanExpression(
-Arrays.asList(new BeamSqlInputRefExpression(SqlTypeName.BIGINT, 0),
-BeamSqlPrimitive.of(SqlTypeName.BIGINT, 1234566L)));
-Assert.assertEquals(true, exp2.evaluate(record).getValue());
-  }
-
-  @Test
-  public void testLargerThanEqual(){
-BeamSqlGreaterThanOrEqualsExpression exp1 = new 
BeamSqlGreaterThanOrEqualsExpression(
-Arrays.asList(new BeamSqlInputRefExpression(SqlTypeName.BIGINT, 0),
-BeamSqlPrimitive.of(SqlTypeName.BIGINT, 1234567L)));
-Assert.assertEquals(true, exp1.evaluate(record).getValue());
-
-BeamSqlGreaterThanOrEqualsExpression exp2 = new 
BeamSqlGreaterThanOrEqualsExpression(
-Arrays.asList(new BeamSqlInputRefExpression(SqlTypeName.BIGINT, 0),
-BeamSqlPrimitive.of(SqlTypeName.BIGINT, 1234568L)));
-Assert.assertEquals(false, exp2.evaluate(record).getValue());
-  }
-
-  @Test
-  public void testLessThan(){
-BeamSqlLessThanExpression exp1 = new BeamSqlLessThanExpression(
-Arrays.asList(new BeamSqlInputRefExpression(SqlTypeName.INTEGER, 1),
-BeamSqlPrimitive.of(SqlTypeName.INTEGER, 1)));
-Assert.assertEquals(true, exp1.evaluate(record).getValue());
-
-BeamSqlLessThanExpression exp2 = new 

[26/59] beam git commit: rename package org.apache.beam.dsls.sql to org.apache.beam.sdk.extensions.sql

[takidau]

http://git-wip-us.apache.org/repos/asf/beam/blob/c1b5482d/sdks/java/extensions/sql/src/test/java/org/apache/beam/dsls/sql/schema/transform/BeamAggregationTransformTest.java
--
diff --git 
a/sdks/java/extensions/sql/src/test/java/org/apache/beam/dsls/sql/schema/transform/BeamAggregationTransformTest.java
 
b/sdks/java/extensions/sql/src/test/java/org/apache/beam/dsls/sql/schema/transform/BeamAggregationTransformTest.java
deleted file mode 100644
index 5d5d4fc..000
--- 
a/sdks/java/extensions/sql/src/test/java/org/apache/beam/dsls/sql/schema/transform/BeamAggregationTransformTest.java
+++ /dev/null
@@ -1,453 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * 
- * http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.beam.dsls.sql.schema.transform;
-
-import java.text.ParseException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-import org.apache.beam.dsls.sql.planner.BeamQueryPlanner;
-import org.apache.beam.dsls.sql.schema.BeamSqlRow;
-import org.apache.beam.dsls.sql.schema.BeamSqlRowCoder;
-import org.apache.beam.dsls.sql.schema.BeamSqlRowType;
-import org.apache.beam.dsls.sql.transform.BeamAggregationTransforms;
-import org.apache.beam.dsls.sql.utils.CalciteUtils;
-import org.apache.beam.sdk.coders.IterableCoder;
-import org.apache.beam.sdk.coders.KvCoder;
-import org.apache.beam.sdk.testing.PAssert;
-import org.apache.beam.sdk.testing.TestPipeline;
-import org.apache.beam.sdk.transforms.Combine;
-import org.apache.beam.sdk.transforms.Create;
-import org.apache.beam.sdk.transforms.GroupByKey;
-import org.apache.beam.sdk.transforms.ParDo;
-import org.apache.beam.sdk.transforms.WithKeys;
-import org.apache.beam.sdk.values.KV;
-import org.apache.beam.sdk.values.PCollection;
-import org.apache.calcite.rel.core.AggregateCall;
-import org.apache.calcite.rel.type.RelDataTypeFactory.FieldInfoBuilder;
-import org.apache.calcite.rel.type.RelDataTypeSystem;
-import org.apache.calcite.sql.SqlKind;
-import org.apache.calcite.sql.fun.SqlAvgAggFunction;
-import org.apache.calcite.sql.fun.SqlCountAggFunction;
-import org.apache.calcite.sql.fun.SqlMinMaxAggFunction;
-import org.apache.calcite.sql.fun.SqlSumAggFunction;
-import org.apache.calcite.sql.type.BasicSqlType;
-import org.apache.calcite.sql.type.SqlTypeName;
-import org.apache.calcite.util.ImmutableBitSet;
-import org.junit.Rule;
-import org.junit.Test;
-
-/**
- * Unit tests for {@link BeamAggregationTransforms}.
- *
- */
-public class BeamAggregationTransformTest extends BeamTransformBaseTest{
-
-  @Rule
-  public TestPipeline p = TestPipeline.create();
-
-  private List aggCalls;
-
-  private BeamSqlRowType keyType;
-  private BeamSqlRowType aggPartType;
-  private BeamSqlRowType outputType;
-
-  private BeamSqlRowCoder inRecordCoder;
-  private BeamSqlRowCoder keyCoder;
-  private BeamSqlRowCoder aggCoder;
-  private BeamSqlRowCoder outRecordCoder;
-
-  /**
-   * This step equals to below query.
-   * 
-   * SELECT `f_int`
-   * , COUNT(*) AS `size`
-   * , SUM(`f_long`) AS `sum1`, AVG(`f_long`) AS `avg1`
-   * , MAX(`f_long`) AS `max1`, MIN(`f_long`) AS `min1`
-   * , SUM(`f_short`) AS `sum2`, AVG(`f_short`) AS `avg2`
-   * , MAX(`f_short`) AS `max2`, MIN(`f_short`) AS `min2`
-   * , SUM(`f_byte`) AS `sum3`, AVG(`f_byte`) AS `avg3`
-   * , MAX(`f_byte`) AS `max3`, MIN(`f_byte`) AS `min3`
-   * , SUM(`f_float`) AS `sum4`, AVG(`f_float`) AS `avg4`
-   * , MAX(`f_float`) AS `max4`, MIN(`f_float`) AS `min4`
-   * , SUM(`f_double`) AS `sum5`, AVG(`f_double`) AS `avg5`
-   * , MAX(`f_double`) AS `max5`, MIN(`f_double`) AS `min5`
-   * , MAX(`f_timestamp`) AS `max7`, MIN(`f_timestamp`) AS `min7`
-   * ,SUM(`f_int2`) AS `sum8`, AVG(`f_int2`) AS `avg8`
-   * , MAX(`f_int2`) AS `max8`, MIN(`f_int2`) AS `min8`
-   * FROM TABLE_NAME
-   * GROUP BY `f_int`
-   * 
-   * @throws ParseException
-   */
-  @Test
-  public void testCountPerElementBasic() throws ParseException {
-setupEnvironment();
-
-PCollection input = p.apply(Create.of(inputRows));
-
-//1. extract fields in group-by key part
-PCollection> exGroupByStream = 
input.apply("exGroupBy",
-WithKeys
-.of(new 

[37/59] beam git commit: rename package org.apache.beam.dsls.sql to org.apache.beam.sdk.extensions.sql

[takidau]

http://git-wip-us.apache.org/repos/asf/beam/blob/c1b5482d/sdks/java/extensions/sql/src/main/java/org/apache/beam/dsls/sql/schema/BeamSqlUdaf.java
--
diff --git 
a/sdks/java/extensions/sql/src/main/java/org/apache/beam/dsls/sql/schema/BeamSqlUdaf.java
 
b/sdks/java/extensions/sql/src/main/java/org/apache/beam/dsls/sql/schema/BeamSqlUdaf.java
deleted file mode 100644
index 9582ffa..000
--- 
a/sdks/java/extensions/sql/src/main/java/org/apache/beam/dsls/sql/schema/BeamSqlUdaf.java
+++ /dev/null
@@ -1,72 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.beam.dsls.sql.schema;
-
-import java.io.Serializable;
-import java.lang.reflect.ParameterizedType;
-import org.apache.beam.sdk.coders.CannotProvideCoderException;
-import org.apache.beam.sdk.coders.Coder;
-import org.apache.beam.sdk.coders.CoderRegistry;
-import org.apache.beam.sdk.transforms.Combine.CombineFn;
-
-/**
- * abstract class of aggregation functions in Beam SQL.
- *
- * There're several constrains for a UDAF:
- * 1. A constructor with an empty argument list is required;
- * 2. The type of {@code InputT} and {@code OutputT} can only be 
Interger/Long/Short/Byte/Double
- * /Float/Date/BigDecimal, mapping as SQL type 
INTEGER/BIGINT/SMALLINT/TINYINE/DOUBLE/FLOAT
- * /TIMESTAMP/DECIMAL;
- * 3. Keep intermediate data in {@code AccumT}, and do not rely on elements in 
class;
- */
-public abstract class BeamSqlUdaf implements 
Serializable {
-  public BeamSqlUdaf(){}
-
-  /**
-   * create an initial aggregation object, equals to {@link 
CombineFn#createAccumulator()}.
-   */
-  public abstract AccumT init();
-
-  /**
-   * add an input value, equals to {@link CombineFn#addInput(Object, Object)}.
-   */
-  public abstract AccumT add(AccumT accumulator, InputT input);
-
-  /**
-   * merge aggregation objects from parallel tasks, equals to
-   *  {@link CombineFn#mergeAccumulators(Iterable)}.
-   */
-  public abstract AccumT merge(Iterable accumulators);
-
-  /**
-   * extract output value from aggregation object, equals to
-   * {@link CombineFn#extractOutput(Object)}.
-   */
-  public abstract OutputT result(AccumT accumulator);
-
-  /**
-   * get the coder for AccumT which stores the intermediate result.
-   * By default it's fetched from {@link CoderRegistry}.
-   */
-  public Coder getAccumulatorCoder(CoderRegistry registry)
-  throws CannotProvideCoderException {
-return registry.getCoder(
-(Class) ((ParameterizedType) getClass()
-.getGenericSuperclass()).getActualTypeArguments()[1]);
-  }
-}

http://git-wip-us.apache.org/repos/asf/beam/blob/c1b5482d/sdks/java/extensions/sql/src/main/java/org/apache/beam/dsls/sql/schema/BeamSqlUdf.java
--
diff --git 
a/sdks/java/extensions/sql/src/main/java/org/apache/beam/dsls/sql/schema/BeamSqlUdf.java
 
b/sdks/java/extensions/sql/src/main/java/org/apache/beam/dsls/sql/schema/BeamSqlUdf.java
deleted file mode 100644
index 2066353..000
--- 
a/sdks/java/extensions/sql/src/main/java/org/apache/beam/dsls/sql/schema/BeamSqlUdf.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.beam.dsls.sql.schema;
-
-import java.io.Serializable;
-
-/**
- * Interface to create a UDF in Beam SQL.
- *
- * A static method {@code eval} is required. Here is 

[29/59] beam git commit: rename package org.apache.beam.dsls.sql to org.apache.beam.sdk.extensions.sql

[takidau]

http://git-wip-us.apache.org/repos/asf/beam/blob/c1b5482d/sdks/java/extensions/sql/src/test/java/org/apache/beam/dsls/sql/integrationtest/BeamSqlMathFunctionsIntegrationTest.java
--
diff --git 
a/sdks/java/extensions/sql/src/test/java/org/apache/beam/dsls/sql/integrationtest/BeamSqlMathFunctionsIntegrationTest.java
 
b/sdks/java/extensions/sql/src/test/java/org/apache/beam/dsls/sql/integrationtest/BeamSqlMathFunctionsIntegrationTest.java
deleted file mode 100644
index 9f7d917..000
--- 
a/sdks/java/extensions/sql/src/test/java/org/apache/beam/dsls/sql/integrationtest/BeamSqlMathFunctionsIntegrationTest.java
+++ /dev/null
@@ -1,351 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.beam.dsls.sql.integrationtest;
-
-import java.math.BigDecimal;
-import java.util.Random;
-import org.apache.calcite.runtime.SqlFunctions;
-import org.junit.Test;
-
-/**
- * Integration test for built-in MATH functions.
- */
-public class BeamSqlMathFunctionsIntegrationTest
-extends BeamSqlBuiltinFunctionsIntegrationTestBase {
-  private static final int INTEGER_VALUE = 1;
-  private static final long LONG_VALUE = 1L;
-  private static final short SHORT_VALUE = 1;
-  private static final byte BYTE_VALUE = 1;
-  private static final double DOUBLE_VALUE = 1.0;
-  private static final float FLOAT_VALUE = 1.0f;
-  private static final BigDecimal DECIMAL_VALUE = new BigDecimal(1);
-
-  @Test
-  public void testAbs() throws Exception{
-ExpressionChecker checker = new ExpressionChecker()
-.addExpr("ABS(c_integer)", Math.abs(INTEGER_VALUE))
-.addExpr("ABS(c_bigint)", Math.abs(LONG_VALUE))
-.addExpr("ABS(c_smallint)", (short) Math.abs(SHORT_VALUE))
-.addExpr("ABS(c_tinyint)", (byte) Math.abs(BYTE_VALUE))
-.addExpr("ABS(c_double)", Math.abs(DOUBLE_VALUE))
-.addExpr("ABS(c_float)", Math.abs(FLOAT_VALUE))
-.addExpr("ABS(c_decimal)", new 
BigDecimal(Math.abs(DECIMAL_VALUE.doubleValue(
-;
-
-checker.buildRunAndCheck();
-  }
-
-  @Test
-  public void testSqrt() throws Exception{
-ExpressionChecker checker = new ExpressionChecker()
-.addExpr("SQRT(c_integer)", Math.sqrt(INTEGER_VALUE))
-.addExpr("SQRT(c_bigint)", Math.sqrt(LONG_VALUE))
-.addExpr("SQRT(c_smallint)", Math.sqrt(SHORT_VALUE))
-.addExpr("SQRT(c_tinyint)", Math.sqrt(BYTE_VALUE))
-.addExpr("SQRT(c_double)", Math.sqrt(DOUBLE_VALUE))
-.addExpr("SQRT(c_float)", Math.sqrt(FLOAT_VALUE))
-.addExpr("SQRT(c_decimal)", Math.sqrt(DECIMAL_VALUE.doubleValue()))
-;
-
-checker.buildRunAndCheck();
-  }
-
-  @Test
-  public void testRound() throws Exception{
-ExpressionChecker checker = new ExpressionChecker()
-.addExpr("ROUND(c_integer, 0)", SqlFunctions.sround(INTEGER_VALUE, 0))
-.addExpr("ROUND(c_bigint, 0)", SqlFunctions.sround(LONG_VALUE, 0))
-.addExpr("ROUND(c_smallint, 0)", (short) 
SqlFunctions.sround(SHORT_VALUE, 0))
-.addExpr("ROUND(c_tinyint, 0)", (byte) SqlFunctions.sround(BYTE_VALUE, 
0))
-.addExpr("ROUND(c_double, 0)", SqlFunctions.sround(DOUBLE_VALUE, 0))
-.addExpr("ROUND(c_float, 0)", (float) SqlFunctions.sround(FLOAT_VALUE, 
0))
-.addExpr("ROUND(c_decimal, 0)",
-new BigDecimal(SqlFunctions.sround(DECIMAL_VALUE.doubleValue(), 
0)))
-;
-
-checker.buildRunAndCheck();
-  }
-
-  @Test
-  public void testLn() throws Exception{
-ExpressionChecker checker = new ExpressionChecker()
-.addExpr("LN(c_integer)", Math.log(INTEGER_VALUE))
-.addExpr("LN(c_bigint)", Math.log(LONG_VALUE))
-.addExpr("LN(c_smallint)", Math.log(SHORT_VALUE))
-.addExpr("LN(c_tinyint)", Math.log(BYTE_VALUE))
-.addExpr("LN(c_double)", Math.log(DOUBLE_VALUE))
-.addExpr("LN(c_float)", Math.log(FLOAT_VALUE))
-.addExpr("LN(c_decimal)", Math.log(DECIMAL_VALUE.doubleValue()))
-;
-
-checker.buildRunAndCheck();
-  }
-
-  @Test
-  public void testLog10() throws Exception{
-ExpressionChecker checker = new ExpressionChecker()
-.addExpr("LOG10(c_integer)", 

[12/59] beam git commit: move dsls/sql to sdks/java/extensions/sql

[takidau]

http://git-wip-us.apache.org/repos/asf/beam/blob/ba493f85/dsls/sql/src/test/java/org/apache/beam/dsls/sql/interpreter/operator/string/BeamSqlUpperExpressionTest.java
--
diff --git 
a/dsls/sql/src/test/java/org/apache/beam/dsls/sql/interpreter/operator/string/BeamSqlUpperExpressionTest.java
 
b/dsls/sql/src/test/java/org/apache/beam/dsls/sql/interpreter/operator/string/BeamSqlUpperExpressionTest.java
deleted file mode 100644
index 1a734bc..000
--- 
a/dsls/sql/src/test/java/org/apache/beam/dsls/sql/interpreter/operator/string/BeamSqlUpperExpressionTest.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.beam.dsls.sql.interpreter.operator.string;
-
-import static org.junit.Assert.assertEquals;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.beam.dsls.sql.interpreter.BeamSqlFnExecutorTestBase;
-import org.apache.beam.dsls.sql.interpreter.operator.BeamSqlExpression;
-import org.apache.beam.dsls.sql.interpreter.operator.BeamSqlPrimitive;
-import org.apache.calcite.sql.type.SqlTypeName;
-import org.junit.Test;
-
-/**
- * Test of BeamSqlUpperExpression.
- */
-public class BeamSqlUpperExpressionTest extends BeamSqlFnExecutorTestBase {
-
-  @Test public void evaluate() throws Exception {
-List operands = new ArrayList<>();
-
-operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "hello"));
-assertEquals("HELLO",
-new BeamSqlUpperExpression(operands).evaluate(record).getValue());
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/beam/blob/ba493f85/dsls/sql/src/test/java/org/apache/beam/dsls/sql/mock/MockedBoundedTable.java
--
diff --git 
a/dsls/sql/src/test/java/org/apache/beam/dsls/sql/mock/MockedBoundedTable.java 
b/dsls/sql/src/test/java/org/apache/beam/dsls/sql/mock/MockedBoundedTable.java
deleted file mode 100644
index 6c1dcb2..000
--- 
a/dsls/sql/src/test/java/org/apache/beam/dsls/sql/mock/MockedBoundedTable.java
+++ /dev/null
@@ -1,134 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.beam.dsls.sql.mock;
-
-import static org.apache.beam.dsls.sql.TestUtils.buildBeamSqlRowType;
-import static org.apache.beam.dsls.sql.TestUtils.buildRows;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-import java.util.concurrent.ConcurrentLinkedQueue;
-import org.apache.beam.dsls.sql.schema.BeamIOType;
-import org.apache.beam.dsls.sql.schema.BeamSqlRow;
-import org.apache.beam.dsls.sql.schema.BeamSqlRowType;
-import org.apache.beam.sdk.Pipeline;
-import org.apache.beam.sdk.transforms.Create;
-import org.apache.beam.sdk.transforms.DoFn;
-import org.apache.beam.sdk.transforms.PTransform;
-import org.apache.beam.sdk.transforms.ParDo;
-import org.apache.beam.sdk.values.PBegin;
-import org.apache.beam.sdk.values.PCollection;
-import org.apache.beam.sdk.values.PDone;
-
-/**
- * Mocked table for bounded data sources.
- */
-public class MockedBoundedTable extends MockedTable {
-  /** rows written to this table. */
-  private static final ConcurrentLinkedQueue CONTENT = new 
ConcurrentLinkedQueue<>();
-  /** rows flow out from this table. */
-  private final List rows = new ArrayList<>();
-
-  public MockedBoundedTable(BeamSqlRowType beamSqlRowType) {
-super(beamSqlRowType);
-  }
-
-  /**
-   * Convenient way to 

[30/59] beam git commit: rename package org.apache.beam.dsls.sql to org.apache.beam.sdk.extensions.sql

[takidau]

http://git-wip-us.apache.org/repos/asf/beam/blob/c1b5482d/sdks/java/extensions/sql/src/test/java/org/apache/beam/dsls/sql/BeamSqlDslBase.java
--
diff --git 
a/sdks/java/extensions/sql/src/test/java/org/apache/beam/dsls/sql/BeamSqlDslBase.java
 
b/sdks/java/extensions/sql/src/test/java/org/apache/beam/dsls/sql/BeamSqlDslBase.java
deleted file mode 100644
index a5d92e7..000
--- 
a/sdks/java/extensions/sql/src/test/java/org/apache/beam/dsls/sql/BeamSqlDslBase.java
+++ /dev/null
@@ -1,170 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.beam.dsls.sql;
-
-import java.math.BigDecimal;
-import java.sql.Types;
-import java.text.DateFormat;
-import java.text.ParseException;
-import java.text.SimpleDateFormat;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-import org.apache.beam.dsls.sql.schema.BeamSqlRow;
-import org.apache.beam.dsls.sql.schema.BeamSqlRowCoder;
-import org.apache.beam.dsls.sql.schema.BeamSqlRowType;
-import org.apache.beam.sdk.testing.TestPipeline;
-import org.apache.beam.sdk.testing.TestStream;
-import org.apache.beam.sdk.transforms.Create;
-import org.apache.beam.sdk.values.PBegin;
-import org.apache.beam.sdk.values.PCollection;
-import org.joda.time.Instant;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Rule;
-import org.junit.rules.ExpectedException;
-
-/**
- * prepare input records to test {@link BeamSql}.
- *
- * Note that, any change in these records would impact tests in this 
package.
- *
- */
-public class BeamSqlDslBase {
-  public static final DateFormat FORMAT = new SimpleDateFormat("-MM-dd 
HH:mm:ss");
-
-  @Rule
-  public final TestPipeline pipeline = TestPipeline.create();
-  @Rule
-  public ExpectedException exceptions = ExpectedException.none();
-
-  public static BeamSqlRowType rowTypeInTableA;
-  public static List recordsInTableA;
-
-  //bounded PCollections
-  public PCollection boundedInput1;
-  public PCollection boundedInput2;
-
-  //unbounded PCollections
-  public PCollection unboundedInput1;
-  public PCollection unboundedInput2;
-
-  @BeforeClass
-  public static void prepareClass() throws ParseException {
-rowTypeInTableA = BeamSqlRowType.create(
-Arrays.asList("f_int", "f_long", "f_short", "f_byte", "f_float", 
"f_double", "f_string",
-"f_timestamp", "f_int2", "f_decimal"),
-Arrays.asList(Types.INTEGER, Types.BIGINT, Types.SMALLINT, 
Types.TINYINT, Types.FLOAT,
-Types.DOUBLE, Types.VARCHAR, Types.TIMESTAMP, Types.INTEGER, 
Types.DECIMAL));
-
-recordsInTableA = prepareInputRowsInTableA();
-  }
-
-  @Before
-  public void preparePCollections(){
-boundedInput1 = PBegin.in(pipeline).apply("boundedInput1",
-Create.of(recordsInTableA).withCoder(new 
BeamSqlRowCoder(rowTypeInTableA)));
-
-boundedInput2 = PBegin.in(pipeline).apply("boundedInput2",
-Create.of(recordsInTableA.get(0)).withCoder(new 
BeamSqlRowCoder(rowTypeInTableA)));
-
-unboundedInput1 = prepareUnboundedPCollection1();
-unboundedInput2 = prepareUnboundedPCollection2();
-  }
-
-  private PCollection prepareUnboundedPCollection1() {
-TestStream.Builder values = TestStream
-.create(new BeamSqlRowCoder(rowTypeInTableA));
-
-for (BeamSqlRow row : recordsInTableA) {
-  values = values.advanceWatermarkTo(new 
Instant(row.getDate("f_timestamp")));
-  values = values.addElements(row);
-}
-
-return PBegin.in(pipeline).apply("unboundedInput1", 
values.advanceWatermarkToInfinity());
-  }
-
-  private PCollection prepareUnboundedPCollection2() {
-TestStream.Builder values = TestStream
-.create(new BeamSqlRowCoder(rowTypeInTableA));
-
-BeamSqlRow row = recordsInTableA.get(0);
-values = values.advanceWatermarkTo(new 
Instant(row.getDate("f_timestamp")));
-values = values.addElements(row);
-
-return PBegin.in(pipeline).apply("unboundedInput2", 
values.advanceWatermarkToInfinity());
-  }
-
-  private static List prepareInputRowsInTableA() throws 
ParseException{
-List rows = new ArrayList<>();
-
-BeamSqlRow row1 = new BeamSqlRow(rowTypeInTableA);
-

[32/59] beam git commit: rename package org.apache.beam.dsls.sql to org.apache.beam.sdk.extensions.sql

[takidau]

http://git-wip-us.apache.org/repos/asf/beam/blob/c1b5482d/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/rel/package-info.java
--
diff --git 
a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/rel/package-info.java
 
b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/rel/package-info.java
new file mode 100644
index 000..fb0a8e2
--- /dev/null
+++ 
b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/rel/package-info.java
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * BeamSQL specified nodes, to replace {@link org.apache.calcite.rel.RelNode}.
+ *
+ */
+package org.apache.beam.sdk.extensions.sql.rel;

http://git-wip-us.apache.org/repos/asf/beam/blob/c1b5482d/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/rule/BeamAggregationRule.java
--
diff --git 
a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/rule/BeamAggregationRule.java
 
b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/rule/BeamAggregationRule.java
new file mode 100644
index 000..17e3f80
--- /dev/null
+++ 
b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/rule/BeamAggregationRule.java
@@ -0,0 +1,162 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.extensions.sql.rule;
+
+import com.google.common.collect.ImmutableList;
+import java.util.GregorianCalendar;
+import java.util.List;
+import org.apache.beam.sdk.extensions.sql.rel.BeamAggregationRel;
+import org.apache.beam.sdk.extensions.sql.rel.BeamLogicalConvention;
+import org.apache.beam.sdk.transforms.windowing.AfterProcessingTime;
+import org.apache.beam.sdk.transforms.windowing.AfterWatermark;
+import org.apache.beam.sdk.transforms.windowing.FixedWindows;
+import org.apache.beam.sdk.transforms.windowing.GlobalWindows;
+import org.apache.beam.sdk.transforms.windowing.Repeatedly;
+import org.apache.beam.sdk.transforms.windowing.Sessions;
+import org.apache.beam.sdk.transforms.windowing.SlidingWindows;
+import org.apache.beam.sdk.transforms.windowing.Trigger;
+import org.apache.beam.sdk.transforms.windowing.WindowFn;
+import org.apache.calcite.plan.RelOptRule;
+import org.apache.calcite.plan.RelOptRuleCall;
+import org.apache.calcite.plan.RelOptRuleOperand;
+import org.apache.calcite.rel.core.Aggregate;
+import org.apache.calcite.rel.core.Project;
+import org.apache.calcite.rel.core.RelFactories;
+import org.apache.calcite.rex.RexCall;
+import org.apache.calcite.rex.RexLiteral;
+import org.apache.calcite.rex.RexNode;
+import org.apache.calcite.sql.SqlOperator;
+import org.apache.calcite.tools.RelBuilderFactory;
+import org.apache.calcite.util.ImmutableBitSet;
+import org.joda.time.Duration;
+
+/**
+ * Rule to detect the window/trigger settings.
+ *
+ */
+public class BeamAggregationRule extends RelOptRule {
+  public static final BeamAggregationRule INSTANCE =
+  new BeamAggregationRule(Aggregate.class, Project.class, 
RelFactories.LOGICAL_BUILDER);
+
+  public BeamAggregationRule(
+  Class aggregateClass,
+  Class projectClass,
+  RelBuilderFactory relBuilderFactory) {
+super(
+operand(aggregateClass,
+operand(projectClass, any())),
+   

[36/59] beam git commit: rename package org.apache.beam.dsls.sql to org.apache.beam.sdk.extensions.sql

[takidau]

http://git-wip-us.apache.org/repos/asf/beam/blob/c1b5482d/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/BeamSql.java
--
diff --git 
a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/BeamSql.java
 
b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/BeamSql.java
new file mode 100644
index 000..d64ae41
--- /dev/null
+++ 
b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/BeamSql.java
@@ -0,0 +1,244 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.extensions.sql;
+
+import com.google.auto.value.AutoValue;
+import org.apache.beam.sdk.annotations.Experimental;
+import org.apache.beam.sdk.extensions.sql.rel.BeamRelNode;
+import org.apache.beam.sdk.extensions.sql.schema.BeamPCollectionTable;
+import org.apache.beam.sdk.extensions.sql.schema.BeamSqlRow;
+import org.apache.beam.sdk.extensions.sql.schema.BeamSqlRowCoder;
+import org.apache.beam.sdk.extensions.sql.schema.BeamSqlUdaf;
+import org.apache.beam.sdk.extensions.sql.schema.BeamSqlUdf;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PCollectionTuple;
+import org.apache.beam.sdk.values.TupleTag;
+import org.apache.calcite.sql.SqlNode;
+import org.apache.calcite.sql.SqlSelect;
+import org.apache.calcite.sql.parser.SqlParseException;
+import org.apache.calcite.tools.RelConversionException;
+import org.apache.calcite.tools.ValidationException;
+
+/**
+ * {@code BeamSql} is the DSL interface of BeamSQL. It translates a SQL query 
as a
+ * {@link PTransform}, so developers can use standard SQL queries in a Beam 
pipeline.
+ *
+ * Beam SQL DSL usage:
+ * A typical pipeline with Beam SQL DSL is:
+ * 
+ *{@code
+PipelineOptions options = PipelineOptionsFactory.create();
+Pipeline p = Pipeline.create(options);
+
+//create table from TextIO;
+PCollection inputTableA = 
p.apply(TextIO.read().from("/my/input/patha"))
+.apply(...);
+PCollection inputTableB = 
p.apply(TextIO.read().from("/my/input/pathb"))
+.apply(...);
+
+//run a simple query, and register the output as a table in BeamSql;
+String sql1 = "select MY_FUNC(c1), c2 from PCOLLECTION";
+PCollection outputTableA = inputTableA.apply(
+BeamSql.simpleQuery(sql1)
+.withUdf("MY_FUNC", MY_FUNC.class, "FUNC"));
+
+//run a JOIN with one table from TextIO, and one table from another query
+PCollection outputTableB = PCollectionTuple.of(
+new TupleTag("TABLE_O_A"), outputTableA)
+.and(new TupleTag("TABLE_B"), inputTableB)
+.apply(BeamSql.query("select * from TABLE_O_A JOIN TABLE_B where ..."));
+
+//output the final result with TextIO
+outputTableB.apply(...).apply(TextIO.write().to("/my/output/path"));
+
+p.run().waitUntilFinish();
+ * }
+ * 
+ */
+@Experimental
+public class BeamSql {
+  /**
+   * Transforms a SQL query into a {@link PTransform} representing an 
equivalent execution plan.
+   *
+   * The returned {@link PTransform} can be applied to a {@link 
PCollectionTuple} representing
+   * all the input tables and results in a {@code PCollection} 
representing the output
+   * table. The {@link PCollectionTuple} contains the mapping from {@code 
table names} to
+   * {@code PCollection}, each representing an input table.
+   *
+   * It is an error to apply a {@link PCollectionTuple} missing any {@code 
table names}
+   * referenced within the query.
+   */
+  public static QueryTransform query(String sqlQuery) {
+return QueryTransform.builder()
+.setSqlEnv(new BeamSqlEnv())
+.setSqlQuery(sqlQuery)
+.build();
+  }
+
+  /**
+   * Transforms a SQL query into a {@link PTransform} representing an 
equivalent execution plan.
+   *
+   * This is a simplified form of {@link #query(String)} where the query 
must reference
+   * a single input table.
+   *
+   * Make sure to query it from a static table name PCOLLECTION.
+   */
+  public static SimpleQueryTransform simpleQuery(String sqlQuery) throws 
Exception {
+return SimpleQueryTransform.builder()
+.setSqlEnv(new BeamSqlEnv())
+

[55/59] beam git commit: move all implementation classes/packages into impl package

[takidau]

http://git-wip-us.apache.org/repos/asf/beam/blob/febd044a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/planner/BeamQueryPlanner.java
--
diff --git 
a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/planner/BeamQueryPlanner.java
 
b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/planner/BeamQueryPlanner.java
new file mode 100644
index 000..dd01a87
--- /dev/null
+++ 
b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/planner/BeamQueryPlanner.java
@@ -0,0 +1,167 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.extensions.sql.impl.planner;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import org.apache.beam.sdk.Pipeline;
+import org.apache.beam.sdk.extensions.sql.BeamSqlEnv;
+import org.apache.beam.sdk.extensions.sql.impl.rel.BeamLogicalConvention;
+import org.apache.beam.sdk.extensions.sql.impl.rel.BeamRelNode;
+import org.apache.beam.sdk.extensions.sql.schema.BaseBeamTable;
+import org.apache.beam.sdk.extensions.sql.schema.BeamSqlRow;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PCollectionTuple;
+import org.apache.calcite.adapter.java.JavaTypeFactory;
+import org.apache.calcite.config.Lex;
+import org.apache.calcite.jdbc.CalciteSchema;
+import org.apache.calcite.jdbc.JavaTypeFactoryImpl;
+import org.apache.calcite.plan.Contexts;
+import org.apache.calcite.plan.ConventionTraitDef;
+import org.apache.calcite.plan.RelOptUtil;
+import org.apache.calcite.plan.RelTraitDef;
+import org.apache.calcite.plan.RelTraitSet;
+import org.apache.calcite.prepare.CalciteCatalogReader;
+import org.apache.calcite.rel.RelCollationTraitDef;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.rel.type.RelDataTypeSystem;
+import org.apache.calcite.schema.SchemaPlus;
+import org.apache.calcite.sql.SqlNode;
+import org.apache.calcite.sql.SqlOperatorTable;
+import org.apache.calcite.sql.fun.SqlStdOperatorTable;
+import org.apache.calcite.sql.parser.SqlParseException;
+import org.apache.calcite.sql.parser.SqlParser;
+import org.apache.calcite.sql.util.ChainedSqlOperatorTable;
+import org.apache.calcite.tools.FrameworkConfig;
+import org.apache.calcite.tools.Frameworks;
+import org.apache.calcite.tools.Planner;
+import org.apache.calcite.tools.RelConversionException;
+import org.apache.calcite.tools.ValidationException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * The core component to handle through a SQL statement, from explain 
execution plan,
+ * to generate a Beam pipeline.
+ *
+ */
+public class BeamQueryPlanner {
+  private static final Logger LOG = 
LoggerFactory.getLogger(BeamQueryPlanner.class);
+
+  protected final Planner planner;
+  private Map sourceTables = new HashMap<>();
+
+  public static final JavaTypeFactory TYPE_FACTORY = new JavaTypeFactoryImpl(
+  RelDataTypeSystem.DEFAULT);
+
+  public BeamQueryPlanner(SchemaPlus schema) {
+final List traitDefs = new ArrayList<>();
+traitDefs.add(ConventionTraitDef.INSTANCE);
+traitDefs.add(RelCollationTraitDef.INSTANCE);
+
+List sqlOperatorTables = new ArrayList<>();
+sqlOperatorTables.add(SqlStdOperatorTable.instance());
+sqlOperatorTables.add(new CalciteCatalogReader(CalciteSchema.from(schema), 
false,
+Collections.emptyList(), TYPE_FACTORY));
+
+FrameworkConfig config = Frameworks.newConfigBuilder()
+
.parserConfig(SqlParser.configBuilder().setLex(Lex.MYSQL).build()).defaultSchema(schema)
+
.traitDefs(traitDefs).context(Contexts.EMPTY_CONTEXT).ruleSets(BeamRuleSets.getRuleSets())
+
.costFactory(null).typeSystem(BeamRelDataTypeSystem.BEAM_REL_DATATYPE_SYSTEM)
+.operatorTable(new ChainedSqlOperatorTable(sqlOperatorTables))
+.build();
+this.planner = Frameworks.getPlanner(config);
+
+for (String t : schema.getTableNames()) {
+  sourceTables.put(t, (BaseBeamTable) schema.getTable(t));
+}
+  }
+
+  /**
+   * 

[20/59] beam git commit: move dsls/sql to sdks/java/extensions/sql

[takidau]

http://git-wip-us.apache.org/repos/asf/beam/blob/ba493f85/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/BeamSqlCastExpression.java
--
diff --git 
a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/BeamSqlCastExpression.java
 
b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/BeamSqlCastExpression.java
deleted file mode 100644
index 524d1df..000
--- 
a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/BeamSqlCastExpression.java
+++ /dev/null
@@ -1,132 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.beam.dsls.sql.interpreter.operator;
-
-import java.sql.Date;
-import java.sql.Timestamp;
-import java.util.List;
-
-import org.apache.beam.dsls.sql.schema.BeamSqlRow;
-import org.apache.calcite.runtime.SqlFunctions;
-import org.apache.calcite.sql.type.SqlTypeName;
-import org.joda.time.format.DateTimeFormat;
-import org.joda.time.format.DateTimeFormatter;
-import org.joda.time.format.DateTimeFormatterBuilder;
-import org.joda.time.format.DateTimeParser;
-
-/**
- * Base class to support 'CAST' operations for all {@link SqlTypeName}.
- */
-public class BeamSqlCastExpression extends BeamSqlExpression {
-
-  private static final int index = 0;
-  private static final String outputTimestampFormat = "-MM-dd HH:mm:ss";
-  private static final String outputDateFormat = "-MM-dd";
-  /**
-   * Date and Timestamp formats used to parse
-   * {@link SqlTypeName#DATE}, {@link SqlTypeName#TIMESTAMP}.
-   */
-  private static final DateTimeFormatter dateTimeFormatter = new 
DateTimeFormatterBuilder()
-  .append(null/*printer*/, new DateTimeParser[] {
-  // date formats
-  DateTimeFormat.forPattern("yy-MM-dd").getParser(),
-  DateTimeFormat.forPattern("yy/MM/dd").getParser(),
-  DateTimeFormat.forPattern("yy.MM.dd").getParser(),
-  DateTimeFormat.forPattern("yyMMdd").getParser(),
-  DateTimeFormat.forPattern("MMdd").getParser(),
-  DateTimeFormat.forPattern("-MM-dd").getParser(),
-  DateTimeFormat.forPattern("/MM/dd").getParser(),
-  DateTimeFormat.forPattern(".MM.dd").getParser(),
-  // datetime formats
-  DateTimeFormat.forPattern("-MM-dd HH:mm:ss").getParser(),
-  DateTimeFormat.forPattern("-MM-dd HH:mm:ssz").getParser(),
-  DateTimeFormat.forPattern("-MM-dd HH:mm:ss z").getParser(),
-  DateTimeFormat.forPattern("-MM-dd 
HH:mm:ss.S").getParser(),
-  DateTimeFormat.forPattern("-MM-dd 
HH:mm:ss.Sz").getParser(),
-  DateTimeFormat.forPattern("-MM-dd HH:mm:ss.S 
z").getParser() }).toFormatter()
-  .withPivotYear(2020);
-
-  public BeamSqlCastExpression(List operands, SqlTypeName 
castType) {
-super(operands, castType);
-  }
-
-  @Override
-  public boolean accept() {
-return numberOfOperands() == 1;
-  }
-
-  @Override
-  public BeamSqlPrimitive evaluate(BeamSqlRow inputRow) {
-SqlTypeName castOutputType = getOutputType();
-switch (castOutputType) {
-  case INTEGER:
-return BeamSqlPrimitive
-.of(SqlTypeName.INTEGER, 
SqlFunctions.toInt(opValueEvaluated(index, inputRow)));
-  case DOUBLE:
-return BeamSqlPrimitive
-.of(SqlTypeName.DOUBLE, 
SqlFunctions.toDouble(opValueEvaluated(index, inputRow)));
-  case SMALLINT:
-return BeamSqlPrimitive
-.of(SqlTypeName.SMALLINT, 
SqlFunctions.toShort(opValueEvaluated(index, inputRow)));
-  case TINYINT:
-return BeamSqlPrimitive
-.of(SqlTypeName.TINYINT, 
SqlFunctions.toByte(opValueEvaluated(index, inputRow)));
-  case BIGINT:
-return BeamSqlPrimitive
-.of(SqlTypeName.BIGINT, 
SqlFunctions.toLong(opValueEvaluated(index, inputRow)));
-  case DECIMAL:
-return BeamSqlPrimitive.of(SqlTypeName.DECIMAL,
-SqlFunctions.toBigDecimal(opValueEvaluated(index, inputRow)));
-  case FLOAT:
-return BeamSqlPrimitive
-.of(SqlTypeName.FLOAT, 
SqlFunctions.toFloat(opValueEvaluated(index, 

[51/59] beam git commit: move all implementation classes/packages into impl package

[takidau]

http://git-wip-us.apache.org/repos/asf/beam/blob/febd044a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/interpreter/operator/math/BeamSqlPiExpression.java
--
diff --git 
a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/interpreter/operator/math/BeamSqlPiExpression.java
 
b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/interpreter/operator/math/BeamSqlPiExpression.java
deleted file mode 100644
index ed89c49..000
--- 
a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/interpreter/operator/math/BeamSqlPiExpression.java
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.beam.sdk.extensions.sql.interpreter.operator.math;
-
-import 
org.apache.beam.sdk.extensions.sql.interpreter.operator.BeamSqlExpression;
-import 
org.apache.beam.sdk.extensions.sql.interpreter.operator.BeamSqlPrimitive;
-import org.apache.beam.sdk.extensions.sql.schema.BeamSqlRow;
-import org.apache.calcite.sql.type.SqlTypeName;
-
-/**
- * Base class for the PI function.
- */
-public class BeamSqlPiExpression extends BeamSqlExpression {
-
-  public BeamSqlPiExpression() {
-this.outputType = SqlTypeName.DOUBLE;
-  }
-
-  @Override public boolean accept() {
-return true;
-  }
-
-  @Override public BeamSqlPrimitive evaluate(BeamSqlRow inputRow) {
-return BeamSqlPrimitive.of(SqlTypeName.DOUBLE, Math.PI);
-  }
-}

http://git-wip-us.apache.org/repos/asf/beam/blob/febd044a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/interpreter/operator/math/BeamSqlPowerExpression.java
--
diff --git 
a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/interpreter/operator/math/BeamSqlPowerExpression.java
 
b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/interpreter/operator/math/BeamSqlPowerExpression.java
deleted file mode 100644
index e2bdd05..000
--- 
a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/interpreter/operator/math/BeamSqlPowerExpression.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.beam.sdk.extensions.sql.interpreter.operator.math;
-
-import java.util.List;
-import 
org.apache.beam.sdk.extensions.sql.interpreter.operator.BeamSqlExpression;
-import 
org.apache.beam.sdk.extensions.sql.interpreter.operator.BeamSqlPrimitive;
-import org.apache.calcite.runtime.SqlFunctions;
-import org.apache.calcite.sql.type.SqlTypeName;
-
-/**
- * {@code BeamSqlMathBinaryExpression} for 'POWER' function.
- */
-public class BeamSqlPowerExpression extends BeamSqlMathBinaryExpression {
-
-  public BeamSqlPowerExpression(List operands) {
-super(operands, SqlTypeName.DOUBLE);
-  }
-
-  @Override
-  public BeamSqlPrimitive calculate(BeamSqlPrimitive leftOp,
-  BeamSqlPrimitive rightOp) {
-return BeamSqlPrimitive.of(SqlTypeName.DOUBLE, SqlFunctions
-.power(SqlFunctions.toDouble(leftOp.getValue()),
-SqlFunctions.toDouble(rightOp.getValue(;
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/beam/blob/febd044a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/interpreter/operator/math/BeamSqlRadiansExpression.java

[31/59] beam git commit: rename package org.apache.beam.dsls.sql to org.apache.beam.sdk.extensions.sql

[takidau]

http://git-wip-us.apache.org/repos/asf/beam/blob/c1b5482d/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/schema/package-info.java
--
diff --git 
a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/schema/package-info.java
 
b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/schema/package-info.java
new file mode 100644
index 000..9655ebd
--- /dev/null
+++ 
b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/schema/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/**
+ * define table schema, to map with Beam IO components.
+ *
+ */
+package org.apache.beam.sdk.extensions.sql.schema;

http://git-wip-us.apache.org/repos/asf/beam/blob/c1b5482d/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/schema/text/BeamTextCSVTable.java
--
diff --git 
a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/schema/text/BeamTextCSVTable.java
 
b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/schema/text/BeamTextCSVTable.java
new file mode 100644
index 000..c44faab
--- /dev/null
+++ 
b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/schema/text/BeamTextCSVTable.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.extensions.sql.schema.text;
+
+import org.apache.beam.sdk.Pipeline;
+import org.apache.beam.sdk.extensions.sql.schema.BeamSqlRow;
+import org.apache.beam.sdk.extensions.sql.schema.BeamSqlRowType;
+import org.apache.beam.sdk.io.TextIO;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.values.PBegin;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PDone;
+import org.apache.commons.csv.CSVFormat;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * {@code BeamTextCSVTable} is a {@code BeamTextTable} which formatted in CSV.
+ *
+ * 
+ * {@link CSVFormat} itself has many dialects, check its javadoc for more info.
+ * 
+ */
+public class BeamTextCSVTable extends BeamTextTable {
+  private static final Logger LOG = LoggerFactory
+  .getLogger(BeamTextCSVTable.class);
+
+  private CSVFormat csvFormat;
+
+  /**
+   * CSV table with {@link CSVFormat#DEFAULT DEFAULT} format.
+   */
+  public BeamTextCSVTable(BeamSqlRowType beamSqlRowType, String filePattern)  {
+this(beamSqlRowType, filePattern, CSVFormat.DEFAULT);
+  }
+
+  public BeamTextCSVTable(BeamSqlRowType beamSqlRowType, String filePattern,
+  CSVFormat csvFormat) {
+super(beamSqlRowType, filePattern);
+this.csvFormat = csvFormat;
+  }
+
+  @Override
+  public PCollection buildIOReader(Pipeline pipeline) {
+return PBegin.in(pipeline).apply("decodeRecord", 
TextIO.read().from(filePattern))
+.apply("parseCSVLine",
+new BeamTextCSVTableIOReader(beamSqlRowType, filePattern, 
csvFormat));
+  }
+
+  @Override
+  public PTransform, PDone> buildIOWriter() {
+return new BeamTextCSVTableIOWriter(beamSqlRowType, filePattern, 
csvFormat);
+  }
+}

[28/59] beam git commit: rename package org.apache.beam.dsls.sql to org.apache.beam.sdk.extensions.sql

[takidau]

http://git-wip-us.apache.org/repos/asf/beam/blob/c1b5482d/sdks/java/extensions/sql/src/test/java/org/apache/beam/dsls/sql/interpreter/operator/date/BeamSqlCurrentTimeExpressionTest.java
--
diff --git 
a/sdks/java/extensions/sql/src/test/java/org/apache/beam/dsls/sql/interpreter/operator/date/BeamSqlCurrentTimeExpressionTest.java
 
b/sdks/java/extensions/sql/src/test/java/org/apache/beam/dsls/sql/interpreter/operator/date/BeamSqlCurrentTimeExpressionTest.java
deleted file mode 100644
index ddf0a22..000
--- 
a/sdks/java/extensions/sql/src/test/java/org/apache/beam/dsls/sql/interpreter/operator/date/BeamSqlCurrentTimeExpressionTest.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.beam.dsls.sql.interpreter.operator.date;
-
-import static org.junit.Assert.assertEquals;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.beam.dsls.sql.interpreter.operator.BeamSqlExpression;
-import org.apache.calcite.sql.type.SqlTypeName;
-import org.junit.Test;
-
-/**
- * Test for BeamSqlLocalTimeExpression.
- */
-public class BeamSqlCurrentTimeExpressionTest extends 
BeamSqlDateExpressionTestBase {
-  @Test
-  public void test() {
-List operands = new ArrayList<>();
-assertEquals(SqlTypeName.TIME,
-new 
BeamSqlCurrentTimeExpression(operands).evaluate(record).getOutputType());
-  }
-}

http://git-wip-us.apache.org/repos/asf/beam/blob/c1b5482d/sdks/java/extensions/sql/src/test/java/org/apache/beam/dsls/sql/interpreter/operator/date/BeamSqlCurrentTimestampExpressionTest.java
--
diff --git 
a/sdks/java/extensions/sql/src/test/java/org/apache/beam/dsls/sql/interpreter/operator/date/BeamSqlCurrentTimestampExpressionTest.java
 
b/sdks/java/extensions/sql/src/test/java/org/apache/beam/dsls/sql/interpreter/operator/date/BeamSqlCurrentTimestampExpressionTest.java
deleted file mode 100644
index a1554f1..000
--- 
a/sdks/java/extensions/sql/src/test/java/org/apache/beam/dsls/sql/interpreter/operator/date/BeamSqlCurrentTimestampExpressionTest.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.beam.dsls.sql.interpreter.operator.date;
-
-import static org.junit.Assert.assertEquals;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.beam.dsls.sql.interpreter.operator.BeamSqlExpression;
-import org.apache.calcite.sql.type.SqlTypeName;
-import org.junit.Test;
-
-/**
- * Test for BeamSqlLocalTimestampExpression.
- */
-public class BeamSqlCurrentTimestampExpressionTest extends 
BeamSqlDateExpressionTestBase {
-  @Test
-  public void test() {
-List operands = new ArrayList<>();
-assertEquals(SqlTypeName.TIMESTAMP,
-new 
BeamSqlCurrentTimestampExpression(operands).evaluate(record).getOutputType());
-  }
-}

http://git-wip-us.apache.org/repos/asf/beam/blob/c1b5482d/sdks/java/extensions/sql/src/test/java/org/apache/beam/dsls/sql/interpreter/operator/date/BeamSqlDateCeilExpressionTest.java
--
diff --git 
a/sdks/java/extensions/sql/src/test/java/org/apache/beam/dsls/sql/interpreter/operator/date/BeamSqlDateCeilExpressionTest.java
 

[33/59] beam git commit: rename package org.apache.beam.dsls.sql to org.apache.beam.sdk.extensions.sql

[takidau]

http://git-wip-us.apache.org/repos/asf/beam/blob/c1b5482d/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/planner/BeamQueryPlanner.java
--
diff --git 
a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/planner/BeamQueryPlanner.java
 
b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/planner/BeamQueryPlanner.java
new file mode 100644
index 000..ba6235f
--- /dev/null
+++ 
b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/planner/BeamQueryPlanner.java
@@ -0,0 +1,167 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.extensions.sql.planner;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import org.apache.beam.sdk.Pipeline;
+import org.apache.beam.sdk.extensions.sql.BeamSqlEnv;
+import org.apache.beam.sdk.extensions.sql.rel.BeamLogicalConvention;
+import org.apache.beam.sdk.extensions.sql.rel.BeamRelNode;
+import org.apache.beam.sdk.extensions.sql.schema.BaseBeamTable;
+import org.apache.beam.sdk.extensions.sql.schema.BeamSqlRow;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PCollectionTuple;
+import org.apache.calcite.adapter.java.JavaTypeFactory;
+import org.apache.calcite.config.Lex;
+import org.apache.calcite.jdbc.CalciteSchema;
+import org.apache.calcite.jdbc.JavaTypeFactoryImpl;
+import org.apache.calcite.plan.Contexts;
+import org.apache.calcite.plan.ConventionTraitDef;
+import org.apache.calcite.plan.RelOptUtil;
+import org.apache.calcite.plan.RelTraitDef;
+import org.apache.calcite.plan.RelTraitSet;
+import org.apache.calcite.prepare.CalciteCatalogReader;
+import org.apache.calcite.rel.RelCollationTraitDef;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.rel.type.RelDataTypeSystem;
+import org.apache.calcite.schema.SchemaPlus;
+import org.apache.calcite.sql.SqlNode;
+import org.apache.calcite.sql.SqlOperatorTable;
+import org.apache.calcite.sql.fun.SqlStdOperatorTable;
+import org.apache.calcite.sql.parser.SqlParseException;
+import org.apache.calcite.sql.parser.SqlParser;
+import org.apache.calcite.sql.util.ChainedSqlOperatorTable;
+import org.apache.calcite.tools.FrameworkConfig;
+import org.apache.calcite.tools.Frameworks;
+import org.apache.calcite.tools.Planner;
+import org.apache.calcite.tools.RelConversionException;
+import org.apache.calcite.tools.ValidationException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * The core component to handle through a SQL statement, from explain 
execution plan,
+ * to generate a Beam pipeline.
+ *
+ */
+public class BeamQueryPlanner {
+  private static final Logger LOG = 
LoggerFactory.getLogger(BeamQueryPlanner.class);
+
+  protected final Planner planner;
+  private Map sourceTables = new HashMap<>();
+
+  public static final JavaTypeFactory TYPE_FACTORY = new JavaTypeFactoryImpl(
+  RelDataTypeSystem.DEFAULT);
+
+  public BeamQueryPlanner(SchemaPlus schema) {
+final List traitDefs = new ArrayList<>();
+traitDefs.add(ConventionTraitDef.INSTANCE);
+traitDefs.add(RelCollationTraitDef.INSTANCE);
+
+List sqlOperatorTables = new ArrayList<>();
+sqlOperatorTables.add(SqlStdOperatorTable.instance());
+sqlOperatorTables.add(new CalciteCatalogReader(CalciteSchema.from(schema), 
false,
+Collections.emptyList(), TYPE_FACTORY));
+
+FrameworkConfig config = Frameworks.newConfigBuilder()
+
.parserConfig(SqlParser.configBuilder().setLex(Lex.MYSQL).build()).defaultSchema(schema)
+
.traitDefs(traitDefs).context(Contexts.EMPTY_CONTEXT).ruleSets(BeamRuleSets.getRuleSets())
+
.costFactory(null).typeSystem(BeamRelDataTypeSystem.BEAM_REL_DATATYPE_SYSTEM)
+.operatorTable(new ChainedSqlOperatorTable(sqlOperatorTables))
+.build();
+this.planner = Frameworks.getPlanner(config);
+
+for (String t : schema.getTableNames()) {
+  sourceTables.put(t, (BaseBeamTable) schema.getTable(t));
+}
+  }
+
+  /**
+   * Parse input SQL query, and return a 

[34/59] beam git commit: rename package org.apache.beam.dsls.sql to org.apache.beam.sdk.extensions.sql

[takidau]

http://git-wip-us.apache.org/repos/asf/beam/blob/c1b5482d/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/interpreter/operator/math/BeamSqlDegreesExpression.java
--
diff --git 
a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/interpreter/operator/math/BeamSqlDegreesExpression.java
 
b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/interpreter/operator/math/BeamSqlDegreesExpression.java
new file mode 100644
index 000..b41f090
--- /dev/null
+++ 
b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/interpreter/operator/math/BeamSqlDegreesExpression.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.extensions.sql.interpreter.operator.math;
+
+import java.util.List;
+import 
org.apache.beam.sdk.extensions.sql.interpreter.operator.BeamSqlExpression;
+import 
org.apache.beam.sdk.extensions.sql.interpreter.operator.BeamSqlPrimitive;
+import org.apache.calcite.runtime.SqlFunctions;
+import org.apache.calcite.sql.type.SqlTypeName;
+
+/**
+ * {@code BeamSqlMathUnaryExpression} for 'DEGREES' function.
+ */
+public class BeamSqlDegreesExpression extends BeamSqlMathUnaryExpression {
+
+  public BeamSqlDegreesExpression(List operands) {
+super(operands, SqlTypeName.DOUBLE);
+  }
+
+  @Override public BeamSqlPrimitive calculate(BeamSqlPrimitive op) {
+return BeamSqlPrimitive
+.of(SqlTypeName.DOUBLE, 
SqlFunctions.degrees(SqlFunctions.toDouble(op.getValue(;
+  }
+}

http://git-wip-us.apache.org/repos/asf/beam/blob/c1b5482d/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/interpreter/operator/math/BeamSqlExpExpression.java
--
diff --git 
a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/interpreter/operator/math/BeamSqlExpExpression.java
 
b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/interpreter/operator/math/BeamSqlExpExpression.java
new file mode 100644
index 000..f7a8f11
--- /dev/null
+++ 
b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/interpreter/operator/math/BeamSqlExpExpression.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.extensions.sql.interpreter.operator.math;
+
+import java.util.List;
+import 
org.apache.beam.sdk.extensions.sql.interpreter.operator.BeamSqlExpression;
+import 
org.apache.beam.sdk.extensions.sql.interpreter.operator.BeamSqlPrimitive;
+import org.apache.calcite.runtime.SqlFunctions;
+import org.apache.calcite.sql.type.SqlTypeName;
+
+/**
+ * {@code BeamSqlMathUnaryExpression} for 'EXP' function.
+ */
+public class BeamSqlExpExpression extends BeamSqlMathUnaryExpression {
+
+  public BeamSqlExpExpression(List operands) {
+super(operands, SqlTypeName.DOUBLE);
+  }
+
+  @Override public BeamSqlPrimitive calculate(BeamSqlPrimitive op) {
+return BeamSqlPrimitive
+.of(SqlTypeName.DOUBLE, 
SqlFunctions.exp(SqlFunctions.toDouble(op.getValue(;
+  }
+}

http://git-wip-us.apache.org/repos/asf/beam/blob/c1b5482d/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/interpreter/operator/math/BeamSqlFloorExpression.java

[21/59] beam git commit: move dsls/sql to sdks/java/extensions/sql

[takidau]

move dsls/sql to sdks/java/extensions/sql


Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/ba493f85
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/ba493f85
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/ba493f85

Branch: refs/heads/DSL_SQL
Commit: ba493f85a5a18665cd6ae4eb61e0a86fab1a6c07
Parents: d32aea9
Author: James Xu 
Authored: Sun Jul 30 23:39:37 2017 +0800
Committer: James Xu 
Committed: Sun Jul 30 23:39:37 2017 +0800

--
 dsls/pom.xml|  60 ---
 dsls/sql/pom.xml| 226 -
 .../java/org/apache/beam/dsls/sql/BeamSql.java  | 244 --
 .../org/apache/beam/dsls/sql/BeamSqlCli.java|  65 ---
 .../org/apache/beam/dsls/sql/BeamSqlEnv.java| 120 -
 .../beam/dsls/sql/example/BeamSqlExample.java   |  97 
 .../beam/dsls/sql/example/package-info.java |  23 -
 .../interpreter/BeamSqlExpressionExecutor.java  |  43 --
 .../dsls/sql/interpreter/BeamSqlFnExecutor.java | 442 --
 .../operator/BeamSqlCaseExpression.java |  64 ---
 .../operator/BeamSqlCastExpression.java | 132 --
 .../interpreter/operator/BeamSqlExpression.java |  78 
 .../operator/BeamSqlInputRefExpression.java |  43 --
 .../interpreter/operator/BeamSqlPrimitive.java  | 152 ---
 .../operator/BeamSqlReinterpretExpression.java  |  55 ---
 .../operator/BeamSqlUdfExpression.java  |  86 
 .../operator/BeamSqlWindowEndExpression.java|  42 --
 .../operator/BeamSqlWindowExpression.java   |  50 --
 .../operator/BeamSqlWindowStartExpression.java  |  43 --
 .../arithmetic/BeamSqlArithmeticExpression.java | 122 -
 .../arithmetic/BeamSqlDivideExpression.java |  37 --
 .../arithmetic/BeamSqlMinusExpression.java  |  36 --
 .../arithmetic/BeamSqlModExpression.java|  36 --
 .../arithmetic/BeamSqlMultiplyExpression.java   |  36 --
 .../arithmetic/BeamSqlPlusExpression.java   |  36 --
 .../operator/arithmetic/package-info.java   |  22 -
 .../comparison/BeamSqlCompareExpression.java|  96 
 .../comparison/BeamSqlEqualsExpression.java |  49 --
 .../BeamSqlGreaterThanExpression.java   |  49 --
 .../BeamSqlGreaterThanOrEqualsExpression.java   |  49 --
 .../comparison/BeamSqlIsNotNullExpression.java  |  53 ---
 .../comparison/BeamSqlIsNullExpression.java |  53 ---
 .../comparison/BeamSqlLessThanExpression.java   |  49 --
 .../BeamSqlLessThanOrEqualsExpression.java  |  49 --
 .../comparison/BeamSqlNotEqualsExpression.java  |  49 --
 .../operator/comparison/package-info.java   |  22 -
 .../date/BeamSqlCurrentDateExpression.java  |  45 --
 .../date/BeamSqlCurrentTimeExpression.java  |  53 ---
 .../date/BeamSqlCurrentTimestampExpression.java |  49 --
 .../date/BeamSqlDateCeilExpression.java |  55 ---
 .../date/BeamSqlDateFloorExpression.java|  55 ---
 .../operator/date/BeamSqlExtractExpression.java | 101 -
 .../interpreter/operator/date/package-info.java |  22 -
 .../operator/logical/BeamSqlAndExpression.java  |  48 --
 .../logical/BeamSqlLogicalExpression.java   |  47 --
 .../operator/logical/BeamSqlNotExpression.java  |  54 ---
 .../operator/logical/BeamSqlOrExpression.java   |  48 --
 .../operator/logical/package-info.java  |  22 -
 .../operator/math/BeamSqlAbsExpression.java |  74 ---
 .../operator/math/BeamSqlAcosExpression.java|  41 --
 .../operator/math/BeamSqlAsinExpression.java|  41 --
 .../operator/math/BeamSqlAtan2Expression.java   |  43 --
 .../operator/math/BeamSqlAtanExpression.java|  41 --
 .../operator/math/BeamSqlCeilExpression.java|  46 --
 .../operator/math/BeamSqlCosExpression.java |  41 --
 .../operator/math/BeamSqlCotExpression.java |  41 --
 .../operator/math/BeamSqlDegreesExpression.java |  41 --
 .../operator/math/BeamSqlExpExpression.java |  41 --
 .../operator/math/BeamSqlFloorExpression.java   |  46 --
 .../operator/math/BeamSqlLnExpression.java  |  41 --
 .../operator/math/BeamSqlLogExpression.java |  41 --
 .../math/BeamSqlMathBinaryExpression.java   |  64 ---
 .../math/BeamSqlMathUnaryExpression.java|  58 ---
 .../operator/math/BeamSqlPiExpression.java  |  42 --
 .../operator/math/BeamSqlPowerExpression.java   |  45 --
 .../operator/math/BeamSqlRadiansExpression.java |  41 --
 .../operator/math/BeamSqlRandExpression.java|  54 ---
 .../math/BeamSqlRandIntegerExpression.java  |  58 ---
 .../operator/math/BeamSqlRoundExpression.java   | 108 -
 .../operator/math/BeamSqlSignExpression.java|  72 ---
 .../operator/math/BeamSqlSinExpression.java |  41 --
 .../operator/math/BeamSqlTanExpression.java |  41 --
 .../math/BeamSqlTruncateExpression.java |  76 
 .../interpreter/operator/math/package-info.java |  22 -
 

[24/59] beam git commit: rename package org.apache.beam.dsls.sql to org.apache.beam.sdk.extensions.sql

[takidau]

http://git-wip-us.apache.org/repos/asf/beam/blob/c1b5482d/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/interpreter/operator/BeamSqlCompareExpressionTest.java
--
diff --git 
a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/interpreter/operator/BeamSqlCompareExpressionTest.java
 
b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/interpreter/operator/BeamSqlCompareExpressionTest.java
new file mode 100644
index 000..ed77ffb
--- /dev/null
+++ 
b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/interpreter/operator/BeamSqlCompareExpressionTest.java
@@ -0,0 +1,115 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.extensions.sql.interpreter.operator;
+
+import java.util.Arrays;
+import 
org.apache.beam.sdk.extensions.sql.interpreter.BeamSqlFnExecutorTestBase;
+import 
org.apache.beam.sdk.extensions.sql.interpreter.operator.comparison.BeamSqlCompareExpression;
+import 
org.apache.beam.sdk.extensions.sql.interpreter.operator.comparison.BeamSqlEqualsExpression;
+import 
org.apache.beam.sdk.extensions.sql.interpreter.operator.comparison.BeamSqlGreaterThanExpression;
+import 
org.apache.beam.sdk.extensions.sql.interpreter.operator.comparison.BeamSqlGreaterThanOrEqualsExpression;
+import 
org.apache.beam.sdk.extensions.sql.interpreter.operator.comparison.BeamSqlLessThanExpression;
+import 
org.apache.beam.sdk.extensions.sql.interpreter.operator.comparison.BeamSqlLessThanOrEqualsExpression;
+import 
org.apache.beam.sdk.extensions.sql.interpreter.operator.comparison.BeamSqlNotEqualsExpression;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * Test cases for the collections of {@link BeamSqlCompareExpression}.
+ */
+public class BeamSqlCompareExpressionTest extends BeamSqlFnExecutorTestBase {
+
+  @Test
+  public void testEqual() {
+BeamSqlEqualsExpression exp1 = new BeamSqlEqualsExpression(
+Arrays.asList(new BeamSqlInputRefExpression(SqlTypeName.BIGINT, 0),
+BeamSqlPrimitive.of(SqlTypeName.BIGINT, 100L)));
+Assert.assertEquals(false, exp1.evaluate(record).getValue());
+
+BeamSqlEqualsExpression exp2 = new BeamSqlEqualsExpression(
+Arrays.asList(new BeamSqlInputRefExpression(SqlTypeName.BIGINT, 0),
+BeamSqlPrimitive.of(SqlTypeName.BIGINT, 1234567L)));
+Assert.assertEquals(true, exp2.evaluate(record).getValue());
+  }
+
+  @Test
+  public void testLargerThan(){
+BeamSqlGreaterThanExpression exp1 = new BeamSqlGreaterThanExpression(
+Arrays.asList(new BeamSqlInputRefExpression(SqlTypeName.BIGINT, 0),
+BeamSqlPrimitive.of(SqlTypeName.BIGINT, 1234567L)));
+Assert.assertEquals(false, exp1.evaluate(record).getValue());
+
+BeamSqlGreaterThanExpression exp2 = new BeamSqlGreaterThanExpression(
+Arrays.asList(new BeamSqlInputRefExpression(SqlTypeName.BIGINT, 0),
+BeamSqlPrimitive.of(SqlTypeName.BIGINT, 1234566L)));
+Assert.assertEquals(true, exp2.evaluate(record).getValue());
+  }
+
+  @Test
+  public void testLargerThanEqual(){
+BeamSqlGreaterThanOrEqualsExpression exp1 = new 
BeamSqlGreaterThanOrEqualsExpression(
+Arrays.asList(new BeamSqlInputRefExpression(SqlTypeName.BIGINT, 0),
+BeamSqlPrimitive.of(SqlTypeName.BIGINT, 1234567L)));
+Assert.assertEquals(true, exp1.evaluate(record).getValue());
+
+BeamSqlGreaterThanOrEqualsExpression exp2 = new 
BeamSqlGreaterThanOrEqualsExpression(
+Arrays.asList(new BeamSqlInputRefExpression(SqlTypeName.BIGINT, 0),
+BeamSqlPrimitive.of(SqlTypeName.BIGINT, 1234568L)));
+Assert.assertEquals(false, exp2.evaluate(record).getValue());
+  }
+
+  @Test
+  public void testLessThan(){
+BeamSqlLessThanExpression exp1 = new BeamSqlLessThanExpression(
+Arrays.asList(new BeamSqlInputRefExpression(SqlTypeName.INTEGER, 1),
+BeamSqlPrimitive.of(SqlTypeName.INTEGER, 1)));
+Assert.assertEquals(true, exp1.evaluate(record).getValue());
+
+BeamSqlLessThanExpression exp2 = new 

[25/59] beam git commit: rename package org.apache.beam.dsls.sql to org.apache.beam.sdk.extensions.sql

[takidau]

http://git-wip-us.apache.org/repos/asf/beam/blob/c1b5482d/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/integrationtest/BeamSqlBuiltinFunctionsIntegrationTestBase.java
--
diff --git 
a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/integrationtest/BeamSqlBuiltinFunctionsIntegrationTestBase.java
 
b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/integrationtest/BeamSqlBuiltinFunctionsIntegrationTestBase.java
new file mode 100644
index 000..ffc6833
--- /dev/null
+++ 
b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/integrationtest/BeamSqlBuiltinFunctionsIntegrationTestBase.java
@@ -0,0 +1,169 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.extensions.sql.integrationtest;
+
+import com.google.common.base.Joiner;
+import java.math.BigDecimal;
+import java.sql.Types;
+import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.TimeZone;
+import org.apache.beam.sdk.extensions.sql.BeamSql;
+import org.apache.beam.sdk.extensions.sql.TestUtils;
+import org.apache.beam.sdk.extensions.sql.mock.MockedBoundedTable;
+import org.apache.beam.sdk.extensions.sql.schema.BeamSqlRow;
+import org.apache.beam.sdk.extensions.sql.schema.BeamSqlRowCoder;
+import org.apache.beam.sdk.extensions.sql.schema.BeamSqlRowType;
+import org.apache.beam.sdk.testing.PAssert;
+import org.apache.beam.sdk.testing.TestPipeline;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.calcite.util.Pair;
+import org.junit.Rule;
+
+/**
+ * Base class for all built-in functions integration tests.
+ */
+public class BeamSqlBuiltinFunctionsIntegrationTestBase {
+  private static final Map JAVA_CLASS_TO_SQL_TYPE = new 
HashMap<>();
+  static {
+JAVA_CLASS_TO_SQL_TYPE.put(Byte.class, Types.TINYINT);
+JAVA_CLASS_TO_SQL_TYPE.put(Short.class, Types.SMALLINT);
+JAVA_CLASS_TO_SQL_TYPE.put(Integer.class, Types.INTEGER);
+JAVA_CLASS_TO_SQL_TYPE.put(Long.class, Types.BIGINT);
+JAVA_CLASS_TO_SQL_TYPE.put(Float.class, Types.FLOAT);
+JAVA_CLASS_TO_SQL_TYPE.put(Double.class, Types.DOUBLE);
+JAVA_CLASS_TO_SQL_TYPE.put(BigDecimal.class, Types.DECIMAL);
+JAVA_CLASS_TO_SQL_TYPE.put(String.class, Types.VARCHAR);
+JAVA_CLASS_TO_SQL_TYPE.put(Date.class, Types.DATE);
+JAVA_CLASS_TO_SQL_TYPE.put(Boolean.class, Types.BOOLEAN);
+  }
+
+  @Rule
+  public final TestPipeline pipeline = TestPipeline.create();
+
+  protected PCollection getTestPCollection() {
+BeamSqlRowType type = BeamSqlRowType.create(
+Arrays.asList("ts", "c_tinyint", "c_smallint",
+"c_integer", "c_bigint", "c_float", "c_double", "c_decimal",
+"c_tinyint_max", "c_smallint_max", "c_integer_max", 
"c_bigint_max"),
+Arrays.asList(Types.DATE, Types.TINYINT, Types.SMALLINT,
+Types.INTEGER, Types.BIGINT, Types.FLOAT, Types.DOUBLE, 
Types.DECIMAL,
+Types.TINYINT, Types.SMALLINT, Types.INTEGER, Types.BIGINT)
+);
+try {
+  return MockedBoundedTable
+  .of(type)
+  .addRows(
+  parseDate("1986-02-15 11:35:26"),
+  (byte) 1,
+  (short) 1,
+  1,
+  1L,
+  1.0f,
+  1.0,
+  BigDecimal.ONE,
+  (byte) 127,
+  (short) 32767,
+  2147483647,
+  9223372036854775807L
+  )
+  .buildIOReader(pipeline)
+  .setCoder(new BeamSqlRowCoder(type));
+} catch (Exception e) {
+  throw new RuntimeException(e);
+}
+  }
+
+  protected static Date parseDate(String str) {
+try {
+  SimpleDateFormat sdf = new SimpleDateFormat("-MM-dd HH:mm:ss");
+  sdf.setTimeZone(TimeZone.getTimeZone("GMT"));
+  return sdf.parse(str);
+} catch (Exception e) {
+  throw new RuntimeException(e);
+}
+  }
+
+
+  /**
+   * Helper class to make write integration test for built-in functions 

[53/59] beam git commit: move all implementation classes/packages into impl package

[takidau]

http://git-wip-us.apache.org/repos/asf/beam/blob/febd044a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/utils/CalciteUtils.java
--
diff --git 
a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/utils/CalciteUtils.java
 
b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/utils/CalciteUtils.java
new file mode 100644
index 000..b80e045
--- /dev/null
+++ 
b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/utils/CalciteUtils.java
@@ -0,0 +1,113 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.extensions.sql.impl.utils;
+
+import java.sql.Types;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import org.apache.beam.sdk.extensions.sql.schema.BeamSqlRowType;
+import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.rel.type.RelDataTypeFactory;
+import org.apache.calcite.rel.type.RelDataTypeField;
+import org.apache.calcite.rel.type.RelProtoDataType;
+import org.apache.calcite.sql.type.SqlTypeName;
+
+/**
+ * Utility methods for Calcite related operations.
+ */
+public class CalciteUtils {
+  private static final Map JAVA_TO_CALCITE_MAPPING = new 
HashMap<>();
+  private static final Map CALCITE_TO_JAVA_MAPPING = new 
HashMap<>();
+  static {
+JAVA_TO_CALCITE_MAPPING.put(Types.TINYINT, SqlTypeName.TINYINT);
+JAVA_TO_CALCITE_MAPPING.put(Types.SMALLINT, SqlTypeName.SMALLINT);
+JAVA_TO_CALCITE_MAPPING.put(Types.INTEGER, SqlTypeName.INTEGER);
+JAVA_TO_CALCITE_MAPPING.put(Types.BIGINT, SqlTypeName.BIGINT);
+
+JAVA_TO_CALCITE_MAPPING.put(Types.FLOAT, SqlTypeName.FLOAT);
+JAVA_TO_CALCITE_MAPPING.put(Types.DOUBLE, SqlTypeName.DOUBLE);
+
+JAVA_TO_CALCITE_MAPPING.put(Types.DECIMAL, SqlTypeName.DECIMAL);
+
+JAVA_TO_CALCITE_MAPPING.put(Types.CHAR, SqlTypeName.CHAR);
+JAVA_TO_CALCITE_MAPPING.put(Types.VARCHAR, SqlTypeName.VARCHAR);
+
+JAVA_TO_CALCITE_MAPPING.put(Types.DATE, SqlTypeName.DATE);
+JAVA_TO_CALCITE_MAPPING.put(Types.TIME, SqlTypeName.TIME);
+JAVA_TO_CALCITE_MAPPING.put(Types.TIMESTAMP, SqlTypeName.TIMESTAMP);
+
+JAVA_TO_CALCITE_MAPPING.put(Types.BOOLEAN, SqlTypeName.BOOLEAN);
+
+for (Map.Entry pair : 
JAVA_TO_CALCITE_MAPPING.entrySet()) {
+  CALCITE_TO_JAVA_MAPPING.put(pair.getValue(), pair.getKey());
+}
+  }
+
+  /**
+   * Get the corresponding {@code SqlTypeName} for an integer sql type.
+   */
+  public static SqlTypeName toCalciteType(int type) {
+return JAVA_TO_CALCITE_MAPPING.get(type);
+  }
+
+  /**
+   * Get the integer sql type from Calcite {@code SqlTypeName}.
+   */
+  public static Integer toJavaType(SqlTypeName typeName) {
+return CALCITE_TO_JAVA_MAPPING.get(typeName);
+  }
+
+  /**
+   * Get the {@code SqlTypeName} for the specified column of a table.
+   */
+  public static SqlTypeName getFieldType(BeamSqlRowType schema, int index) {
+return toCalciteType(schema.getFieldsType().get(index));
+  }
+
+  /**
+   * Generate {@code BeamSqlRowType} from {@code RelDataType} which is used to 
create table.
+   */
+  public static BeamSqlRowType toBeamRowType(RelDataType tableInfo) {
+List fieldNames = new ArrayList<>();
+List fieldTypes = new ArrayList<>();
+for (RelDataTypeField f : tableInfo.getFieldList()) {
+  fieldNames.add(f.getName());
+  fieldTypes.add(toJavaType(f.getType().getSqlTypeName()));
+}
+return BeamSqlRowType.create(fieldNames, fieldTypes);
+  }
+
+  /**
+   * Create an instance of {@code RelDataType} so it can be used to create a 
table.
+   */
+  public static RelProtoDataType toCalciteRowType(final BeamSqlRowType that) {
+return new RelProtoDataType() {
+  @Override
+  public RelDataType apply(RelDataTypeFactory a) {
+RelDataTypeFactory.FieldInfoBuilder builder = a.builder();
+for (int idx = 0; idx < that.getFieldsName().size(); ++idx) {
+  builder.add(that.getFieldsName().get(idx), 
toCalciteType(that.getFieldsType().get(idx)));

[49/59] beam git commit: move all implementation classes/packages into impl package

[takidau]

http://git-wip-us.apache.org/repos/asf/beam/blob/febd044a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/rule/BeamMinusRule.java
--
diff --git 
a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/rule/BeamMinusRule.java
 
b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/rule/BeamMinusRule.java
deleted file mode 100644
index 363cf3b..000
--- 
a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/rule/BeamMinusRule.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.beam.sdk.extensions.sql.rule;
-
-import java.util.List;
-import org.apache.beam.sdk.extensions.sql.rel.BeamLogicalConvention;
-import org.apache.beam.sdk.extensions.sql.rel.BeamMinusRel;
-import org.apache.calcite.plan.Convention;
-import org.apache.calcite.rel.RelNode;
-import org.apache.calcite.rel.convert.ConverterRule;
-import org.apache.calcite.rel.core.Minus;
-import org.apache.calcite.rel.logical.LogicalMinus;
-
-/**
- * {@code ConverterRule} to replace {@code Minus} with {@code BeamMinusRel}.
- */
-public class BeamMinusRule extends ConverterRule {
-  public static final BeamMinusRule INSTANCE = new BeamMinusRule();
-  private BeamMinusRule() {
-super(LogicalMinus.class, Convention.NONE,
-BeamLogicalConvention.INSTANCE, "BeamMinusRule");
-  }
-
-  @Override public RelNode convert(RelNode rel) {
-Minus minus = (Minus) rel;
-final List inputs = minus.getInputs();
-return new BeamMinusRel(
-minus.getCluster(),
-minus.getTraitSet().replace(BeamLogicalConvention.INSTANCE),
-convertList(inputs, BeamLogicalConvention.INSTANCE),
-minus.all
-);
-  }
-}

http://git-wip-us.apache.org/repos/asf/beam/blob/febd044a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/rule/BeamProjectRule.java
--
diff --git 
a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/rule/BeamProjectRule.java
 
b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/rule/BeamProjectRule.java
deleted file mode 100644
index 4f2f8c9..000
--- 
a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/rule/BeamProjectRule.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * 
- * http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.beam.sdk.extensions.sql.rule;
-
-import org.apache.beam.sdk.extensions.sql.rel.BeamLogicalConvention;
-import org.apache.beam.sdk.extensions.sql.rel.BeamProjectRel;
-import org.apache.calcite.plan.Convention;
-import org.apache.calcite.rel.RelNode;
-import org.apache.calcite.rel.convert.ConverterRule;
-import org.apache.calcite.rel.core.Project;
-import org.apache.calcite.rel.logical.LogicalProject;
-
-/**
- * A {@code ConverterRule} to replace {@link Project} with
- * {@link BeamProjectRel}.
- *
- */
-public class BeamProjectRule extends ConverterRule {
-  public static final BeamProjectRule INSTANCE = new BeamProjectRule();
-
-  private BeamProjectRule() {
-super(LogicalProject.class, Convention.NONE, 
BeamLogicalConvention.INSTANCE, "BeamProjectRule");
-  }
-
-  @Override
-  public RelNode convert(RelNode rel) {
-final Project project = (Project) rel;
-final RelNode input = project.getInput();
-

[54/59] beam git commit: move all implementation classes/packages into impl package

[takidau]

http://git-wip-us.apache.org/repos/asf/beam/blob/febd044a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rel/package-info.java
--
diff --git 
a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rel/package-info.java
 
b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rel/package-info.java
new file mode 100644
index 000..76b335d
--- /dev/null
+++ 
b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rel/package-info.java
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * BeamSQL specified nodes, to replace {@link org.apache.calcite.rel.RelNode}.
+ *
+ */
+package org.apache.beam.sdk.extensions.sql.impl.rel;

http://git-wip-us.apache.org/repos/asf/beam/blob/febd044a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rule/BeamAggregationRule.java
--
diff --git 
a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rule/BeamAggregationRule.java
 
b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rule/BeamAggregationRule.java
new file mode 100644
index 000..cdf6712
--- /dev/null
+++ 
b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rule/BeamAggregationRule.java
@@ -0,0 +1,162 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.extensions.sql.impl.rule;
+
+import com.google.common.collect.ImmutableList;
+import java.util.GregorianCalendar;
+import java.util.List;
+import org.apache.beam.sdk.extensions.sql.impl.rel.BeamAggregationRel;
+import org.apache.beam.sdk.extensions.sql.impl.rel.BeamLogicalConvention;
+import org.apache.beam.sdk.transforms.windowing.AfterProcessingTime;
+import org.apache.beam.sdk.transforms.windowing.AfterWatermark;
+import org.apache.beam.sdk.transforms.windowing.FixedWindows;
+import org.apache.beam.sdk.transforms.windowing.GlobalWindows;
+import org.apache.beam.sdk.transforms.windowing.Repeatedly;
+import org.apache.beam.sdk.transforms.windowing.Sessions;
+import org.apache.beam.sdk.transforms.windowing.SlidingWindows;
+import org.apache.beam.sdk.transforms.windowing.Trigger;
+import org.apache.beam.sdk.transforms.windowing.WindowFn;
+import org.apache.calcite.plan.RelOptRule;
+import org.apache.calcite.plan.RelOptRuleCall;
+import org.apache.calcite.plan.RelOptRuleOperand;
+import org.apache.calcite.rel.core.Aggregate;
+import org.apache.calcite.rel.core.Project;
+import org.apache.calcite.rel.core.RelFactories;
+import org.apache.calcite.rex.RexCall;
+import org.apache.calcite.rex.RexLiteral;
+import org.apache.calcite.rex.RexNode;
+import org.apache.calcite.sql.SqlOperator;
+import org.apache.calcite.tools.RelBuilderFactory;
+import org.apache.calcite.util.ImmutableBitSet;
+import org.joda.time.Duration;
+
+/**
+ * Rule to detect the window/trigger settings.
+ *
+ */
+public class BeamAggregationRule extends RelOptRule {
+  public static final BeamAggregationRule INSTANCE =
+  new BeamAggregationRule(Aggregate.class, Project.class, 
RelFactories.LOGICAL_BUILDER);
+
+  public BeamAggregationRule(
+  Class aggregateClass,
+  Class projectClass,
+  RelBuilderFactory relBuilderFactory) {
+super(
+

[46/59] beam git commit: move all implementation classes/packages into impl package

[takidau]

http://git-wip-us.apache.org/repos/asf/beam/blob/febd044a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamJoinRelUnboundedVsBoundedTest.java
--
diff --git 
a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamJoinRelUnboundedVsBoundedTest.java
 
b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamJoinRelUnboundedVsBoundedTest.java
new file mode 100644
index 000..1dbd8b4
--- /dev/null
+++ 
b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamJoinRelUnboundedVsBoundedTest.java
@@ -0,0 +1,241 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.extensions.sql.impl.rel;
+
+import java.sql.Types;
+import java.util.Date;
+import org.apache.beam.sdk.extensions.sql.BeamSqlCli;
+import org.apache.beam.sdk.extensions.sql.BeamSqlEnv;
+import org.apache.beam.sdk.extensions.sql.TestUtils;
+import 
org.apache.beam.sdk.extensions.sql.impl.transform.BeamSqlOutputToConsoleFn;
+import org.apache.beam.sdk.extensions.sql.mock.MockedBoundedTable;
+import org.apache.beam.sdk.extensions.sql.mock.MockedUnboundedTable;
+import org.apache.beam.sdk.extensions.sql.schema.BeamSqlRow;
+import org.apache.beam.sdk.testing.PAssert;
+import org.apache.beam.sdk.testing.TestPipeline;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.values.PCollection;
+import org.joda.time.Duration;
+import org.junit.BeforeClass;
+import org.junit.Rule;
+import org.junit.Test;
+
+/**
+ * Unbounded + Unbounded Test for {@code BeamJoinRel}.
+ */
+public class BeamJoinRelUnboundedVsBoundedTest {
+  @Rule
+  public final TestPipeline pipeline = TestPipeline.create();
+  private static final BeamSqlEnv beamSqlEnv = new BeamSqlEnv();
+  public static final Date FIRST_DATE = new Date(1);
+  public static final Date SECOND_DATE = new Date(1 + 3600 * 1000);
+  public static final Date THIRD_DATE = new Date(1 + 3600 * 1000 + 3600 * 1000 
+ 1);
+  private static final Duration WINDOW_SIZE = Duration.standardHours(1);
+
+  @BeforeClass
+  public static void prepare() {
+beamSqlEnv.registerTable("ORDER_DETAILS", MockedUnboundedTable
+.of(
+Types.INTEGER, "order_id",
+Types.INTEGER, "site_id",
+Types.INTEGER, "price",
+Types.TIMESTAMP, "order_time"
+)
+.timestampColumnIndex(3)
+.addRows(
+Duration.ZERO,
+1, 1, 1, FIRST_DATE,
+1, 2, 2, FIRST_DATE
+)
+.addRows(
+WINDOW_SIZE.plus(Duration.standardSeconds(1)),
+2, 2, 3, SECOND_DATE,
+2, 3, 3, SECOND_DATE,
+// this late data is omitted
+1, 2, 3, FIRST_DATE
+)
+.addRows(
+WINDOW_SIZE.plus(WINDOW_SIZE).plus(Duration.standardSeconds(1)),
+3, 3, 3, THIRD_DATE,
+// this late data is omitted
+2, 2, 3, SECOND_DATE
+)
+);
+
+beamSqlEnv.registerTable("ORDER_DETAILS1", MockedBoundedTable
+.of(Types.INTEGER, "order_id",
+Types.VARCHAR, "buyer"
+).addRows(
+1, "james",
+2, "bond"
+));
+  }
+
+  @Test
+  public void testInnerJoin_unboundedTableOnTheLeftSide() throws Exception {
+String sql = "SELECT o1.order_id, o1.sum_site_id, o2.buyer FROM "
++ "(select order_id, sum(site_id) as sum_site_id FROM ORDER_DETAILS "
++ "  GROUP BY order_id, TUMBLE(order_time, INTERVAL '1' HOUR)) 
o1 "
++ " JOIN "
++ " ORDER_DETAILS1 o2 "
++ " on "
++ " o1.order_id=o2.order_id"
+;
+
+PCollection rows = BeamSqlCli.compilePipeline(sql, pipeline, 
beamSqlEnv);
+PAssert.that(rows.apply(ParDo.of(new TestUtils.BeamSqlRow2StringDoFn(
+.containsInAnyOrder(
+TestUtils.RowsBuilder.of(
+Types.INTEGER, "order_id",
+Types.INTEGER, "sum_site_id",
+Types.VARCHAR, "buyer"
+).addRows(
+1, 3, "james",
+2, 5, "bond"
+

[48/59] beam git commit: move all implementation classes/packages into impl package

[takidau]

http://git-wip-us.apache.org/repos/asf/beam/blob/febd044a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/BeamSqlFnExecutorTest.java
--
diff --git 
a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/BeamSqlFnExecutorTest.java
 
b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/BeamSqlFnExecutorTest.java
new file mode 100644
index 000..f350087
--- /dev/null
+++ 
b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/BeamSqlFnExecutorTest.java
@@ -0,0 +1,416 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.extensions.sql.impl.interpreter;
+
+import static org.junit.Assert.assertTrue;
+
+import java.math.BigDecimal;
+import java.util.Arrays;
+import java.util.Calendar;
+import java.util.Date;
+import java.util.TimeZone;
+import 
org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlCaseExpression;
+import 
org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlExpression;
+import 
org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlInputRefExpression;
+import 
org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlPrimitive;
+import 
org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.arithmetic.BeamSqlDivideExpression;
+import 
org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.arithmetic.BeamSqlMinusExpression;
+import 
org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.arithmetic.BeamSqlModExpression;
+import 
org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.arithmetic.BeamSqlMultiplyExpression;
+import 
org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.arithmetic.BeamSqlPlusExpression;
+import 
org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.comparison.BeamSqlEqualsExpression;
+import 
org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.comparison.BeamSqlLessThanOrEqualsExpression;
+import 
org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.date.BeamSqlCurrentDateExpression;
+import 
org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.date.BeamSqlCurrentTimeExpression;
+import 
org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.date.BeamSqlCurrentTimestampExpression;
+import 
org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.date.BeamSqlDateCeilExpression;
+import 
org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.date.BeamSqlDateFloorExpression;
+import 
org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.date.BeamSqlExtractExpression;
+import 
org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.logical.BeamSqlAndExpression;
+import 
org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.logical.BeamSqlNotExpression;
+import 
org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.logical.BeamSqlOrExpression;
+import 
org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.string.BeamSqlCharLengthExpression;
+import 
org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.string.BeamSqlConcatExpression;
+import 
org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.string.BeamSqlInitCapExpression;
+import 
org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.string.BeamSqlLowerExpression;
+import 
org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.string.BeamSqlOverlayExpression;
+import 
org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.string.BeamSqlPositionExpression;
+import 
org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.string.BeamSqlSubstringExpression;
+import 
org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.string.BeamSqlTrimExpression;
+import 
org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.string.BeamSqlUpperExpression;
+import org.apache.beam.sdk.extensions.sql.impl.planner.BeamQueryPlanner;
+import org.apache.beam.sdk.extensions.sql.impl.rel.BeamFilterRel;
+import org.apache.beam.sdk.extensions.sql.impl.rel.BeamProjectRel;
+import 

[42/59] beam git commit: rename package org.apache.beam.dsls.sql to org.apache.beam.sdk.extensions.sql

[takidau]

rename package org.apache.beam.dsls.sql to org.apache.beam.sdk.extensions.sql


Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/c1b5482d
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/c1b5482d
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/c1b5482d

Branch: refs/heads/DSL_SQL
Commit: c1b5482d3fb13a14926f0ffc23d0810b3105ed24
Parents: ba493f8
Author: James Xu 
Authored: Sun Jul 30 23:58:02 2017 +0800
Committer: James Xu 
Committed: Sun Jul 30 23:58:02 2017 +0800

--
 .../java/org/apache/beam/dsls/sql/BeamSql.java  | 244 --
 .../org/apache/beam/dsls/sql/BeamSqlCli.java|  65 ---
 .../org/apache/beam/dsls/sql/BeamSqlEnv.java| 120 -
 .../beam/dsls/sql/example/BeamSqlExample.java   |  97 
 .../beam/dsls/sql/example/package-info.java |  23 -
 .../interpreter/BeamSqlExpressionExecutor.java  |  43 --
 .../dsls/sql/interpreter/BeamSqlFnExecutor.java | 442 --
 .../operator/BeamSqlCaseExpression.java |  64 ---
 .../operator/BeamSqlCastExpression.java | 132 --
 .../interpreter/operator/BeamSqlExpression.java |  78 
 .../operator/BeamSqlInputRefExpression.java |  43 --
 .../interpreter/operator/BeamSqlPrimitive.java  | 152 ---
 .../operator/BeamSqlReinterpretExpression.java  |  55 ---
 .../operator/BeamSqlUdfExpression.java  |  86 
 .../operator/BeamSqlWindowEndExpression.java|  42 --
 .../operator/BeamSqlWindowExpression.java   |  50 --
 .../operator/BeamSqlWindowStartExpression.java  |  43 --
 .../arithmetic/BeamSqlArithmeticExpression.java | 122 -
 .../arithmetic/BeamSqlDivideExpression.java |  37 --
 .../arithmetic/BeamSqlMinusExpression.java  |  36 --
 .../arithmetic/BeamSqlModExpression.java|  36 --
 .../arithmetic/BeamSqlMultiplyExpression.java   |  36 --
 .../arithmetic/BeamSqlPlusExpression.java   |  36 --
 .../operator/arithmetic/package-info.java   |  22 -
 .../comparison/BeamSqlCompareExpression.java|  96 
 .../comparison/BeamSqlEqualsExpression.java |  49 --
 .../BeamSqlGreaterThanExpression.java   |  49 --
 .../BeamSqlGreaterThanOrEqualsExpression.java   |  49 --
 .../comparison/BeamSqlIsNotNullExpression.java  |  53 ---
 .../comparison/BeamSqlIsNullExpression.java |  53 ---
 .../comparison/BeamSqlLessThanExpression.java   |  49 --
 .../BeamSqlLessThanOrEqualsExpression.java  |  49 --
 .../comparison/BeamSqlNotEqualsExpression.java  |  49 --
 .../operator/comparison/package-info.java   |  22 -
 .../date/BeamSqlCurrentDateExpression.java  |  45 --
 .../date/BeamSqlCurrentTimeExpression.java  |  53 ---
 .../date/BeamSqlCurrentTimestampExpression.java |  49 --
 .../date/BeamSqlDateCeilExpression.java |  55 ---
 .../date/BeamSqlDateFloorExpression.java|  55 ---
 .../operator/date/BeamSqlExtractExpression.java | 101 -
 .../interpreter/operator/date/package-info.java |  22 -
 .../operator/logical/BeamSqlAndExpression.java  |  48 --
 .../logical/BeamSqlLogicalExpression.java   |  47 --
 .../operator/logical/BeamSqlNotExpression.java  |  54 ---
 .../operator/logical/BeamSqlOrExpression.java   |  48 --
 .../operator/logical/package-info.java  |  22 -
 .../operator/math/BeamSqlAbsExpression.java |  74 ---
 .../operator/math/BeamSqlAcosExpression.java|  41 --
 .../operator/math/BeamSqlAsinExpression.java|  41 --
 .../operator/math/BeamSqlAtan2Expression.java   |  43 --
 .../operator/math/BeamSqlAtanExpression.java|  41 --
 .../operator/math/BeamSqlCeilExpression.java|  46 --
 .../operator/math/BeamSqlCosExpression.java |  41 --
 .../operator/math/BeamSqlCotExpression.java |  41 --
 .../operator/math/BeamSqlDegreesExpression.java |  41 --
 .../operator/math/BeamSqlExpExpression.java |  41 --
 .../operator/math/BeamSqlFloorExpression.java   |  46 --
 .../operator/math/BeamSqlLnExpression.java  |  41 --
 .../operator/math/BeamSqlLogExpression.java |  41 --
 .../math/BeamSqlMathBinaryExpression.java   |  64 ---
 .../math/BeamSqlMathUnaryExpression.java|  58 ---
 .../operator/math/BeamSqlPiExpression.java  |  42 --
 .../operator/math/BeamSqlPowerExpression.java   |  45 --
 .../operator/math/BeamSqlRadiansExpression.java |  41 --
 .../operator/math/BeamSqlRandExpression.java|  54 ---
 .../math/BeamSqlRandIntegerExpression.java  |  58 ---
 .../operator/math/BeamSqlRoundExpression.java   | 108 -
 .../operator/math/BeamSqlSignExpression.java|  72 ---
 .../operator/math/BeamSqlSinExpression.java |  41 --
 .../operator/math/BeamSqlTanExpression.java |  41 --
 .../math/BeamSqlTruncateExpression.java |  76 
 .../interpreter/operator/math/package-info.java |  22 -
 .../sql/interpreter/operator/package-info.java  |  22 -
 

[40/59] beam git commit: rename package org.apache.beam.dsls.sql to org.apache.beam.sdk.extensions.sql

[takidau]

http://git-wip-us.apache.org/repos/asf/beam/blob/c1b5482d/sdks/java/extensions/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/logical/BeamSqlOrExpression.java
--
diff --git 
a/sdks/java/extensions/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/logical/BeamSqlOrExpression.java
 
b/sdks/java/extensions/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/logical/BeamSqlOrExpression.java
deleted file mode 100644
index 450638c..000
--- 
a/sdks/java/extensions/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/logical/BeamSqlOrExpression.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.beam.dsls.sql.interpreter.operator.logical;
-
-import java.util.List;
-
-import org.apache.beam.dsls.sql.interpreter.operator.BeamSqlExpression;
-import org.apache.beam.dsls.sql.interpreter.operator.BeamSqlPrimitive;
-import org.apache.beam.dsls.sql.schema.BeamSqlRow;
-import org.apache.calcite.sql.type.SqlTypeName;
-
-/**
- * {@code BeamSqlExpression} for 'OR' operation.
- */
-public class BeamSqlOrExpression extends BeamSqlLogicalExpression {
-  public BeamSqlOrExpression(List operands) {
-super(operands);
-  }
-
-  @Override
-  public BeamSqlPrimitive evaluate(BeamSqlRow inputRow) {
-boolean result = false;
-for (BeamSqlExpression exp : operands) {
-  BeamSqlPrimitive expOut = exp.evaluate(inputRow);
-result = result || expOut.getValue();
-if (result) {
-  break;
-}
-}
-return BeamSqlPrimitive.of(SqlTypeName.BOOLEAN, result);
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/beam/blob/c1b5482d/sdks/java/extensions/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/logical/package-info.java
--
diff --git 
a/sdks/java/extensions/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/logical/package-info.java
 
b/sdks/java/extensions/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/logical/package-info.java
deleted file mode 100644
index 7862045..000
--- 
a/sdks/java/extensions/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/logical/package-info.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * Logical operators.
- */
-package org.apache.beam.dsls.sql.interpreter.operator.logical;

http://git-wip-us.apache.org/repos/asf/beam/blob/c1b5482d/sdks/java/extensions/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/math/BeamSqlAbsExpression.java
--
diff --git 
a/sdks/java/extensions/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/math/BeamSqlAbsExpression.java
 
b/sdks/java/extensions/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/math/BeamSqlAbsExpression.java
deleted file mode 100644
index e563634..000
--- 
a/sdks/java/extensions/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/math/BeamSqlAbsExpression.java
+++ /dev/null
@@ -1,74 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses 

[07/59] beam git commit: move dsls/sql to sdks/java/extensions/sql

[takidau]

http://git-wip-us.apache.org/repos/asf/beam/blob/ba493f85/sdks/java/extensions/sql/src/main/java/org/apache/beam/dsls/sql/rel/BeamSortRel.java
--
diff --git 
a/sdks/java/extensions/sql/src/main/java/org/apache/beam/dsls/sql/rel/BeamSortRel.java
 
b/sdks/java/extensions/sql/src/main/java/org/apache/beam/dsls/sql/rel/BeamSortRel.java
new file mode 100644
index 000..ba344df
--- /dev/null
+++ 
b/sdks/java/extensions/sql/src/main/java/org/apache/beam/dsls/sql/rel/BeamSortRel.java
@@ -0,0 +1,247 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.dsls.sql.rel;
+
+import java.io.Serializable;
+import java.lang.reflect.Type;
+import java.math.BigDecimal;
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.List;
+import org.apache.beam.dsls.sql.BeamSqlEnv;
+import org.apache.beam.dsls.sql.schema.BeamSqlRow;
+import org.apache.beam.dsls.sql.schema.BeamSqlRowCoder;
+import org.apache.beam.dsls.sql.utils.CalciteUtils;
+import org.apache.beam.sdk.coders.ListCoder;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.Flatten;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.transforms.Top;
+import org.apache.beam.sdk.transforms.windowing.GlobalWindow;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PCollectionTuple;
+import org.apache.calcite.plan.RelOptCluster;
+import org.apache.calcite.plan.RelTraitSet;
+import org.apache.calcite.rel.RelCollation;
+import org.apache.calcite.rel.RelCollationImpl;
+import org.apache.calcite.rel.RelFieldCollation;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.rel.core.Sort;
+import org.apache.calcite.rex.RexInputRef;
+import org.apache.calcite.rex.RexLiteral;
+import org.apache.calcite.rex.RexNode;
+import org.apache.calcite.sql.type.SqlTypeName;
+
+/**
+ * {@code BeamRelNode} to replace a {@code Sort} node.
+ *
+ * Since Beam does not fully supported global sort we are using {@link Top} 
to implement
+ * the {@code Sort} algebra. The following types of ORDER BY are supported:
+
+ * {@code
+ * select * from t order by id desc limit 10;
+ * select * from t order by id desc limit 10, 5;
+ * }
+ *
+ * but Order BY without a limit is NOT supported:
+ *
+ * {@code
+ *   select * from t order by id desc
+ * }
+ *
+ * Constraints
+ * 
+ *   Due to the constraints of {@link Top}, the result of a `ORDER BY 
LIMIT`
+ *   must fit into the memory of a single machine.
+ *   Since `WINDOW`(HOP, TUMBLE, SESSION etc) is always associated with 
`GroupBy`,
+ *   it does not make much sense to use `ORDER BY` with `WINDOW`.
+ *   
+ * 
+ */
+public class BeamSortRel extends Sort implements BeamRelNode {
+  private List fieldIndices = new ArrayList<>();
+  private List orientation = new ArrayList<>();
+  private List nullsFirst = new ArrayList<>();
+
+  private int startIndex = 0;
+  private int count;
+
+  public BeamSortRel(
+  RelOptCluster cluster,
+  RelTraitSet traits,
+  RelNode child,
+  RelCollation collation,
+  RexNode offset,
+  RexNode fetch) {
+super(cluster, traits, child, collation, offset, fetch);
+
+List fieldExps = getChildExps();
+RelCollationImpl collationImpl = (RelCollationImpl) collation;
+List collations = collationImpl.getFieldCollations();
+for (int i = 0; i < fieldExps.size(); i++) {
+  RexNode fieldExp = fieldExps.get(i);
+  RexInputRef inputRef = (RexInputRef) fieldExp;
+  fieldIndices.add(inputRef.getIndex());
+  orientation.add(collations.get(i).getDirection() == 
RelFieldCollation.Direction.ASCENDING);
+
+  RelFieldCollation.NullDirection rawNullDirection = 
collations.get(i).nullDirection;
+  if (rawNullDirection == RelFieldCollation.NullDirection.UNSPECIFIED) {
+rawNullDirection = 
collations.get(i).getDirection().defaultNullDirection();
+  }
+  nullsFirst.add(rawNullDirection == 
RelFieldCollation.NullDirection.FIRST);
+}
+
+if (fetch == null) {
+  throw new UnsupportedOperationException("ORDER BY without a LIMIT is not 
supported!");
+}
+
+RexLiteral fetchLiteral = 

[16/59] beam git commit: move dsls/sql to sdks/java/extensions/sql

[takidau]

http://git-wip-us.apache.org/repos/asf/beam/blob/ba493f85/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/BeamSqlTable.java
--
diff --git 
a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/BeamSqlTable.java 
b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/BeamSqlTable.java
deleted file mode 100644
index d419473..000
--- a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/BeamSqlTable.java
+++ /dev/null
@@ -1,52 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.beam.dsls.sql.schema;
-
-import org.apache.beam.sdk.Pipeline;
-import org.apache.beam.sdk.transforms.PTransform;
-import org.apache.beam.sdk.values.PCollection;
-import org.apache.beam.sdk.values.PDone;
-
-/**
- * This interface defines a Beam Sql Table.
- */
-public interface BeamSqlTable {
-  /**
-   * In Beam SQL, there's no difference between a batch query and a streaming
-   * query. {@link BeamIOType} is used to validate the sources.
-   */
-  BeamIOType getSourceType();
-
-  /**
-   * create a {@code PCollection} from source.
-   *
-   */
-  PCollection buildIOReader(Pipeline pipeline);
-
-  /**
-   * create a {@code IO.write()} instance to write to target.
-   *
-   */
-   PTransform, PDone> buildIOWriter();
-
-  /**
-   * Get the schema info of the table.
-   */
-   BeamSqlRowType getRowType();
-}

http://git-wip-us.apache.org/repos/asf/beam/blob/ba493f85/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/BeamSqlUdaf.java
--
diff --git 
a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/BeamSqlUdaf.java 
b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/BeamSqlUdaf.java
deleted file mode 100644
index 9582ffa..000
--- a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/schema/BeamSqlUdaf.java
+++ /dev/null
@@ -1,72 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.beam.dsls.sql.schema;
-
-import java.io.Serializable;
-import java.lang.reflect.ParameterizedType;
-import org.apache.beam.sdk.coders.CannotProvideCoderException;
-import org.apache.beam.sdk.coders.Coder;
-import org.apache.beam.sdk.coders.CoderRegistry;
-import org.apache.beam.sdk.transforms.Combine.CombineFn;
-
-/**
- * abstract class of aggregation functions in Beam SQL.
- *
- * There're several constrains for a UDAF:
- * 1. A constructor with an empty argument list is required;
- * 2. The type of {@code InputT} and {@code OutputT} can only be 
Interger/Long/Short/Byte/Double
- * /Float/Date/BigDecimal, mapping as SQL type 
INTEGER/BIGINT/SMALLINT/TINYINE/DOUBLE/FLOAT
- * /TIMESTAMP/DECIMAL;
- * 3. Keep intermediate data in {@code AccumT}, and do not rely on elements in 
class;
- */
-public abstract class BeamSqlUdaf implements 
Serializable {
-  public BeamSqlUdaf(){}
-
-  /**
-   * create an initial aggregation object, equals to {@link 
CombineFn#createAccumulator()}.
-   */
-  public abstract AccumT init();
-
-  /**
-   * add an input value, equals to {@link CombineFn#addInput(Object, Object)}.
-   */
-  public abstract AccumT add(AccumT accumulator, InputT input);
-
-  /**
-   * merge aggregation objects from parallel tasks, equals to
-   *  {@link CombineFn#mergeAccumulators(Iterable)}.
-   */
-  public abstract AccumT merge(Iterable accumulators);
-
-  /**
-   * extract output value from 

[14/59] beam git commit: move dsls/sql to sdks/java/extensions/sql

[takidau]

http://git-wip-us.apache.org/repos/asf/beam/blob/ba493f85/dsls/sql/src/test/java/org/apache/beam/dsls/sql/integrationtest/BeamSqlConditionalFunctionsIntegrationTest.java
--
diff --git 
a/dsls/sql/src/test/java/org/apache/beam/dsls/sql/integrationtest/BeamSqlConditionalFunctionsIntegrationTest.java
 
b/dsls/sql/src/test/java/org/apache/beam/dsls/sql/integrationtest/BeamSqlConditionalFunctionsIntegrationTest.java
deleted file mode 100644
index 6233aeb..000
--- 
a/dsls/sql/src/test/java/org/apache/beam/dsls/sql/integrationtest/BeamSqlConditionalFunctionsIntegrationTest.java
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.beam.dsls.sql.integrationtest;
-
-import org.junit.Test;
-
-/**
- * Integration test for conditional functions.
- */
-public class BeamSqlConditionalFunctionsIntegrationTest
-extends BeamSqlBuiltinFunctionsIntegrationTestBase {
-@Test
-public void testConditionalFunctions() throws Exception {
-  ExpressionChecker checker = new ExpressionChecker()
-  .addExpr(
-  "CASE 1 WHEN 1 THEN 'hello' ELSE 'world' END",
-  "hello"
-  )
-  .addExpr(
-  "CASE 2 "
-  + "WHEN 1 THEN 'hello' "
-  + "WHEN 3 THEN 'bond' "
-  + "ELSE 'world' END",
-  "world"
-  )
-  .addExpr(
-  "CASE "
-  + "WHEN 1 = 1 THEN 'hello' "
-  + "ELSE 'world' END",
-  "hello"
-  )
-  .addExpr(
-  "CASE "
-  + "WHEN 1 > 1 THEN 'hello' "
-  + "ELSE 'world' END",
-  "world"
-  )
-  .addExpr("NULLIF(5, 4) ", 5)
-  .addExpr("COALESCE(1, 5) ", 1)
-  .addExpr("COALESCE(NULL, 5) ", 5)
-  ;
-
-  checker.buildRunAndCheck();
-}
-}

http://git-wip-us.apache.org/repos/asf/beam/blob/ba493f85/dsls/sql/src/test/java/org/apache/beam/dsls/sql/integrationtest/BeamSqlDateFunctionsIntegrationTest.java
--
diff --git 
a/dsls/sql/src/test/java/org/apache/beam/dsls/sql/integrationtest/BeamSqlDateFunctionsIntegrationTest.java
 
b/dsls/sql/src/test/java/org/apache/beam/dsls/sql/integrationtest/BeamSqlDateFunctionsIntegrationTest.java
deleted file mode 100644
index bd0d3ba..000
--- 
a/dsls/sql/src/test/java/org/apache/beam/dsls/sql/integrationtest/BeamSqlDateFunctionsIntegrationTest.java
+++ /dev/null
@@ -1,88 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.beam.dsls.sql.integrationtest;
-
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-
-import java.util.Date;
-import java.util.Iterator;
-import org.apache.beam.dsls.sql.BeamSql;
-import org.apache.beam.dsls.sql.schema.BeamSqlRow;
-import org.apache.beam.sdk.testing.PAssert;
-import org.apache.beam.sdk.transforms.SerializableFunction;
-import org.apache.beam.sdk.values.PCollection;
-import org.junit.Test;
-
-/**
- * Integration test for date functions.
- */
-public class BeamSqlDateFunctionsIntegrationTest
-extends BeamSqlBuiltinFunctionsIntegrationTestBase {
-  @Test public void testDateTimeFunctions() throws Exception {
-ExpressionChecker checker = new ExpressionChecker()
-

[56/59] beam git commit: move all implementation classes/packages into impl package

[takidau]

http://git-wip-us.apache.org/repos/asf/beam/blob/febd044a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/math/BeamSqlCotExpression.java
--
diff --git 
a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/math/BeamSqlCotExpression.java
 
b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/math/BeamSqlCotExpression.java
new file mode 100644
index 000..68d56b5
--- /dev/null
+++ 
b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/math/BeamSqlCotExpression.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.math;
+
+import java.util.List;
+import 
org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlExpression;
+import 
org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlPrimitive;
+import org.apache.calcite.runtime.SqlFunctions;
+import org.apache.calcite.sql.type.SqlTypeName;
+
+/**
+ * {@code BeamSqlMathUnaryExpression} for 'COT' function.
+ */
+public class BeamSqlCotExpression extends BeamSqlMathUnaryExpression {
+
+  public BeamSqlCotExpression(List operands) {
+super(operands, SqlTypeName.DOUBLE);
+  }
+
+  @Override public BeamSqlPrimitive calculate(BeamSqlPrimitive op) {
+return BeamSqlPrimitive
+.of(SqlTypeName.DOUBLE, 
SqlFunctions.cot(SqlFunctions.toDouble(op.getValue(;
+  }
+}

http://git-wip-us.apache.org/repos/asf/beam/blob/febd044a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/math/BeamSqlDegreesExpression.java
--
diff --git 
a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/math/BeamSqlDegreesExpression.java
 
b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/math/BeamSqlDegreesExpression.java
new file mode 100644
index 000..de4eac2
--- /dev/null
+++ 
b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/math/BeamSqlDegreesExpression.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.math;
+
+import java.util.List;
+import 
org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlExpression;
+import 
org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlPrimitive;
+import org.apache.calcite.runtime.SqlFunctions;
+import org.apache.calcite.sql.type.SqlTypeName;
+
+/**
+ * {@code BeamSqlMathUnaryExpression} for 'DEGREES' function.
+ */
+public class BeamSqlDegreesExpression extends BeamSqlMathUnaryExpression {
+
+  public BeamSqlDegreesExpression(List operands) {
+super(operands, SqlTypeName.DOUBLE);
+  }
+
+  @Override public BeamSqlPrimitive calculate(BeamSqlPrimitive op) {
+return BeamSqlPrimitive
+.of(SqlTypeName.DOUBLE, 
SqlFunctions.degrees(SqlFunctions.toDouble(op.getValue(;
+  }
+}

[59/59] beam git commit: [BEAM-2441] This closes #3666

[takidau]

[BEAM-2441] This closes #3666


Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/10962a34
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/10962a34
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/10962a34

Branch: refs/heads/DSL_SQL
Commit: 10962a34d3727f5ee2ee8df030847765e8eaab63
Parents: d32aea9 febd044
Author: Tyler Akidau 
Authored: Tue Aug 1 22:06:25 2017 -0700
Committer: Tyler Akidau 
Committed: Tue Aug 1 22:06:25 2017 -0700

--
 dsls/pom.xml|  60 ---
 dsls/sql/pom.xml| 226 -
 .../java/org/apache/beam/dsls/sql/BeamSql.java  | 244 --
 .../org/apache/beam/dsls/sql/BeamSqlCli.java|  65 ---
 .../org/apache/beam/dsls/sql/BeamSqlEnv.java| 120 -
 .../beam/dsls/sql/example/BeamSqlExample.java   |  97 
 .../beam/dsls/sql/example/package-info.java |  23 -
 .../interpreter/BeamSqlExpressionExecutor.java  |  43 --
 .../dsls/sql/interpreter/BeamSqlFnExecutor.java | 442 --
 .../operator/BeamSqlCaseExpression.java |  64 ---
 .../operator/BeamSqlCastExpression.java | 132 --
 .../interpreter/operator/BeamSqlExpression.java |  78 
 .../operator/BeamSqlInputRefExpression.java |  43 --
 .../interpreter/operator/BeamSqlPrimitive.java  | 152 ---
 .../operator/BeamSqlReinterpretExpression.java  |  55 ---
 .../operator/BeamSqlUdfExpression.java  |  86 
 .../operator/BeamSqlWindowEndExpression.java|  42 --
 .../operator/BeamSqlWindowExpression.java   |  50 --
 .../operator/BeamSqlWindowStartExpression.java  |  43 --
 .../arithmetic/BeamSqlArithmeticExpression.java | 122 -
 .../arithmetic/BeamSqlDivideExpression.java |  37 --
 .../arithmetic/BeamSqlMinusExpression.java  |  36 --
 .../arithmetic/BeamSqlModExpression.java|  36 --
 .../arithmetic/BeamSqlMultiplyExpression.java   |  36 --
 .../arithmetic/BeamSqlPlusExpression.java   |  36 --
 .../operator/arithmetic/package-info.java   |  22 -
 .../comparison/BeamSqlCompareExpression.java|  96 
 .../comparison/BeamSqlEqualsExpression.java |  49 --
 .../BeamSqlGreaterThanExpression.java   |  49 --
 .../BeamSqlGreaterThanOrEqualsExpression.java   |  49 --
 .../comparison/BeamSqlIsNotNullExpression.java  |  53 ---
 .../comparison/BeamSqlIsNullExpression.java |  53 ---
 .../comparison/BeamSqlLessThanExpression.java   |  49 --
 .../BeamSqlLessThanOrEqualsExpression.java  |  49 --
 .../comparison/BeamSqlNotEqualsExpression.java  |  49 --
 .../operator/comparison/package-info.java   |  22 -
 .../date/BeamSqlCurrentDateExpression.java  |  45 --
 .../date/BeamSqlCurrentTimeExpression.java  |  53 ---
 .../date/BeamSqlCurrentTimestampExpression.java |  49 --
 .../date/BeamSqlDateCeilExpression.java |  55 ---
 .../date/BeamSqlDateFloorExpression.java|  55 ---
 .../operator/date/BeamSqlExtractExpression.java | 101 -
 .../interpreter/operator/date/package-info.java |  22 -
 .../operator/logical/BeamSqlAndExpression.java  |  48 --
 .../logical/BeamSqlLogicalExpression.java   |  47 --
 .../operator/logical/BeamSqlNotExpression.java  |  54 ---
 .../operator/logical/BeamSqlOrExpression.java   |  48 --
 .../operator/logical/package-info.java  |  22 -
 .../operator/math/BeamSqlAbsExpression.java |  74 ---
 .../operator/math/BeamSqlAcosExpression.java|  41 --
 .../operator/math/BeamSqlAsinExpression.java|  41 --
 .../operator/math/BeamSqlAtan2Expression.java   |  43 --
 .../operator/math/BeamSqlAtanExpression.java|  41 --
 .../operator/math/BeamSqlCeilExpression.java|  46 --
 .../operator/math/BeamSqlCosExpression.java |  41 --
 .../operator/math/BeamSqlCotExpression.java |  41 --
 .../operator/math/BeamSqlDegreesExpression.java |  41 --
 .../operator/math/BeamSqlExpExpression.java |  41 --
 .../operator/math/BeamSqlFloorExpression.java   |  46 --
 .../operator/math/BeamSqlLnExpression.java  |  41 --
 .../operator/math/BeamSqlLogExpression.java |  41 --
 .../math/BeamSqlMathBinaryExpression.java   |  64 ---
 .../math/BeamSqlMathUnaryExpression.java|  58 ---
 .../operator/math/BeamSqlPiExpression.java  |  42 --
 .../operator/math/BeamSqlPowerExpression.java   |  45 --
 .../operator/math/BeamSqlRadiansExpression.java |  41 --
 .../operator/math/BeamSqlRandExpression.java|  54 ---
 .../math/BeamSqlRandIntegerExpression.java  |  58 ---
 .../operator/math/BeamSqlRoundExpression.java   | 108 -
 .../operator/math/BeamSqlSignExpression.java|  72 ---
 .../operator/math/BeamSqlSinExpression.java |  41 --
 .../operator/math/BeamSqlTanExpression.java |  41 --
 .../math/BeamSqlTruncateExpression.java |  76 
 .../interpreter/operator/math/package-info.java |  22 -
 

[47/59] beam git commit: move all implementation classes/packages into impl package

[takidau]

http://git-wip-us.apache.org/repos/asf/beam/blob/febd044a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/date/BeamSqlExtractExpressionTest.java
--
diff --git 
a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/date/BeamSqlExtractExpressionTest.java
 
b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/date/BeamSqlExtractExpressionTest.java
new file mode 100644
index 000..0ca7e3e
--- /dev/null
+++ 
b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/date/BeamSqlExtractExpressionTest.java
@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.date;
+
+import static org.junit.Assert.assertEquals;
+
+import java.util.ArrayList;
+import java.util.List;
+import 
org.apache.beam.sdk.extensions.sql.impl.interpreter.BeamSqlFnExecutorTestBase;
+import 
org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlExpression;
+import 
org.apache.beam.sdk.extensions.sql.impl.interpreter.operator.BeamSqlPrimitive;
+import org.apache.calcite.avatica.util.TimeUnitRange;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.junit.Test;
+
+/**
+ * Test for {@code BeamSqlExtractExpression}.
+ */
+public class BeamSqlExtractExpressionTest extends 
BeamSqlDateExpressionTestBase {
+  @Test public void evaluate() throws Exception {
+List operands = new ArrayList<>();
+long time = str2LongTime("2017-05-22 16:17:18");
+
+// YEAR
+operands.add(BeamSqlPrimitive.of(SqlTypeName.SYMBOL, TimeUnitRange.YEAR));
+operands.add(BeamSqlPrimitive.of(SqlTypeName.BIGINT,
+time));
+assertEquals(2017L,
+new BeamSqlExtractExpression(operands)
+.evaluate(BeamSqlFnExecutorTestBase.record).getValue());
+
+// MONTH
+operands.clear();
+operands.add(BeamSqlPrimitive.of(SqlTypeName.SYMBOL, TimeUnitRange.MONTH));
+operands.add(BeamSqlPrimitive.of(SqlTypeName.BIGINT,
+time));
+assertEquals(5L,
+new BeamSqlExtractExpression(operands)
+.evaluate(BeamSqlFnExecutorTestBase.record).getValue());
+
+// DAY
+operands.clear();
+operands.add(BeamSqlPrimitive.of(SqlTypeName.SYMBOL, TimeUnitRange.DAY));
+operands.add(BeamSqlPrimitive.of(SqlTypeName.BIGINT,
+time));
+assertEquals(22L,
+new BeamSqlExtractExpression(operands)
+.evaluate(BeamSqlFnExecutorTestBase.record).getValue());
+
+// DAY_OF_WEEK
+operands.clear();
+operands.add(BeamSqlPrimitive.of(SqlTypeName.SYMBOL, TimeUnitRange.DOW));
+operands.add(BeamSqlPrimitive.of(SqlTypeName.BIGINT,
+time));
+assertEquals(2L,
+new BeamSqlExtractExpression(operands)
+.evaluate(BeamSqlFnExecutorTestBase.record).getValue());
+
+// DAY_OF_YEAR
+operands.clear();
+operands.add(BeamSqlPrimitive.of(SqlTypeName.SYMBOL, TimeUnitRange.DOY));
+operands.add(BeamSqlPrimitive.of(SqlTypeName.BIGINT,
+time));
+assertEquals(142L,
+new BeamSqlExtractExpression(operands)
+.evaluate(BeamSqlFnExecutorTestBase.record).getValue());
+
+// WEEK
+operands.clear();
+operands.add(BeamSqlPrimitive.of(SqlTypeName.SYMBOL, TimeUnitRange.WEEK));
+operands.add(BeamSqlPrimitive.of(SqlTypeName.BIGINT,
+time));
+assertEquals(21L,
+new BeamSqlExtractExpression(operands)
+.evaluate(BeamSqlFnExecutorTestBase.record).getValue());
+
+// QUARTER
+operands.clear();
+operands.add(BeamSqlPrimitive.of(SqlTypeName.SYMBOL, 
TimeUnitRange.QUARTER));
+operands.add(BeamSqlPrimitive.of(SqlTypeName.BIGINT,
+time));
+assertEquals(2L,
+new BeamSqlExtractExpression(operands)
+.evaluate(BeamSqlFnExecutorTestBase.record).getValue());
+
+  }
+}

[22/59] beam git commit: rename package org.apache.beam.dsls.sql to org.apache.beam.sdk.extensions.sql

[takidau]

http://git-wip-us.apache.org/repos/asf/beam/blob/c1b5482d/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/rel/BeamValuesRelTest.java
--
diff --git 
a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/rel/BeamValuesRelTest.java
 
b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/rel/BeamValuesRelTest.java
new file mode 100644
index 000..ace1a3e
--- /dev/null
+++ 
b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/rel/BeamValuesRelTest.java
@@ -0,0 +1,105 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.extensions.sql.rel;
+
+import java.sql.Types;
+import org.apache.beam.sdk.extensions.sql.BeamSqlCli;
+import org.apache.beam.sdk.extensions.sql.BeamSqlEnv;
+import org.apache.beam.sdk.extensions.sql.TestUtils;
+import org.apache.beam.sdk.extensions.sql.mock.MockedBoundedTable;
+import org.apache.beam.sdk.extensions.sql.schema.BeamSqlRow;
+import org.apache.beam.sdk.testing.PAssert;
+import org.apache.beam.sdk.testing.TestPipeline;
+import org.apache.beam.sdk.values.PCollection;
+import org.junit.BeforeClass;
+import org.junit.Rule;
+import org.junit.Test;
+
+/**
+ * Test for {@code BeamValuesRel}.
+ */
+public class BeamValuesRelTest {
+  static BeamSqlEnv sqlEnv = new BeamSqlEnv();
+
+  @Rule
+  public final TestPipeline pipeline = TestPipeline.create();
+
+  @BeforeClass
+  public static void prepare() {
+sqlEnv.registerTable("string_table",
+MockedBoundedTable.of(
+Types.VARCHAR, "name",
+Types.VARCHAR, "description"
+)
+);
+sqlEnv.registerTable("int_table",
+MockedBoundedTable.of(
+Types.INTEGER, "c0",
+Types.INTEGER, "c1"
+)
+);
+  }
+
+  @Test
+  public void testValues() throws Exception {
+String sql = "insert into string_table(name, description) values "
++ "('hello', 'world'), ('james', 'bond')";
+PCollection rows = BeamSqlCli.compilePipeline(sql, pipeline, 
sqlEnv);
+PAssert.that(rows).containsInAnyOrder(
+TestUtils.RowsBuilder.of(
+Types.VARCHAR, "name",
+Types.VARCHAR, "description"
+).addRows(
+"hello", "world",
+"james", "bond"
+).getRows()
+);
+pipeline.run();
+  }
+
+  @Test
+  public void testValues_castInt() throws Exception {
+String sql = "insert into int_table (c0, c1) values(cast(1 as int), cast(2 
as int))";
+PCollection rows = BeamSqlCli.compilePipeline(sql, pipeline, 
sqlEnv);
+PAssert.that(rows).containsInAnyOrder(
+TestUtils.RowsBuilder.of(
+Types.INTEGER, "c0",
+Types.INTEGER, "c1"
+).addRows(
+1, 2
+).getRows()
+);
+pipeline.run();
+  }
+
+  @Test
+  public void testValues_onlySelect() throws Exception {
+String sql = "select 1, '1'";
+PCollection rows = BeamSqlCli.compilePipeline(sql, pipeline, 
sqlEnv);
+PAssert.that(rows).containsInAnyOrder(
+TestUtils.RowsBuilder.of(
+Types.INTEGER, "EXPR$0",
+Types.CHAR, "EXPR$1"
+).addRows(
+1, "1"
+).getRows()
+);
+pipeline.run();
+  }
+}

http://git-wip-us.apache.org/repos/asf/beam/blob/c1b5482d/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/rel/CheckSize.java
--
diff --git 
a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/rel/CheckSize.java
 
b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/rel/CheckSize.java
new file mode 100644
index 000..f369076
--- /dev/null
+++ 
b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/rel/CheckSize.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 

[15/59] beam git commit: move dsls/sql to sdks/java/extensions/sql

[takidau]

http://git-wip-us.apache.org/repos/asf/beam/blob/ba493f85/dsls/sql/src/main/java/org/apache/beam/dsls/sql/utils/package-info.java
--
diff --git 
a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/utils/package-info.java 
b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/utils/package-info.java
deleted file mode 100644
index b5c861a..000
--- a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/utils/package-info.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * Utility classes.
- */
-package org.apache.beam.dsls.sql.utils;

http://git-wip-us.apache.org/repos/asf/beam/blob/ba493f85/dsls/sql/src/main/resources/log4j.properties
--
diff --git a/dsls/sql/src/main/resources/log4j.properties 
b/dsls/sql/src/main/resources/log4j.properties
deleted file mode 100644
index 709484b..000
--- a/dsls/sql/src/main/resources/log4j.properties
+++ /dev/null
@@ -1,23 +0,0 @@
-
-#  Licensed to the Apache Software Foundation (ASF) under one
-#  or more contributor license agreements.  See the NOTICE file
-#  distributed with this work for additional information
-#  regarding copyright ownership.  The ASF licenses this file
-#  to you under the Apache License, Version 2.0 (the
-#  "License"); you may not use this file except in compliance
-#  with the License.  You may obtain a copy of the License at
-#
-#  http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-# limitations under the License.
-
-
-log4j.rootLogger=ERROR,console
-log4j.appender.console=org.apache.log4j.ConsoleAppender
-log4j.appender.console.target=System.err
-log4j.appender.console.layout=org.apache.log4j.PatternLayout
-log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p 
%c{2}: %m%n
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/beam/blob/ba493f85/dsls/sql/src/test/java/org/apache/beam/dsls/sql/BeamSqlApiSurfaceTest.java
--
diff --git 
a/dsls/sql/src/test/java/org/apache/beam/dsls/sql/BeamSqlApiSurfaceTest.java 
b/dsls/sql/src/test/java/org/apache/beam/dsls/sql/BeamSqlApiSurfaceTest.java
deleted file mode 100644
index 922931c..000
--- a/dsls/sql/src/test/java/org/apache/beam/dsls/sql/BeamSqlApiSurfaceTest.java
+++ /dev/null
@@ -1,59 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.beam.dsls.sql;
-
-import static org.apache.beam.sdk.util.ApiSurface.containsOnlyPackages;
-import static org.junit.Assert.assertThat;
-
-import com.google.common.collect.ImmutableSet;
-import java.util.Set;
-import org.apache.beam.sdk.util.ApiSurface;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.JUnit4;
-
-/**
- * Surface test for BeamSql api.
- */
-@RunWith(JUnit4.class)
-public class BeamSqlApiSurfaceTest {
-  @Test
-  public void testSdkApiSurface() throws 

[17/59] beam git commit: move dsls/sql to sdks/java/extensions/sql

[takidau]

http://git-wip-us.apache.org/repos/asf/beam/blob/ba493f85/dsls/sql/src/main/java/org/apache/beam/dsls/sql/rel/BeamRelNode.java
--
diff --git 
a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/rel/BeamRelNode.java 
b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/rel/BeamRelNode.java
deleted file mode 100644
index d4c98a3..000
--- a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/rel/BeamRelNode.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.beam.dsls.sql.rel;
-
-import org.apache.beam.dsls.sql.BeamSqlEnv;
-import org.apache.beam.dsls.sql.schema.BeamSqlRow;
-import org.apache.beam.sdk.values.PCollection;
-import org.apache.beam.sdk.values.PCollectionTuple;
-import org.apache.calcite.rel.RelNode;
-
-/**
- * A new method {@link #buildBeamPipeline(PCollectionTuple, BeamSqlEnv)} is 
added.
- */
-public interface BeamRelNode extends RelNode {
-
-  /**
-   * A {@link BeamRelNode} is a recursive structure, the
-   * {@code BeamQueryPlanner} visits it with a DFS(Depth-First-Search)
-   * algorithm.
-   */
-  PCollection buildBeamPipeline(PCollectionTuple 
inputPCollections, BeamSqlEnv sqlEnv)
-  throws Exception;
-}

http://git-wip-us.apache.org/repos/asf/beam/blob/ba493f85/dsls/sql/src/main/java/org/apache/beam/dsls/sql/rel/BeamSetOperatorRelBase.java
--
diff --git 
a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/rel/BeamSetOperatorRelBase.java
 
b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/rel/BeamSetOperatorRelBase.java
deleted file mode 100644
index 939c9c8..000
--- 
a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/rel/BeamSetOperatorRelBase.java
+++ /dev/null
@@ -1,98 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.beam.dsls.sql.rel;
-
-import java.io.Serializable;
-import java.util.List;
-import org.apache.beam.dsls.sql.BeamSqlEnv;
-import org.apache.beam.dsls.sql.schema.BeamSqlRow;
-import org.apache.beam.dsls.sql.transform.BeamSetOperatorsTransforms;
-import org.apache.beam.sdk.transforms.MapElements;
-import org.apache.beam.sdk.transforms.ParDo;
-import org.apache.beam.sdk.transforms.join.CoGbkResult;
-import org.apache.beam.sdk.transforms.join.CoGroupByKey;
-import org.apache.beam.sdk.transforms.join.KeyedPCollectionTuple;
-import org.apache.beam.sdk.transforms.windowing.WindowFn;
-import org.apache.beam.sdk.values.KV;
-import org.apache.beam.sdk.values.PCollection;
-import org.apache.beam.sdk.values.PCollectionTuple;
-import org.apache.beam.sdk.values.TupleTag;
-import org.apache.calcite.rel.RelNode;
-
-/**
- * Delegate for Set operators: {@code BeamUnionRel}, {@code BeamIntersectRel}
- * and {@code BeamMinusRel}.
- */
-public class BeamSetOperatorRelBase {
-  /**
-   * Set operator type.
-   */
-  public enum OpType implements Serializable {
-UNION,
-INTERSECT,
-MINUS
-  }
-
-  private BeamRelNode beamRelNode;
-  private List inputs;
-  private boolean all;
-  private OpType opType;
-
-  public BeamSetOperatorRelBase(BeamRelNode beamRelNode, OpType opType,
-  List inputs, boolean all) {
-this.beamRelNode = beamRelNode;
-this.opType = opType;
-this.inputs = inputs;
-this.all = all;
-  }
-
-  public PCollection buildBeamPipeline(PCollectionTuple 
inputPCollections
-  , BeamSqlEnv sqlEnv) throws Exception {
-   

[09/59] beam git commit: move dsls/sql to sdks/java/extensions/sql

[takidau]

http://git-wip-us.apache.org/repos/asf/beam/blob/ba493f85/sdks/java/extensions/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/logical/BeamSqlLogicalExpression.java
--
diff --git 
a/sdks/java/extensions/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/logical/BeamSqlLogicalExpression.java
 
b/sdks/java/extensions/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/logical/BeamSqlLogicalExpression.java
new file mode 100644
index 000..c9ff186
--- /dev/null
+++ 
b/sdks/java/extensions/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/logical/BeamSqlLogicalExpression.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.dsls.sql.interpreter.operator.logical;
+
+import java.util.List;
+
+import org.apache.beam.dsls.sql.interpreter.operator.BeamSqlExpression;
+import org.apache.calcite.sql.type.SqlTypeName;
+
+/**
+ * {@code BeamSqlExpression} for Logical operators.
+ */
+public abstract class BeamSqlLogicalExpression extends BeamSqlExpression {
+  private BeamSqlLogicalExpression(List operands, 
SqlTypeName outputType) {
+super(operands, outputType);
+  }
+  public BeamSqlLogicalExpression(List operands) {
+this(operands, SqlTypeName.BOOLEAN);
+  }
+
+  @Override
+  public boolean accept() {
+for (BeamSqlExpression exp : operands) {
+  // only accept BOOLEAN expression as operand
+  if (!exp.getOutputType().equals(SqlTypeName.BOOLEAN)) {
+return false;
+  }
+}
+return true;
+  }
+}

http://git-wip-us.apache.org/repos/asf/beam/blob/ba493f85/sdks/java/extensions/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/logical/BeamSqlNotExpression.java
--
diff --git 
a/sdks/java/extensions/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/logical/BeamSqlNotExpression.java
 
b/sdks/java/extensions/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/logical/BeamSqlNotExpression.java
new file mode 100644
index 000..6df52aa
--- /dev/null
+++ 
b/sdks/java/extensions/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/logical/BeamSqlNotExpression.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.dsls.sql.interpreter.operator.logical;
+
+import java.util.List;
+
+import org.apache.beam.dsls.sql.interpreter.operator.BeamSqlExpression;
+import org.apache.beam.dsls.sql.interpreter.operator.BeamSqlPrimitive;
+import org.apache.beam.dsls.sql.schema.BeamSqlRow;
+import org.apache.calcite.sql.type.SqlTypeName;
+
+/**
+ * {@code BeamSqlExpression} for logical operator: NOT.
+ *
+ * Whether boolean is not TRUE; returns UNKNOWN if boolean is UNKNOWN.
+ */
+public class BeamSqlNotExpression extends BeamSqlLogicalExpression {
+  public BeamSqlNotExpression(List operands) {
+super(operands);
+  }
+
+  @Override
+  public boolean accept() {
+if (numberOfOperands() != 1) {
+  return false;
+}
+return super.accept();
+  }
+
+  @Override public BeamSqlPrimitive evaluate(BeamSqlRow inputRow) {
+Boolean value = opValueEvaluated(0, inputRow);
+if (value == null) {
+  return BeamSqlPrimitive.of(SqlTypeName.BOOLEAN, null);
+} else {
+  return BeamSqlPrimitive.of(SqlTypeName.BOOLEAN, !value);
+}
+  }
+}

[35/59] beam git commit: rename package org.apache.beam.dsls.sql to org.apache.beam.sdk.extensions.sql

[takidau]

http://git-wip-us.apache.org/repos/asf/beam/blob/c1b5482d/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/interpreter/operator/arithmetic/BeamSqlModExpression.java
--
diff --git 
a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/interpreter/operator/arithmetic/BeamSqlModExpression.java
 
b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/interpreter/operator/arithmetic/BeamSqlModExpression.java
new file mode 100644
index 000..fc137da
--- /dev/null
+++ 
b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/interpreter/operator/arithmetic/BeamSqlModExpression.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.extensions.sql.interpreter.operator.arithmetic;
+
+import java.math.BigDecimal;
+import java.util.List;
+import 
org.apache.beam.sdk.extensions.sql.interpreter.operator.BeamSqlExpression;
+
+/**
+ * '%' operator.
+ */
+public class BeamSqlModExpression extends BeamSqlArithmeticExpression {
+  public BeamSqlModExpression(List operands) {
+super(operands, operands.get(1).getOutputType());
+  }
+
+  @Override protected BigDecimal calc(BigDecimal left, BigDecimal right) {
+return BigDecimal.valueOf(left.doubleValue() % right.doubleValue());
+  }
+}

http://git-wip-us.apache.org/repos/asf/beam/blob/c1b5482d/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/interpreter/operator/arithmetic/BeamSqlMultiplyExpression.java
--
diff --git 
a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/interpreter/operator/arithmetic/BeamSqlMultiplyExpression.java
 
b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/interpreter/operator/arithmetic/BeamSqlMultiplyExpression.java
new file mode 100644
index 000..7ea974c
--- /dev/null
+++ 
b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/interpreter/operator/arithmetic/BeamSqlMultiplyExpression.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.extensions.sql.interpreter.operator.arithmetic;
+
+import java.math.BigDecimal;
+import java.util.List;
+import 
org.apache.beam.sdk.extensions.sql.interpreter.operator.BeamSqlExpression;
+
+/**
+ * '*' operator.
+ */
+public class BeamSqlMultiplyExpression extends BeamSqlArithmeticExpression {
+  public BeamSqlMultiplyExpression(List operands) {
+super(operands);
+  }
+
+  @Override protected BigDecimal calc(BigDecimal left, BigDecimal right) {
+return left.multiply(right);
+  }
+}

http://git-wip-us.apache.org/repos/asf/beam/blob/c1b5482d/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/interpreter/operator/arithmetic/BeamSqlPlusExpression.java
--
diff --git 
a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/interpreter/operator/arithmetic/BeamSqlPlusExpression.java
 
b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/interpreter/operator/arithmetic/BeamSqlPlusExpression.java
new file mode 100644
index 000..3ce806f
--- /dev/null
+++ 

[02/59] beam git commit: move dsls/sql to sdks/java/extensions/sql

[takidau]

http://git-wip-us.apache.org/repos/asf/beam/blob/ba493f85/sdks/java/extensions/sql/src/test/java/org/apache/beam/dsls/sql/interpreter/operator/string/BeamSqlTrimExpressionTest.java
--
diff --git 
a/sdks/java/extensions/sql/src/test/java/org/apache/beam/dsls/sql/interpreter/operator/string/BeamSqlTrimExpressionTest.java
 
b/sdks/java/extensions/sql/src/test/java/org/apache/beam/dsls/sql/interpreter/operator/string/BeamSqlTrimExpressionTest.java
new file mode 100644
index 000..9ae9212
--- /dev/null
+++ 
b/sdks/java/extensions/sql/src/test/java/org/apache/beam/dsls/sql/interpreter/operator/string/BeamSqlTrimExpressionTest.java
@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.dsls.sql.interpreter.operator.string;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.beam.dsls.sql.interpreter.BeamSqlFnExecutorTestBase;
+import org.apache.beam.dsls.sql.interpreter.operator.BeamSqlExpression;
+import org.apache.beam.dsls.sql.interpreter.operator.BeamSqlPrimitive;
+import org.apache.calcite.sql.fun.SqlTrimFunction;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.junit.Test;
+
+/**
+ * Test for BeamSqlTrimExpression.
+ */
+public class BeamSqlTrimExpressionTest extends BeamSqlFnExecutorTestBase {
+
+  @Test public void accept() throws Exception {
+List operands = new ArrayList<>();
+
+operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, " hello "));
+assertTrue(new BeamSqlTrimExpression(operands).accept());
+
+operands.clear();
+operands.add(BeamSqlPrimitive.of(SqlTypeName.SYMBOL, 
SqlTrimFunction.Flag.BOTH));
+operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "he"));
+operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "hehe__hehe"));
+assertTrue(new BeamSqlTrimExpression(operands).accept());
+
+operands.clear();
+operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "he"));
+operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "hehe__hehe"));
+assertFalse(new BeamSqlTrimExpression(operands).accept());
+  }
+
+  @Test public void evaluate() throws Exception {
+List operands = new ArrayList<>();
+
+operands.add(BeamSqlPrimitive.of(SqlTypeName.SYMBOL, 
SqlTrimFunction.Flag.LEADING));
+operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "he"));
+operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "hehe__hehe"));
+assertEquals("__hehe",
+new BeamSqlTrimExpression(operands).evaluate(record).getValue());
+
+operands.clear();
+operands.add(BeamSqlPrimitive.of(SqlTypeName.SYMBOL, 
SqlTrimFunction.Flag.TRAILING));
+operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "he"));
+operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "hehe__hehe"));
+assertEquals("hehe__",
+new BeamSqlTrimExpression(operands).evaluate(record).getValue());
+
+operands.clear();
+operands.add(BeamSqlPrimitive.of(SqlTypeName.SYMBOL, 
SqlTrimFunction.Flag.BOTH));
+operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "he"));
+operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "__"));
+assertEquals("__",
+new BeamSqlTrimExpression(operands).evaluate(record).getValue());
+
+operands.clear();
+operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, " hello "));
+assertEquals("hello",
+new BeamSqlTrimExpression(operands).evaluate(record).getValue());
+  }
+
+  @Test public void leadingTrim() throws Exception {
+assertEquals("__hehe",
+BeamSqlTrimExpression.leadingTrim("hehe__hehe", "he"));
+  }
+
+  @Test public void trailingTrim() throws Exception {
+assertEquals("hehe__",
+BeamSqlTrimExpression.trailingTrim("hehe__hehe", "he"));
+  }
+
+  @Test public void trim() throws Exception {
+assertEquals("__",
+BeamSqlTrimExpression.leadingTrim(
+BeamSqlTrimExpression.trailingTrim("hehe__hehe", "he"), "he"
+));
+  }
+}

[13/59] beam git commit: move dsls/sql to sdks/java/extensions/sql

[takidau]

http://git-wip-us.apache.org/repos/asf/beam/blob/ba493f85/dsls/sql/src/test/java/org/apache/beam/dsls/sql/interpreter/operator/arithmetic/BeamSqlArithmeticExpressionTest.java
--
diff --git 
a/dsls/sql/src/test/java/org/apache/beam/dsls/sql/interpreter/operator/arithmetic/BeamSqlArithmeticExpressionTest.java
 
b/dsls/sql/src/test/java/org/apache/beam/dsls/sql/interpreter/operator/arithmetic/BeamSqlArithmeticExpressionTest.java
deleted file mode 100644
index a34f109..000
--- 
a/dsls/sql/src/test/java/org/apache/beam/dsls/sql/interpreter/operator/arithmetic/BeamSqlArithmeticExpressionTest.java
+++ /dev/null
@@ -1,237 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.beam.dsls.sql.interpreter.operator.arithmetic;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-
-import java.util.ArrayList;
-import java.util.List;
-import org.apache.beam.dsls.sql.interpreter.BeamSqlFnExecutorTestBase;
-import org.apache.beam.dsls.sql.interpreter.operator.BeamSqlExpression;
-import org.apache.beam.dsls.sql.interpreter.operator.BeamSqlPrimitive;
-import org.apache.calcite.sql.type.SqlTypeName;
-import org.junit.Test;
-
-/**
- * Tests for {@code BeamSqlArithmeticExpression}.
- */
-public class BeamSqlArithmeticExpressionTest extends BeamSqlFnExecutorTestBase 
{
-
-  @Test public void testAccept_normal() {
-List operands = new ArrayList<>();
-
-// byte, short
-operands.add(BeamSqlPrimitive.of(SqlTypeName.TINYINT, Byte.valueOf("1")));
-operands.add(BeamSqlPrimitive.of(SqlTypeName.SMALLINT, Short.MAX_VALUE));
-assertTrue(new BeamSqlPlusExpression(operands).accept());
-
-// integer, long
-operands.clear();
-operands.add(BeamSqlPrimitive.of(SqlTypeName.INTEGER, 1));
-operands.add(BeamSqlPrimitive.of(SqlTypeName.BIGINT, 1L));
-assertTrue(new BeamSqlPlusExpression(operands).accept());
-
-// float, double
-operands.clear();
-operands.add(BeamSqlPrimitive.of(SqlTypeName.FLOAT, 1.1F));
-operands.add(BeamSqlPrimitive.of(SqlTypeName.DOUBLE, 1.1));
-assertTrue(new BeamSqlPlusExpression(operands).accept());
-
-// varchar
-operands.clear();
-operands.add(BeamSqlPrimitive.of(SqlTypeName.FLOAT, 1.1F));
-operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "1"));
-assertFalse(new BeamSqlPlusExpression(operands).accept());
-  }
-
-  @Test public void testAccept_exception() {
-List operands = new ArrayList<>();
-
-// more than 2 operands
-operands.add(BeamSqlPrimitive.of(SqlTypeName.TINYINT, Byte.valueOf("1")));
-operands.add(BeamSqlPrimitive.of(SqlTypeName.SMALLINT, Short.MAX_VALUE));
-operands.add(BeamSqlPrimitive.of(SqlTypeName.SMALLINT, Short.MAX_VALUE));
-assertFalse(new BeamSqlPlusExpression(operands).accept());
-
-// boolean
-operands.clear();
-operands.add(BeamSqlPrimitive.of(SqlTypeName.TINYINT, Byte.valueOf("1")));
-operands.add(BeamSqlPrimitive.of(SqlTypeName.BOOLEAN, true));
-assertFalse(new BeamSqlPlusExpression(operands).accept());
-  }
-
-  @Test public void testPlus() {
-List operands = new ArrayList<>();
-
-// integer + integer => integer
-operands.add(BeamSqlPrimitive.of(SqlTypeName.INTEGER, 1));
-operands.add(BeamSqlPrimitive.of(SqlTypeName.INTEGER, 1));
-assertEquals(2, new 
BeamSqlPlusExpression(operands).evaluate(record).getValue());
-
-// integer + long => long
-operands.clear();
-operands.add(BeamSqlPrimitive.of(SqlTypeName.INTEGER, 1));
-operands.add(BeamSqlPrimitive.of(SqlTypeName.BIGINT, 1L));
-assertEquals(2L, new 
BeamSqlPlusExpression(operands).evaluate(record).getValue());
-
-// long + long => long
-operands.clear();
-operands.add(BeamSqlPrimitive.of(SqlTypeName.BIGINT, 1L));
-operands.add(BeamSqlPrimitive.of(SqlTypeName.BIGINT, 1L));
-assertEquals(2L, new 
BeamSqlPlusExpression(operands).evaluate(record).getValue());
-
-// float + long => float
-operands.clear();
-operands.add(BeamSqlPrimitive.of(SqlTypeName.FLOAT, 1.1F));
-

[44/59] beam git commit: move all implementation classes/packages into impl package

[takidau]

http://git-wip-us.apache.org/repos/asf/beam/blob/febd044a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/interpreter/operator/string/BeamSqlOverlayExpressionTest.java
--
diff --git 
a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/interpreter/operator/string/BeamSqlOverlayExpressionTest.java
 
b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/interpreter/operator/string/BeamSqlOverlayExpressionTest.java
deleted file mode 100644
index 2ca0a98..000
--- 
a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/interpreter/operator/string/BeamSqlOverlayExpressionTest.java
+++ /dev/null
@@ -1,87 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.beam.sdk.extensions.sql.interpreter.operator.string;
-
-import static org.junit.Assert.assertTrue;
-
-import java.util.ArrayList;
-import java.util.List;
-import 
org.apache.beam.sdk.extensions.sql.interpreter.BeamSqlFnExecutorTestBase;
-import 
org.apache.beam.sdk.extensions.sql.interpreter.operator.BeamSqlExpression;
-import 
org.apache.beam.sdk.extensions.sql.interpreter.operator.BeamSqlPrimitive;
-import org.apache.calcite.sql.type.SqlTypeName;
-import org.junit.Assert;
-import org.junit.Test;
-
-/**
- * Test for BeamSqlOverlayExpression.
- */
-public class BeamSqlOverlayExpressionTest extends BeamSqlFnExecutorTestBase {
-
-  @Test public void accept() throws Exception {
-List operands = new ArrayList<>();
-
-operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "hello"));
-operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "hello"));
-operands.add(BeamSqlPrimitive.of(SqlTypeName.INTEGER, 1));
-assertTrue(new BeamSqlOverlayExpression(operands).accept());
-
-operands.clear();
-operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "hello"));
-operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "hello"));
-operands.add(BeamSqlPrimitive.of(SqlTypeName.INTEGER, 1));
-operands.add(BeamSqlPrimitive.of(SqlTypeName.INTEGER, 2));
-assertTrue(new BeamSqlOverlayExpression(operands).accept());
-  }
-
-  @Test public void evaluate() throws Exception {
-List operands = new ArrayList<>();
-
-operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "w333rce"));
-operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "resou"));
-operands.add(BeamSqlPrimitive.of(SqlTypeName.INTEGER, 3));
-Assert.assertEquals("w3resou3rce",
-new BeamSqlOverlayExpression(operands).evaluate(record).getValue());
-
-operands.clear();
-operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "w333rce"));
-operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "resou"));
-operands.add(BeamSqlPrimitive.of(SqlTypeName.INTEGER, 3));
-operands.add(BeamSqlPrimitive.of(SqlTypeName.INTEGER, 4));
-Assert.assertEquals("w3resou33rce",
-new BeamSqlOverlayExpression(operands).evaluate(record).getValue());
-
-operands.clear();
-operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "w333rce"));
-operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "resou"));
-operands.add(BeamSqlPrimitive.of(SqlTypeName.INTEGER, 3));
-operands.add(BeamSqlPrimitive.of(SqlTypeName.INTEGER, 5));
-Assert.assertEquals("w3resou3rce",
-new BeamSqlOverlayExpression(operands).evaluate(record).getValue());
-
-operands.clear();
-operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "w333rce"));
-operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "resou"));
-operands.add(BeamSqlPrimitive.of(SqlTypeName.INTEGER, 3));
-operands.add(BeamSqlPrimitive.of(SqlTypeName.INTEGER, 7));
-Assert.assertEquals("w3resouce",
-new BeamSqlOverlayExpression(operands).evaluate(record).getValue());
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/beam/blob/febd044a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/interpreter/operator/string/BeamSqlPositionExpressionTest.java
--
diff --git 

[03/59] beam git commit: move dsls/sql to sdks/java/extensions/sql

[takidau]

http://git-wip-us.apache.org/repos/asf/beam/blob/ba493f85/sdks/java/extensions/sql/src/test/java/org/apache/beam/dsls/sql/interpreter/operator/arithmetic/BeamSqlArithmeticExpressionTest.java
--
diff --git 
a/sdks/java/extensions/sql/src/test/java/org/apache/beam/dsls/sql/interpreter/operator/arithmetic/BeamSqlArithmeticExpressionTest.java
 
b/sdks/java/extensions/sql/src/test/java/org/apache/beam/dsls/sql/interpreter/operator/arithmetic/BeamSqlArithmeticExpressionTest.java
new file mode 100644
index 000..a34f109
--- /dev/null
+++ 
b/sdks/java/extensions/sql/src/test/java/org/apache/beam/dsls/sql/interpreter/operator/arithmetic/BeamSqlArithmeticExpressionTest.java
@@ -0,0 +1,237 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.dsls.sql.interpreter.operator.arithmetic;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.beam.dsls.sql.interpreter.BeamSqlFnExecutorTestBase;
+import org.apache.beam.dsls.sql.interpreter.operator.BeamSqlExpression;
+import org.apache.beam.dsls.sql.interpreter.operator.BeamSqlPrimitive;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.junit.Test;
+
+/**
+ * Tests for {@code BeamSqlArithmeticExpression}.
+ */
+public class BeamSqlArithmeticExpressionTest extends BeamSqlFnExecutorTestBase 
{
+
+  @Test public void testAccept_normal() {
+List operands = new ArrayList<>();
+
+// byte, short
+operands.add(BeamSqlPrimitive.of(SqlTypeName.TINYINT, Byte.valueOf("1")));
+operands.add(BeamSqlPrimitive.of(SqlTypeName.SMALLINT, Short.MAX_VALUE));
+assertTrue(new BeamSqlPlusExpression(operands).accept());
+
+// integer, long
+operands.clear();
+operands.add(BeamSqlPrimitive.of(SqlTypeName.INTEGER, 1));
+operands.add(BeamSqlPrimitive.of(SqlTypeName.BIGINT, 1L));
+assertTrue(new BeamSqlPlusExpression(operands).accept());
+
+// float, double
+operands.clear();
+operands.add(BeamSqlPrimitive.of(SqlTypeName.FLOAT, 1.1F));
+operands.add(BeamSqlPrimitive.of(SqlTypeName.DOUBLE, 1.1));
+assertTrue(new BeamSqlPlusExpression(operands).accept());
+
+// varchar
+operands.clear();
+operands.add(BeamSqlPrimitive.of(SqlTypeName.FLOAT, 1.1F));
+operands.add(BeamSqlPrimitive.of(SqlTypeName.VARCHAR, "1"));
+assertFalse(new BeamSqlPlusExpression(operands).accept());
+  }
+
+  @Test public void testAccept_exception() {
+List operands = new ArrayList<>();
+
+// more than 2 operands
+operands.add(BeamSqlPrimitive.of(SqlTypeName.TINYINT, Byte.valueOf("1")));
+operands.add(BeamSqlPrimitive.of(SqlTypeName.SMALLINT, Short.MAX_VALUE));
+operands.add(BeamSqlPrimitive.of(SqlTypeName.SMALLINT, Short.MAX_VALUE));
+assertFalse(new BeamSqlPlusExpression(operands).accept());
+
+// boolean
+operands.clear();
+operands.add(BeamSqlPrimitive.of(SqlTypeName.TINYINT, Byte.valueOf("1")));
+operands.add(BeamSqlPrimitive.of(SqlTypeName.BOOLEAN, true));
+assertFalse(new BeamSqlPlusExpression(operands).accept());
+  }
+
+  @Test public void testPlus() {
+List operands = new ArrayList<>();
+
+// integer + integer => integer
+operands.add(BeamSqlPrimitive.of(SqlTypeName.INTEGER, 1));
+operands.add(BeamSqlPrimitive.of(SqlTypeName.INTEGER, 1));
+assertEquals(2, new 
BeamSqlPlusExpression(operands).evaluate(record).getValue());
+
+// integer + long => long
+operands.clear();
+operands.add(BeamSqlPrimitive.of(SqlTypeName.INTEGER, 1));
+operands.add(BeamSqlPrimitive.of(SqlTypeName.BIGINT, 1L));
+assertEquals(2L, new 
BeamSqlPlusExpression(operands).evaluate(record).getValue());
+
+// long + long => long
+operands.clear();
+operands.add(BeamSqlPrimitive.of(SqlTypeName.BIGINT, 1L));
+operands.add(BeamSqlPrimitive.of(SqlTypeName.BIGINT, 1L));
+assertEquals(2L, new 
BeamSqlPlusExpression(operands).evaluate(record).getValue());
+
+// float + long => float
+operands.clear();
+operands.add(BeamSqlPrimitive.of(SqlTypeName.FLOAT, 

[01/59] beam git commit: move dsls/sql to sdks/java/extensions/sql

[takidau]

Repository: beam
Updated Branches:
  refs/heads/DSL_SQL d32aea969 -> 10962a34d


http://git-wip-us.apache.org/repos/asf/beam/blob/ba493f85/sdks/java/extensions/sql/src/test/java/org/apache/beam/dsls/sql/schema/text/BeamTextCSVTableTest.java
--
diff --git 
a/sdks/java/extensions/sql/src/test/java/org/apache/beam/dsls/sql/schema/text/BeamTextCSVTableTest.java
 
b/sdks/java/extensions/sql/src/test/java/org/apache/beam/dsls/sql/schema/text/BeamTextCSVTableTest.java
new file mode 100644
index 000..b6e11e5
--- /dev/null
+++ 
b/sdks/java/extensions/sql/src/test/java/org/apache/beam/dsls/sql/schema/text/BeamTextCSVTableTest.java
@@ -0,0 +1,176 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.dsls.sql.schema.text;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.PrintStream;
+import java.nio.file.FileVisitResult;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.SimpleFileVisitor;
+import java.nio.file.attribute.BasicFileAttributes;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import org.apache.beam.dsls.sql.planner.BeamQueryPlanner;
+import org.apache.beam.dsls.sql.schema.BeamSqlRow;
+import org.apache.beam.dsls.sql.schema.BeamSqlRowType;
+import org.apache.beam.dsls.sql.utils.CalciteUtils;
+import org.apache.beam.sdk.testing.PAssert;
+import org.apache.beam.sdk.testing.TestPipeline;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.rel.type.RelDataTypeFactory;
+import org.apache.calcite.rel.type.RelProtoDataType;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.apache.commons.csv.CSVFormat;
+import org.apache.commons.csv.CSVPrinter;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Rule;
+import org.junit.Test;
+
+/**
+ * Tests for {@code BeamTextCSVTable}.
+ */
+public class BeamTextCSVTableTest {
+
+  @Rule public TestPipeline pipeline = TestPipeline.create();
+  @Rule public TestPipeline pipeline2 = TestPipeline.create();
+
+  /**
+   * testData.
+   *
+   * 
+   * The types of the csv fields are:
+   * integer,bigint,float,double,string
+   * 
+   */
+  private static Object[] data1 = new Object[] { 1, 1L, 1.1F, 1.1, "james" };
+  private static Object[] data2 = new Object[] { 2, 2L, 2.2F, 2.2, "bond" };
+
+  private static List testData = Arrays.asList(data1, data2);
+  private static List testDataRows = new ArrayList() {{
+for (Object[] data : testData) {
+  add(buildRow(data));
+}
+  }};
+
+  private static Path tempFolder;
+  private static File readerSourceFile;
+  private static File writerTargetFile;
+
+  @Test public void testBuildIOReader() {
+PCollection rows = new BeamTextCSVTable(buildBeamSqlRowType(),
+readerSourceFile.getAbsolutePath()).buildIOReader(pipeline);
+PAssert.that(rows).containsInAnyOrder(testDataRows);
+pipeline.run();
+  }
+
+  @Test public void testBuildIOWriter() {
+new BeamTextCSVTable(buildBeamSqlRowType(),
+readerSourceFile.getAbsolutePath()).buildIOReader(pipeline)
+.apply(new BeamTextCSVTable(buildBeamSqlRowType(), 
writerTargetFile.getAbsolutePath())
+.buildIOWriter());
+pipeline.run();
+
+PCollection rows = new BeamTextCSVTable(buildBeamSqlRowType(),
+writerTargetFile.getAbsolutePath()).buildIOReader(pipeline2);
+
+// confirm the two reads match
+PAssert.that(rows).containsInAnyOrder(testDataRows);
+pipeline2.run();
+  }
+
+  @BeforeClass public static void setUp() throws IOException {
+tempFolder = Files.createTempDirectory("BeamTextTableTest");
+readerSourceFile = writeToFile(testData, "readerSourceFile.txt");
+writerTargetFile = writeToFile(testData, "writerTargetFile.txt");
+  }
+
+  @AfterClass public static void teardownClass() throws IOException {
+Files.walkFileTree(tempFolder, new SimpleFileVisitor() {
+
+  @Override public FileVisitResult visitFile(Path file, 
BasicFileAttributes attrs)
+  throws IOException {
+

[1/2] beam git commit: remove README.md and update usages in BeamSqlExample

[takidau]

Repository: beam
Updated Branches:
  refs/heads/DSL_SQL ada24c059 -> 0eb8e89ba


remove README.md and update usages in BeamSqlExample


Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/f1aa390d
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/f1aa390d
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/f1aa390d

Branch: refs/heads/DSL_SQL
Commit: f1aa390d1989543f2848dae6b26596ffd1a5d8db
Parents: ada24c0
Author: mingmxu 
Authored: Thu Jul 20 14:32:42 2017 -0700
Committer: mingmxu 
Committed: Thu Jul 20 14:32:42 2017 -0700

--
 dsls/sql/README.md  | 24 
 .../beam/dsls/sql/example/BeamSqlExample.java   | 23 +++
 2 files changed, 13 insertions(+), 34 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/beam/blob/f1aa390d/dsls/sql/README.md
--
diff --git a/dsls/sql/README.md b/dsls/sql/README.md
deleted file mode 100644
index ae9e0f3..000
--- a/dsls/sql/README.md
+++ /dev/null
@@ -1,24 +0,0 @@
-
-
-# Beam SQL
-
-Beam SQL provides a new interface, to execute a SQL query as a Beam pipeline.
-
-*It's working in progress...*

http://git-wip-us.apache.org/repos/asf/beam/blob/f1aa390d/dsls/sql/src/main/java/org/apache/beam/dsls/sql/example/BeamSqlExample.java
--
diff --git 
a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/example/BeamSqlExample.java 
b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/example/BeamSqlExample.java
index 91df2be..4e364e1 100644
--- 
a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/example/BeamSqlExample.java
+++ 
b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/example/BeamSqlExample.java
@@ -34,16 +34,19 @@ import org.apache.beam.sdk.values.PBegin;
 import org.apache.beam.sdk.values.PCollection;
 import org.apache.beam.sdk.values.PCollectionTuple;
 import org.apache.beam.sdk.values.TupleTag;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 
 /**
  * This is a quick example, which uses Beam SQL DSL to create a data pipeline.
  *
+ * Run the example with
+ * 
+ * mvn -pl dsls/sql compile exec:java \
+ *  -Dexec.mainClass=org.apache.beam.dsls.sql.example.BeamSqlExample \
+ *   -Dexec.args="--runner=DirectRunner" -Pdirect-runner
+ * 
+ *
  */
 class BeamSqlExample {
-  private static final Logger LOG = 
LoggerFactory.getLogger(BeamSqlExample.class);
-
   public static void main(String[] args) throws Exception {
 PipelineOptions options = 
PipelineOptionsFactory.fromArgs(args).as(PipelineOptions.class);
 Pipeline p = Pipeline.create(options);
@@ -63,9 +66,9 @@ class BeamSqlExample {
 
 //Case 1. run a simple SQL query over input PCollection with 
BeamSql.simpleQuery;
 PCollection outputStream = inputTable.apply(
-BeamSql.simpleQuery("select c2, c3 from PCOLLECTION where c1=1"));
+BeamSql.simpleQuery("select c1, c2, c3 from PCOLLECTION where c1=1"));
 
-//log out the output record;
+//print the output record of case 1;
 outputStream.apply("log_result",
 MapElements.via(new SimpleFunction() {
   public Void apply(BeamSqlRow input) {
@@ -74,12 +77,12 @@ class BeamSqlExample {
   }
 }));
 
-//Case 2. run the query with BeamSql.query
+//Case 2. run the query with BeamSql.query over result PCollection of case 
1.
 PCollection outputStream2 =
-PCollectionTuple.of(new TupleTag("TABLE_B"), inputTable)
-.apply(BeamSql.query("select c2, c3 from TABLE_B where c1=1"));
+PCollectionTuple.of(new TupleTag("CASE1_RESULT"), 
outputStream)
+.apply(BeamSql.query("select c2, c3 from CASE1_RESULT where c1=1"));
 
-//log out the output record;
+//print the output record of case 2;
 outputStream2.apply("log_result",
 MapElements.via(new SimpleFunction() {
   @Override

[2/2] beam git commit: [BEAM-2555] This closes #3605

[takidau]

[BEAM-2555] This closes #3605


Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/0eb8e89b
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/0eb8e89b
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/0eb8e89b

Branch: refs/heads/DSL_SQL
Commit: 0eb8e89babbe758c15457bc10d44c815ebaf7709
Parents: ada24c0 f1aa390
Author: Tyler Akidau 
Authored: Thu Jul 20 16:14:51 2017 -0700
Committer: Tyler Akidau 
Committed: Thu Jul 20 16:14:51 2017 -0700

--
 dsls/sql/README.md  | 24 
 .../beam/dsls/sql/example/BeamSqlExample.java   | 23 +++
 2 files changed, 13 insertions(+), 34 deletions(-)
--

[1/2] beam git commit: rebased, add RAND/RAND_INTEGER

[takidau]

Repository: beam
Updated Branches:
  refs/heads/DSL_SQL caf647daf -> a1f7cf6de


rebased, add RAND/RAND_INTEGER

update as commented


Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/97c9b075
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/97c9b075
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/97c9b075

Branch: refs/heads/DSL_SQL
Commit: 97c9b07576f9baa1bf10aaf1817f731137c1a1a0
Parents: caf647d
Author: mingmxu 
Authored: Tue Jul 18 00:09:39 2017 -0700
Committer: Tyler Akidau 
Committed: Tue Jul 18 17:54:43 2017 -0700

--
 .../dsls/sql/interpreter/BeamSqlFnExecutor.java |  8 +++
 .../operator/math/BeamSqlRandExpression.java| 54 
 .../math/BeamSqlRandIntegerExpression.java  | 58 +
 .../BeamSqlMathFunctionsIntegrationTest.java| 67 
 4 files changed, 187 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/beam/blob/97c9b075/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/BeamSqlFnExecutor.java
--
diff --git 
a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/BeamSqlFnExecutor.java
 
b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/BeamSqlFnExecutor.java
index 08d124f..64bc880 100644
--- 
a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/BeamSqlFnExecutor.java
+++ 
b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/BeamSqlFnExecutor.java
@@ -68,6 +68,8 @@ import 
org.apache.beam.dsls.sql.interpreter.operator.math.BeamSqlLogExpression;
 import org.apache.beam.dsls.sql.interpreter.operator.math.BeamSqlPiExpression;
 import 
org.apache.beam.dsls.sql.interpreter.operator.math.BeamSqlPowerExpression;
 import 
org.apache.beam.dsls.sql.interpreter.operator.math.BeamSqlRadiansExpression;
+import 
org.apache.beam.dsls.sql.interpreter.operator.math.BeamSqlRandExpression;
+import 
org.apache.beam.dsls.sql.interpreter.operator.math.BeamSqlRandIntegerExpression;
 import 
org.apache.beam.dsls.sql.interpreter.operator.math.BeamSqlRoundExpression;
 import 
org.apache.beam.dsls.sql.interpreter.operator.math.BeamSqlSignExpression;
 import org.apache.beam.dsls.sql.interpreter.operator.math.BeamSqlSinExpression;
@@ -299,6 +301,12 @@ public class BeamSqlFnExecutor implements 
BeamSqlExpressionExecutor {
 case "TRUNCATE":
   ret = new BeamSqlTruncateExpression(subExps);
   break;
+case "RAND":
+  ret = new BeamSqlRandExpression(subExps);
+  break;
+case "RAND_INTEGER":
+  ret = new BeamSqlRandIntegerExpression(subExps);
+  break;
 
 // string operators
 case "||":

http://git-wip-us.apache.org/repos/asf/beam/blob/97c9b075/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/math/BeamSqlRandExpression.java
--
diff --git 
a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/math/BeamSqlRandExpression.java
 
b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/math/BeamSqlRandExpression.java
new file mode 100644
index 000..944936b
--- /dev/null
+++ 
b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/interpreter/operator/math/BeamSqlRandExpression.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.dsls.sql.interpreter.operator.math;
+
+import java.util.List;
+import java.util.Random;
+import org.apache.beam.dsls.sql.interpreter.operator.BeamSqlExpression;
+import org.apache.beam.dsls.sql.interpreter.operator.BeamSqlPrimitive;
+import org.apache.beam.dsls.sql.schema.BeamSqlRow;
+import org.apache.calcite.sql.type.SqlTypeName;
+
+/**
+ * {@code BeamSqlMathUnaryExpression} for 'RAND([seed])' function.
+ */
+public class BeamSqlRandExpression extends BeamSqlExpression {
+  private Random rand = new Random();
+  private Integer seed = null;
+
+  public BeamSqlRandExpression(List 

[2/2] beam git commit: [BEAM-2585] This closes #3552

[takidau]

[BEAM-2585] This closes #3552


Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/a1f7cf6d
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/a1f7cf6d
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/a1f7cf6d

Branch: refs/heads/DSL_SQL
Commit: a1f7cf6deaa2e7344e695c25234d82086194c560
Parents: caf647d 97c9b07
Author: Tyler Akidau 
Authored: Tue Jul 18 17:55:26 2017 -0700
Committer: Tyler Akidau 
Committed: Tue Jul 18 17:55:26 2017 -0700

--
 .../dsls/sql/interpreter/BeamSqlFnExecutor.java |  8 +++
 .../operator/math/BeamSqlRandExpression.java| 54 
 .../math/BeamSqlRandIntegerExpression.java  | 58 +
 .../BeamSqlMathFunctionsIntegrationTest.java| 67 
 4 files changed, 187 insertions(+)
--

[2/2] beam git commit: [BEAM-2610] This closes #3554

[takidau]

[BEAM-2610] This closes #3554


Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/5fea7463
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/5fea7463
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/5fea7463

Branch: refs/heads/DSL_SQL
Commit: 5fea74638551c4e1928ce0f3ddea833354764d9f
Parents: ec494f6 45a3fe0
Author: Tyler Akidau 
Authored: Thu Jul 13 08:56:35 2017 -0700
Committer: Tyler Akidau 
Committed: Thu Jul 13 08:56:35 2017 -0700

--
 dsls/pom.xml | 2 +-
 dsls/sql/pom.xml | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)
--

[1/2] beam git commit: upgrade pom to 2.2.0-SNAPSHOT

[takidau]

Repository: beam
Updated Branches:
  refs/heads/DSL_SQL ec494f675 -> 5fea74638


upgrade pom to 2.2.0-SNAPSHOT


Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/45a3fe0a
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/45a3fe0a
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/45a3fe0a

Branch: refs/heads/DSL_SQL
Commit: 45a3fe0a4c97684d44eb94c1a1da4515ad3779c2
Parents: ec494f6
Author: mingmxu 
Authored: Wed Jul 12 19:04:08 2017 -0700
Committer: mingmxu 
Committed: Wed Jul 12 20:14:54 2017 -0700

--
 dsls/pom.xml | 2 +-
 dsls/sql/pom.xml | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/beam/blob/45a3fe0a/dsls/pom.xml
--
diff --git a/dsls/pom.xml b/dsls/pom.xml
index a518d03..1647114 100644
--- a/dsls/pom.xml
+++ b/dsls/pom.xml
@@ -22,7 +22,7 @@
   
 org.apache.beam
 beam-parent
-2.1.0-SNAPSHOT
+2.2.0-SNAPSHOT
 ../pom.xml
   
 

http://git-wip-us.apache.org/repos/asf/beam/blob/45a3fe0a/dsls/sql/pom.xml
--
diff --git a/dsls/sql/pom.xml b/dsls/sql/pom.xml
index 54f590e..5e670a0 100644
--- a/dsls/sql/pom.xml
+++ b/dsls/sql/pom.xml
@@ -24,7 +24,7 @@
   
 org.apache.beam
 beam-dsls-parent
-2.1.0-SNAPSHOT
+2.2.0-SNAPSHOT
 ../pom.xml
   
 

[18/50] [abbrv] beam git commit: Update Dataflow container version to 20170706

[takidau]

Update Dataflow container version to 20170706


Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/92eec586
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/92eec586
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/92eec586

Branch: refs/heads/DSL_SQL
Commit: 92eec586c966d9cce89539596dd750c757d92316
Parents: 699d59a
Author: Kenneth Knowles 
Authored: Thu Jul 6 11:07:38 2017 -0700
Committer: Tyler Akidau 
Committed: Wed Jul 12 20:01:00 2017 -0700

--
 runners/google-cloud-dataflow-java/pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/beam/blob/92eec586/runners/google-cloud-dataflow-java/pom.xml
--
diff --git a/runners/google-cloud-dataflow-java/pom.xml 
b/runners/google-cloud-dataflow-java/pom.xml
index 91908cd..c8d63ac 100644
--- a/runners/google-cloud-dataflow-java/pom.xml
+++ b/runners/google-cloud-dataflow-java/pom.xml
@@ -33,7 +33,7 @@
   jar
 
   
-
beam-master-20170623
+
beam-master-20170706
 
1
 
6
   

[35/50] [abbrv] beam git commit: Adds DynamicDestinations support to FileBasedSink

[takidau]

http://git-wip-us.apache.org/repos/asf/beam/blob/4c336e84/sdks/java/core/src/main/java/org/apache/beam/sdk/io/DynamicFileDestinations.java
--
diff --git 
a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/DynamicFileDestinations.java
 
b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/DynamicFileDestinations.java
new file mode 100644
index 000..e7ef0f6
--- /dev/null
+++ 
b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/DynamicFileDestinations.java
@@ -0,0 +1,115 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.io;
+
+import javax.annotation.Nullable;
+import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.io.DefaultFilenamePolicy.Params;
+import org.apache.beam.sdk.io.DefaultFilenamePolicy.ParamsCoder;
+import org.apache.beam.sdk.io.FileBasedSink.DynamicDestinations;
+import org.apache.beam.sdk.io.FileBasedSink.FilenamePolicy;
+import org.apache.beam.sdk.transforms.SerializableFunction;
+import org.apache.beam.sdk.transforms.display.DisplayData;
+
+/** Some helper classes that derive from {@link 
FileBasedSink.DynamicDestinations}. */
+public class DynamicFileDestinations {
+  /** Always returns a constant {@link FilenamePolicy}. */
+  private static class ConstantFilenamePolicy extends 
DynamicDestinations {
+private final FilenamePolicy filenamePolicy;
+
+public ConstantFilenamePolicy(FilenamePolicy filenamePolicy) {
+  this.filenamePolicy = filenamePolicy;
+}
+
+@Override
+public Void getDestination(T element) {
+  return (Void) null;
+}
+
+@Override
+public Coder getDestinationCoder() {
+  return null;
+}
+
+@Override
+public Void getDefaultDestination() {
+  return (Void) null;
+}
+
+@Override
+public FilenamePolicy getFilenamePolicy(Void destination) {
+  return filenamePolicy;
+}
+
+@Override
+public void populateDisplayData(DisplayData.Builder builder) {
+  filenamePolicy.populateDisplayData(builder);
+}
+  }
+
+  /**
+   * A base class for a {@link DynamicDestinations} object that returns 
differently-configured
+   * instances of {@link DefaultFilenamePolicy}.
+   */
+  private static class DefaultPolicyDestinations extends 
DynamicDestinations {
+SerializableFunction destinationFunction;
+Params emptyDestination;
+
+public DefaultPolicyDestinations(
+SerializableFunction destinationFunction, Params 
emptyDestination) {
+  this.destinationFunction = destinationFunction;
+  this.emptyDestination = emptyDestination;
+}
+
+@Override
+public Params getDestination(UserT element) {
+  return destinationFunction.apply(element);
+}
+
+@Override
+public Params getDefaultDestination() {
+  return emptyDestination;
+}
+
+@Nullable
+@Override
+public Coder getDestinationCoder() {
+  return ParamsCoder.of();
+}
+
+@Override
+public FilenamePolicy getFilenamePolicy(DefaultFilenamePolicy.Params 
params) {
+  return DefaultFilenamePolicy.fromParams(params);
+}
+  }
+
+  /** Returns a {@link DynamicDestinations} that always returns the same 
{@link FilenamePolicy}. */
+  public static  DynamicDestinations constant(FilenamePolicy 
filenamePolicy) {
+return new ConstantFilenamePolicy<>(filenamePolicy);
+  }
+
+  /**
+   * Returns a {@link DynamicDestinations} that returns instances of {@link 
DefaultFilenamePolicy}
+   * configured with the given {@link Params}.
+   */
+  public static  DynamicDestinations toDefaultPolicies(
+  SerializableFunction destinationFunction, Params 
emptyDestination) {
+return new DefaultPolicyDestinations<>(destinationFunction, 
emptyDestination);
+  }
+}

http://git-wip-us.apache.org/repos/asf/beam/blob/4c336e84/sdks/java/core/src/main/java/org/apache/beam/sdk/io/FileBasedSink.java
--
diff --git 
a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/FileBasedSink.java 

[40/50] [abbrv] beam git commit: [BEAM-2447] Reintroduces DoFn.ProcessContinuation

[takidau]

[BEAM-2447] Reintroduces DoFn.ProcessContinuation


Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/4f7f1699
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/4f7f1699
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/4f7f1699

Branch: refs/heads/DSL_SQL
Commit: 4f7f16990a8fc49a9b6ae199809f0ada7dc7448d
Parents: bd2a8cc
Author: Eugene Kirpichov 
Authored: Tue Jun 13 16:50:35 2017 -0700
Committer: Tyler Akidau 
Committed: Wed Jul 12 20:01:02 2017 -0700

--
 .../core/construction/SplittableParDoTest.java  |  10 +-
 ...eBoundedSplittableProcessElementInvoker.java |  35 ++-
 .../core/SplittableParDoViaKeyedWorkItems.java  |   9 +-
 .../core/SplittableProcessElementInvoker.java   |  25 -
 ...ndedSplittableProcessElementInvokerTest.java |  45 +++--
 .../core/SplittableParDoProcessFnTest.java  |  99 --
 .../org/apache/beam/sdk/transforms/DoFn.java|  51 +-
 .../reflect/ByteBuddyDoFnInvokerFactory.java|  19 +++-
 .../sdk/transforms/reflect/DoFnInvoker.java |   4 +-
 .../sdk/transforms/reflect/DoFnSignature.java   |  10 +-
 .../sdk/transforms/reflect/DoFnSignatures.java  |  22 +++-
 .../splittabledofn/OffsetRangeTracker.java  |  10 ++
 .../splittabledofn/RestrictionTracker.java  |  11 +-
 .../beam/sdk/transforms/SplittableDoFnTest.java | 100 ---
 .../transforms/reflect/DoFnInvokersTest.java|  93 +
 .../DoFnSignaturesProcessElementTest.java   |   2 +-
 .../DoFnSignaturesSplittableDoFnTest.java   |  83 +--
 17 files changed, 487 insertions(+), 141 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/beam/blob/4f7f1699/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/SplittableParDoTest.java
--
diff --git 
a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/SplittableParDoTest.java
 
b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/SplittableParDoTest.java
index f4c596e..267232c 100644
--- 
a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/SplittableParDoTest.java
+++ 
b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/SplittableParDoTest.java
@@ -17,6 +17,7 @@
  */
 package org.apache.beam.runners.core.construction;
 
+import static org.apache.beam.sdk.transforms.DoFn.ProcessContinuation.stop;
 import static org.junit.Assert.assertEquals;
 
 import java.io.Serializable;
@@ -24,8 +25,6 @@ import org.apache.beam.sdk.Pipeline;
 import org.apache.beam.sdk.testing.TestPipeline;
 import org.apache.beam.sdk.transforms.Create;
 import org.apache.beam.sdk.transforms.DoFn;
-import org.apache.beam.sdk.transforms.DoFn.BoundedPerElement;
-import org.apache.beam.sdk.transforms.DoFn.UnboundedPerElement;
 import org.apache.beam.sdk.transforms.ParDo;
 import org.apache.beam.sdk.transforms.splittabledofn.HasDefaultTracker;
 import org.apache.beam.sdk.transforms.splittabledofn.RestrictionTracker;
@@ -70,7 +69,6 @@ public class SplittableParDoTest {
 public void checkDone() {}
   }
 
-  @BoundedPerElement
   private static class BoundedFakeFn extends DoFn {
 @ProcessElement
 public void processElement(ProcessContext context, SomeRestrictionTracker 
tracker) {}
@@ -81,10 +79,12 @@ public class SplittableParDoTest {
 }
   }
 
-  @UnboundedPerElement
   private static class UnboundedFakeFn extends DoFn {
 @ProcessElement
-public void processElement(ProcessContext context, SomeRestrictionTracker 
tracker) {}
+public ProcessContinuation processElement(
+ProcessContext context, SomeRestrictionTracker tracker) {
+  return stop();
+}
 
 @GetInitialRestriction
 public SomeRestriction getInitialRestriction(Integer element) {

http://git-wip-us.apache.org/repos/asf/beam/blob/4f7f1699/runners/core-java/src/main/java/org/apache/beam/runners/core/OutputAndTimeBoundedSplittableProcessElementInvoker.java
--
diff --git 
a/runners/core-java/src/main/java/org/apache/beam/runners/core/OutputAndTimeBoundedSplittableProcessElementInvoker.java
 
b/runners/core-java/src/main/java/org/apache/beam/runners/core/OutputAndTimeBoundedSplittableProcessElementInvoker.java
index 475abf2..0c956d5 100644
--- 
a/runners/core-java/src/main/java/org/apache/beam/runners/core/OutputAndTimeBoundedSplittableProcessElementInvoker.java
+++ 
b/runners/core-java/src/main/java/org/apache/beam/runners/core/OutputAndTimeBoundedSplittableProcessElementInvoker.java
@@ -96,7 +96,7 @@ public class 

[10/50] [abbrv] beam git commit: Disallow Combiner Lifting for multi-window WindowFns

[takidau]

Disallow Combiner Lifting for multi-window WindowFns


Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/d4fa33e3
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/d4fa33e3
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/d4fa33e3

Branch: refs/heads/DSL_SQL
Commit: d4fa33e346185395577aa3ce537bfd4a1eb8b4f7
Parents: a7cad60
Author: Thomas Groh 
Authored: Wed Jul 5 14:16:50 2017 -0700
Committer: Tyler Akidau 
Committed: Wed Jul 12 20:00:59 2017 -0700

--
 .../apache/beam/runners/dataflow/DataflowPipelineTranslator.java| 1 +
 1 file changed, 1 insertion(+)
--


http://git-wip-us.apache.org/repos/asf/beam/blob/d4fa33e3/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPipelineTranslator.java
--
diff --git 
a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPipelineTranslator.java
 
b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPipelineTranslator.java
index 28fd1bb..f1783de 100644
--- 
a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPipelineTranslator.java
+++ 
b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPipelineTranslator.java
@@ -793,6 +793,7 @@ public class DataflowPipelineTranslator {
 
context.getPipelineOptions().as(StreamingOptions.class).isStreaming();
 boolean disallowCombinerLifting =
 !windowingStrategy.getWindowFn().isNonMerging()
+|| !windowingStrategy.getWindowFn().assignsToOneWindow()
 || (isStreaming && !transform.fewKeys())
 // TODO: Allow combiner lifting on the non-default 
trigger, as appropriate.
 || !(windowingStrategy.getTrigger() instanceof 
DefaultTrigger);