atoomula commented on a change in pull request #1386:
URL: https://github.com/apache/samza/pull/1386#discussion_r457129009
##########
File path:
samza-sql/src/main/java/org/apache/samza/sql/planner/QueryPlanner.java
##########
@@ -110,46 +114,50 @@ private void registerSourceSchemas(SchemaPlus rootSchema)
{
}
public RelRoot plan(String query) {
- try {
- Connection connection = DriverManager.getConnection("jdbc:calcite:");
- CalciteConnection calciteConnection =
connection.unwrap(CalciteConnection.class);
- SchemaPlus rootSchema = calciteConnection.getRootSchema();
- registerSourceSchemas(rootSchema);
-
- List<SamzaSqlScalarFunctionImpl> samzaSqlFunctions = udfMetadata.stream()
- .map(x -> new SamzaSqlScalarFunctionImpl(x))
- .collect(Collectors.toList());
-
- final List<RelTraitDef> traitDefs = new ArrayList<>();
-
- traitDefs.add(ConventionTraitDef.INSTANCE);
- traitDefs.add(RelCollationTraitDef.INSTANCE);
-
- List<SqlOperatorTable> sqlOperatorTables = new ArrayList<>();
- sqlOperatorTables.add(new SamzaSqlOperatorTable());
- sqlOperatorTables.add(new SamzaSqlUdfOperatorTable(samzaSqlFunctions));
-
- // Using lenient so that !=,%,- are allowed.
- FrameworkConfig frameworkConfig = Frameworks.newConfigBuilder()
- .parserConfig(SqlParser.configBuilder()
- .setLex(Lex.JAVA)
- .setConformance(SqlConformanceEnum.LENIENT)
- .setCaseSensitive(false) // Make Udfs case insensitive
- .build())
- .defaultSchema(rootSchema)
- .operatorTable(new ChainedSqlOperatorTable(sqlOperatorTables))
- .sqlToRelConverterConfig(SqlToRelConverter.Config.DEFAULT)
- .traitDefs(traitDefs)
- .context(Contexts.EMPTY_CONTEXT)
- .costFactory(null)
- .build();
- Planner planner = Frameworks.getPlanner(frameworkConfig);
-
+ SchemaPlus rootSchema = CalciteSchema.createRootSchema(true, false).plus();
+ registerSourceSchemas(rootSchema);
+
+ List<SamzaSqlScalarFunctionImpl> samzaSqlFunctions =
+ udfMetadata.stream().map(x -> new
SamzaSqlScalarFunctionImpl(x)).collect(Collectors.toList());
+
+ final List<RelTraitDef> traitDefs = new ArrayList<>();
+
+ traitDefs.add(ConventionTraitDef.INSTANCE);
+ traitDefs.add(RelCollationTraitDef.INSTANCE);
+
+ List<SqlOperatorTable> sqlOperatorTables = new ArrayList<>();
+ sqlOperatorTables.add(new SamzaSqlOperatorTable());
+ sqlOperatorTables.add(new SamzaSqlUdfOperatorTable(samzaSqlFunctions));
+
+ // Using lenient so that !=,%,- are allowed.
+ FrameworkConfig frameworkConfig = Frameworks.newConfigBuilder()
+ .parserConfig(SqlParser.configBuilder()
+ .setLex(Lex.JAVA)
+ .setConformance(SqlConformanceEnum.LENIENT)
+ .setCaseSensitive(false) // Make Udfs case insensitive
+ .build())
+ .defaultSchema(rootSchema)
+ .operatorTable(new ChainedSqlOperatorTable(sqlOperatorTables))
+ .sqlToRelConverterConfig(SqlToRelConverter.Config.DEFAULT)
+ .traitDefs(traitDefs)
+ .context(Contexts.EMPTY_CONTEXT)
+ .programs(
+ Programs.hep(ImmutableList.of(FilterJoinRule.FILTER_ON_JOIN),
true, DefaultRelMetadataProvider.INSTANCE))
+ .build();
+
+ // Planner is a auto closable
+ try (Planner planner = Frameworks.getPlanner(frameworkConfig)) {
SqlNode sql = planner.parse(query);
SqlNode validatedSql = planner.validate(sql);
RelRoot relRoot = planner.rel(validatedSql);
LOG.info("query plan:\n" + RelOptUtil.toString(relRoot.rel,
SqlExplainLevel.ALL_ATTRIBUTES));
- return relRoot;
+ RelTraitSet relTraitSet = RelTraitSet.createEmpty();
Review comment:
Could you merge with the latest code ? I have merged Query optimization
code.
##########
File path:
samza-sql/src/main/java/org/apache/samza/sql/planner/QueryPlanner.java
##########
@@ -110,46 +114,50 @@ private void registerSourceSchemas(SchemaPlus rootSchema)
{
}
public RelRoot plan(String query) {
- try {
- Connection connection = DriverManager.getConnection("jdbc:calcite:");
- CalciteConnection calciteConnection =
connection.unwrap(CalciteConnection.class);
- SchemaPlus rootSchema = calciteConnection.getRootSchema();
- registerSourceSchemas(rootSchema);
-
- List<SamzaSqlScalarFunctionImpl> samzaSqlFunctions = udfMetadata.stream()
- .map(x -> new SamzaSqlScalarFunctionImpl(x))
- .collect(Collectors.toList());
-
- final List<RelTraitDef> traitDefs = new ArrayList<>();
-
- traitDefs.add(ConventionTraitDef.INSTANCE);
- traitDefs.add(RelCollationTraitDef.INSTANCE);
-
- List<SqlOperatorTable> sqlOperatorTables = new ArrayList<>();
- sqlOperatorTables.add(new SamzaSqlOperatorTable());
- sqlOperatorTables.add(new SamzaSqlUdfOperatorTable(samzaSqlFunctions));
-
- // Using lenient so that !=,%,- are allowed.
- FrameworkConfig frameworkConfig = Frameworks.newConfigBuilder()
- .parserConfig(SqlParser.configBuilder()
- .setLex(Lex.JAVA)
- .setConformance(SqlConformanceEnum.LENIENT)
- .setCaseSensitive(false) // Make Udfs case insensitive
- .build())
- .defaultSchema(rootSchema)
- .operatorTable(new ChainedSqlOperatorTable(sqlOperatorTables))
- .sqlToRelConverterConfig(SqlToRelConverter.Config.DEFAULT)
- .traitDefs(traitDefs)
- .context(Contexts.EMPTY_CONTEXT)
- .costFactory(null)
- .build();
- Planner planner = Frameworks.getPlanner(frameworkConfig);
-
+ SchemaPlus rootSchema = CalciteSchema.createRootSchema(true, false).plus();
Review comment:
Just curious, is there any reason to set caching to false ?
##########
File path:
samza-sql/src/main/java/org/apache/samza/sql/avro/AvroRelConverter.java
##########
@@ -194,10 +194,12 @@ public static Object convertToAvroObject(Object relObj,
Schema schema) {
.collect(Collectors.toList());
return avroList;
case MAP:
- return ((Map<String, ?>) relObj).entrySet()
- .stream()
- .collect(Collectors.toMap(Map.Entry::getKey,
- e -> convertToAvroObject(e.getValue(),
getNonNullUnionSchema(schema).getValueType())));
+ // If you ask why not using String and that is because some strings
are Wrapped into org.apache.avro.util.Utf8
+ // TODO looking at the Utf8 code base it is not immutable, having it
as a key is calling for trouble!
+ final Map<Object, Object> outputMap = new HashMap<>();
Review comment:
In what scenarios do you expect the map key to be of Utf8 type ?
Considering that Avro mandates the map key type to be a string, isn't it fair
to expect users to convert Utf8s to string ?
##########
File path: samza-sql/src/main/java/org/apache/samza/sql/udf/GetNestedField.java
##########
@@ -0,0 +1,166 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.samza.sql.udf;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+import java.lang.reflect.Type;
+import java.util.Arrays;
+import java.util.List;
+import org.apache.calcite.adapter.enumerable.CallImplementor;
+import org.apache.calcite.adapter.enumerable.EnumUtils;
+import org.apache.calcite.adapter.enumerable.NullPolicy;
+import org.apache.calcite.adapter.enumerable.RexImpTable;
+import org.apache.calcite.jdbc.JavaTypeFactoryImpl;
+import org.apache.calcite.linq4j.tree.ConstantExpression;
+import org.apache.calcite.linq4j.tree.Expression;
+import org.apache.calcite.linq4j.tree.ExpressionType;
+import org.apache.calcite.linq4j.tree.Expressions;
+import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.rel.type.RelDataTypeFactory;
+import org.apache.calcite.rel.type.RelDataTypeField;
+import org.apache.calcite.schema.Function;
+import org.apache.calcite.schema.FunctionParameter;
+import org.apache.calcite.schema.ImplementableFunction;
+import org.apache.calcite.schema.ScalarFunction;
+import org.apache.calcite.sql.SqlCallBinding;
+import org.apache.calcite.sql.SqlFunction;
+import org.apache.calcite.sql.SqlIdentifier;
+import org.apache.calcite.sql.SqlNode;
+import org.apache.calcite.sql.SqlOperandCountRange;
+import org.apache.calcite.sql.SqlOperatorBinding;
+import org.apache.calcite.sql.parser.SqlParserPos;
+import org.apache.calcite.sql.type.OperandTypes;
+import org.apache.calcite.sql.type.SqlOperandCountRanges;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.apache.calcite.sql.validate.SqlUserDefinedFunction;
+
+import static org.apache.calcite.schema.impl.ReflectiveFunctionBase.builder;
+
+
+/**
+ * Operator to extract nested Rows or Fields form a struct row type using a
dotted path.
+ * The goal of this operator is two-fold.
+ * First it is a temporary fix for
https://issues.apache.org/jira/browse/CALCITE-4065 to extract a row from a row.
+ * Second it will enable smooth backward compatible migration from existing
udf that relies on legacy row format.
Review comment:
Is this truly backward compatible with with the existing GetNestedField
udf ? Does this support all the types that are tested in GetSqlFieldUdf ? Esp
map and array.
https://github.com/apache/samza/blob/dcd4b558a2c702f5b5a320fdb9d0c3fcadabd09b/samza-sql/src/test/java/org/apache/samza/sql/fn/TestGetSqlFieldUdf.java
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
