[
https://issues.apache.org/jira/browse/BEAM-8307?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Rahul Patwari updated BEAM-8307:
--------------------------------
Description:
An exception is thrown when SqlTransform is used with JdbcIO.
Schema of the Source Table:
_CREATE TABLE actor (_
_actor_id SMALLINT UNSIGNED NOT NULL AUTO_INCREMENT,_
_first_name VARCHAR(45) NOT NULL,_
_last_name VARCHAR(45) NOT NULL,_
_last_update TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE
CURRENT_TIMESTAMP,_
_PRIMARY KEY (actor_id),_
_KEY idx_actor_last_name (last_name)_
_)ENGINE=InnoDB DEFAULT CHARSET=utf8;_
Many of the Jdbc Types are converted to Logical_Types in Beam.
Refer:
[https://github.com/apache/beam/blob/113461a4275eb28a101e416897ccd607aa51f83f/sdks/java/io/jdbc/src/main/java/org/apache/beam/sdk/io/jdbc/SchemaUtil.java#L102]
The initial observation is that this code:
[https://github.com/apache/beam/blob/113461a4275eb28a101e416897ccd607aa51f83f/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/utils/CalciteUtils.java#L207]
is returning null.
Exception StackTrace:
12:40:18,441 ERROR [stderr] (default task-1)
org.apache.beam.sdk.extensions.sql.impl.ParseException: Unable to parse query
select first_name, count(*) as count_col from PCOLLECTION group by first_name
12:40:18,442 ERROR [stderr] (default task-1) at
org.apache.beam.sdk.extensions.sql.impl.CalciteQueryPlanner.convertToBeamRel(CalciteQueryPlanner.java:165)
12:40:18,442 ERROR [stderr] (default task-1) at
org.apache.beam.sdk.extensions.sql.impl.BeamSqlEnv.parseQuery(BeamSqlEnv.java:103)
12:40:18,442 ERROR [stderr] (default task-1) at
org.apache.beam.sdk.extensions.sql.SqlTransform.expand(SqlTransform.java:124)
12:40:18,443 ERROR [stderr] (default task-1) at
org.apache.beam.sdk.extensions.sql.SqlTransform.expand(SqlTransform.java:82)
12:40:18,443 ERROR [stderr] (default task-1) at
org.apache.beam.sdk.Pipeline.applyInternal(Pipeline.java:539)
12:40:18,443 ERROR [stderr] (default task-1) at
org.apache.beam.sdk.Pipeline.applyTransform(Pipeline.java:473)
12:40:18,443 ERROR [stderr] (default task-1) at
org.apache.beam.sdk.values.PCollectionTuple.apply(PCollectionTuple.java:248)
12:40:18,463 ERROR [stderr] (default task-1) Caused by:
org.apache.beam.repackaged.sql.org.apache.calcite.tools.ValidationException:
java.lang.NullPointerException
12:40:18,464 ERROR [stderr] (default task-1) at
org.apache.beam.repackaged.sql.org.apache.calcite.prepare.PlannerImpl.validate(PlannerImpl.java:190)
12:40:18,464 ERROR [stderr] (default task-1) at
org.apache.beam.sdk.extensions.sql.impl.CalciteQueryPlanner.convertToBeamRel(CalciteQueryPlanner.java:136)
12:40:18,464 ERROR [stderr] (default task-1) ... 100 more
12:40:18,465 ERROR [stderr] (default task-1) Caused by:
java.lang.NullPointerException
12:40:18,465 ERROR [stderr] (default task-1) at
org.apache.beam.repackaged.sql.org.apache.calcite.sql.type.SqlTypeFactoryImpl.createSqlType(SqlTypeFactoryImpl.java:45)
12:40:18,465 ERROR [stderr] (default task-1) at
org.apache.beam.sdk.extensions.sql.impl.utils.CalciteUtils.toRelDataType(CalciteUtils.java:278)
12:40:18,465 ERROR [stderr] (default task-1) at
org.apache.beam.sdk.extensions.sql.impl.utils.CalciteUtils.toRelDataType(CalciteUtils.java:285)
12:40:18,465 ERROR [stderr] (default task-1) at
org.apache.beam.sdk.extensions.sql.impl.utils.CalciteUtils.lambda$toCalciteRowType$0(CalciteUtils.java:260)
12:40:18,466 ERROR [stderr] (default task-1) at
java.base/java.util.stream.Streams$RangeIntSpliterator.forEachRemaining(Streams.java:104)
12:40:18,466 ERROR [stderr] (default task-1) at
java.base/java.util.stream.IntPipeline$Head.forEach(IntPipeline.java:593)
12:40:18,466 ERROR [stderr] (default task-1) at
org.apache.beam.sdk.extensions.sql.impl.utils.CalciteUtils.toCalciteRowType(CalciteUtils.java:257)
12:40:18,466 ERROR [stderr] (default task-1) at
org.apache.beam.sdk.extensions.sql.impl.BeamCalciteTable.getRowType(BeamCalciteTable.java:70)
12:40:18,466 ERROR [stderr] (default task-1) at
org.apache.beam.repackaged.sql.org.apache.calcite.sql.validate.EmptyScope.resolve_(EmptyScope.java:159)
12:40:18,467 ERROR [stderr] (default task-1) at
org.apache.beam.repackaged.sql.org.apache.calcite.sql.validate.EmptyScope.resolveTable(EmptyScope.java:99)
12:40:18,467 ERROR [stderr] (default task-1) at
org.apache.beam.repackaged.sql.org.apache.calcite.sql.validate.DelegatingScope.resolveTable(DelegatingScope.java:203)
12:40:18,467 ERROR [stderr] (default task-1) at
org.apache.beam.repackaged.sql.org.apache.calcite.sql.validate.IdentifierNamespace.resolveImpl(IdentifierNamespace.java:105)
12:40:18,467 ERROR [stderr] (default task-1) at
org.apache.beam.repackaged.sql.org.apache.calcite.sql.validate.IdentifierNamespace.validateImpl(IdentifierNamespace.java:177)
12:40:18,467 ERROR [stderr] (default task-1) at
org.apache.beam.repackaged.sql.org.apache.calcite.sql.validate.AbstractNamespace.validate(AbstractNamespace.java:84)
12:40:18,468 ERROR [stderr] (default task-1) at
org.apache.beam.repackaged.sql.org.apache.calcite.sql.validate.SqlValidatorImpl.validateNamespace(SqlValidatorImpl.java:994)
12:40:18,468 ERROR [stderr] (default task-1) at
org.apache.beam.repackaged.sql.org.apache.calcite.sql.validate.SqlValidatorImpl.validateQuery(SqlValidatorImpl.java:954)
12:40:18,468 ERROR [stderr] (default task-1) at
org.apache.beam.repackaged.sql.org.apache.calcite.sql.validate.SqlValidatorImpl.validateFrom(SqlValidatorImpl.java:3087)
12:40:18,468 ERROR [stderr] (default task-1) at
org.apache.beam.repackaged.sql.org.apache.calcite.sql.validate.SqlValidatorImpl.validateFrom(SqlValidatorImpl.java:3069)
12:40:18,469 ERROR [stderr] (default task-1) at
org.apache.beam.repackaged.sql.org.apache.calcite.sql.validate.SqlValidatorImpl.validateSelect(SqlValidatorImpl.java:3339)
12:40:18,469 ERROR [stderr] (default task-1) at
org.apache.beam.repackaged.sql.org.apache.calcite.sql.validate.SelectNamespace.validateImpl(SelectNamespace.java:60)
12:40:18,469 ERROR [stderr] (default task-1) at
org.apache.beam.repackaged.sql.org.apache.calcite.sql.validate.AbstractNamespace.validate(AbstractNamespace.java:84)
12:40:18,469 ERROR [stderr] (default task-1) at
org.apache.beam.repackaged.sql.org.apache.calcite.sql.validate.SqlValidatorImpl.validateNamespace(SqlValidatorImpl.java:994)
12:40:18,470 ERROR [stderr] (default task-1) at
org.apache.beam.repackaged.sql.org.apache.calcite.sql.validate.SqlValidatorImpl.validateQuery(SqlValidatorImpl.java:954)
12:40:18,470 ERROR [stderr] (default task-1) at
org.apache.beam.repackaged.sql.org.apache.calcite.sql.SqlSelect.validate(SqlSelect.java:216)
12:40:18,470 ERROR [stderr] (default task-1) at
org.apache.beam.repackaged.sql.org.apache.calcite.sql.validate.SqlValidatorImpl.validateScopedExpression(SqlValidatorImpl.java:929)
12:40:18,470 ERROR [stderr] (default task-1) at
org.apache.beam.repackaged.sql.org.apache.calcite.sql.validate.SqlValidatorImpl.validate(SqlValidatorImpl.java:633)
12:40:18,470 ERROR [stderr] (default task-1) at
org.apache.beam.repackaged.sql.org.apache.calcite.prepare.PlannerImpl.validate(PlannerImpl.java:188)
12:40:18,471 ERROR [stderr] (default task-1) ... 101 more
was:
An exception is thrown when SqlTransform is used with JdbcIO. PFA -
[^exception.txt] which contains the Exception StackTrace.
Schema of the Source Table:
_CREATE TABLE actor (_
_actor_id SMALLINT UNSIGNED NOT NULL AUTO_INCREMENT,_
_first_name VARCHAR(45) NOT NULL,_
_last_name VARCHAR(45) NOT NULL,_
_last_update TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE
CURRENT_TIMESTAMP,_
_PRIMARY KEY (actor_id),_
_KEY idx_actor_last_name (last_name)_
_)ENGINE=InnoDB DEFAULT CHARSET=utf8;_
Many of the Jdbc Types are converted to Logical_Types in Beam.
Refer:
[https://github.com/apache/beam/blob/113461a4275eb28a101e416897ccd607aa51f83f/sdks/java/io/jdbc/src/main/java/org/apache/beam/sdk/io/jdbc/SchemaUtil.java#L102]
The initial observation is that this code:
[https://github.com/apache/beam/blob/113461a4275eb28a101e416897ccd607aa51f83f/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/utils/CalciteUtils.java#L207]
is returning null.
> Support for Logical types introduced by JdbcIO Transform in SqlTransform
> ------------------------------------------------------------------------
>
> Key: BEAM-8307
> URL: https://issues.apache.org/jira/browse/BEAM-8307
> Project: Beam
> Issue Type: Bug
> Components: dsl-sql
> Affects Versions: 2.15.0
> Reporter: Rahul Patwari
> Assignee: Rahul Patwari
> Priority: Major
>
> An exception is thrown when SqlTransform is used with JdbcIO.
> Schema of the Source Table:
> _CREATE TABLE actor (_
> _actor_id SMALLINT UNSIGNED NOT NULL AUTO_INCREMENT,_
> _first_name VARCHAR(45) NOT NULL,_
> _last_name VARCHAR(45) NOT NULL,_
> _last_update TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE
> CURRENT_TIMESTAMP,_
> _PRIMARY KEY (actor_id),_
> _KEY idx_actor_last_name (last_name)_
> _)ENGINE=InnoDB DEFAULT CHARSET=utf8;_
>
> Many of the Jdbc Types are converted to Logical_Types in Beam.
> Refer:
> [https://github.com/apache/beam/blob/113461a4275eb28a101e416897ccd607aa51f83f/sdks/java/io/jdbc/src/main/java/org/apache/beam/sdk/io/jdbc/SchemaUtil.java#L102]
> The initial observation is that this code:
> [https://github.com/apache/beam/blob/113461a4275eb28a101e416897ccd607aa51f83f/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/utils/CalciteUtils.java#L207]
> is returning null.
>
> Exception StackTrace:
> 12:40:18,441 ERROR [stderr] (default task-1)
> org.apache.beam.sdk.extensions.sql.impl.ParseException: Unable to parse query
> select first_name, count(*) as count_col from PCOLLECTION group by first_name
> 12:40:18,442 ERROR [stderr] (default task-1) at
> org.apache.beam.sdk.extensions.sql.impl.CalciteQueryPlanner.convertToBeamRel(CalciteQueryPlanner.java:165)
> 12:40:18,442 ERROR [stderr] (default task-1) at
> org.apache.beam.sdk.extensions.sql.impl.BeamSqlEnv.parseQuery(BeamSqlEnv.java:103)
> 12:40:18,442 ERROR [stderr] (default task-1) at
> org.apache.beam.sdk.extensions.sql.SqlTransform.expand(SqlTransform.java:124)
> 12:40:18,443 ERROR [stderr] (default task-1) at
> org.apache.beam.sdk.extensions.sql.SqlTransform.expand(SqlTransform.java:82)
> 12:40:18,443 ERROR [stderr] (default task-1) at
> org.apache.beam.sdk.Pipeline.applyInternal(Pipeline.java:539)
> 12:40:18,443 ERROR [stderr] (default task-1) at
> org.apache.beam.sdk.Pipeline.applyTransform(Pipeline.java:473)
> 12:40:18,443 ERROR [stderr] (default task-1) at
> org.apache.beam.sdk.values.PCollectionTuple.apply(PCollectionTuple.java:248)
> 12:40:18,463 ERROR [stderr] (default task-1) Caused by:
> org.apache.beam.repackaged.sql.org.apache.calcite.tools.ValidationException:
> java.lang.NullPointerException
> 12:40:18,464 ERROR [stderr] (default task-1) at
> org.apache.beam.repackaged.sql.org.apache.calcite.prepare.PlannerImpl.validate(PlannerImpl.java:190)
> 12:40:18,464 ERROR [stderr] (default task-1) at
> org.apache.beam.sdk.extensions.sql.impl.CalciteQueryPlanner.convertToBeamRel(CalciteQueryPlanner.java:136)
> 12:40:18,464 ERROR [stderr] (default task-1) ... 100 more
> 12:40:18,465 ERROR [stderr] (default task-1) Caused by:
> java.lang.NullPointerException
> 12:40:18,465 ERROR [stderr] (default task-1) at
> org.apache.beam.repackaged.sql.org.apache.calcite.sql.type.SqlTypeFactoryImpl.createSqlType(SqlTypeFactoryImpl.java:45)
> 12:40:18,465 ERROR [stderr] (default task-1) at
> org.apache.beam.sdk.extensions.sql.impl.utils.CalciteUtils.toRelDataType(CalciteUtils.java:278)
> 12:40:18,465 ERROR [stderr] (default task-1) at
> org.apache.beam.sdk.extensions.sql.impl.utils.CalciteUtils.toRelDataType(CalciteUtils.java:285)
> 12:40:18,465 ERROR [stderr] (default task-1) at
> org.apache.beam.sdk.extensions.sql.impl.utils.CalciteUtils.lambda$toCalciteRowType$0(CalciteUtils.java:260)
> 12:40:18,466 ERROR [stderr] (default task-1) at
> java.base/java.util.stream.Streams$RangeIntSpliterator.forEachRemaining(Streams.java:104)
> 12:40:18,466 ERROR [stderr] (default task-1) at
> java.base/java.util.stream.IntPipeline$Head.forEach(IntPipeline.java:593)
> 12:40:18,466 ERROR [stderr] (default task-1) at
> org.apache.beam.sdk.extensions.sql.impl.utils.CalciteUtils.toCalciteRowType(CalciteUtils.java:257)
> 12:40:18,466 ERROR [stderr] (default task-1) at
> org.apache.beam.sdk.extensions.sql.impl.BeamCalciteTable.getRowType(BeamCalciteTable.java:70)
> 12:40:18,466 ERROR [stderr] (default task-1) at
> org.apache.beam.repackaged.sql.org.apache.calcite.sql.validate.EmptyScope.resolve_(EmptyScope.java:159)
> 12:40:18,467 ERROR [stderr] (default task-1) at
> org.apache.beam.repackaged.sql.org.apache.calcite.sql.validate.EmptyScope.resolveTable(EmptyScope.java:99)
> 12:40:18,467 ERROR [stderr] (default task-1) at
> org.apache.beam.repackaged.sql.org.apache.calcite.sql.validate.DelegatingScope.resolveTable(DelegatingScope.java:203)
> 12:40:18,467 ERROR [stderr] (default task-1) at
> org.apache.beam.repackaged.sql.org.apache.calcite.sql.validate.IdentifierNamespace.resolveImpl(IdentifierNamespace.java:105)
> 12:40:18,467 ERROR [stderr] (default task-1) at
> org.apache.beam.repackaged.sql.org.apache.calcite.sql.validate.IdentifierNamespace.validateImpl(IdentifierNamespace.java:177)
> 12:40:18,467 ERROR [stderr] (default task-1) at
> org.apache.beam.repackaged.sql.org.apache.calcite.sql.validate.AbstractNamespace.validate(AbstractNamespace.java:84)
> 12:40:18,468 ERROR [stderr] (default task-1) at
> org.apache.beam.repackaged.sql.org.apache.calcite.sql.validate.SqlValidatorImpl.validateNamespace(SqlValidatorImpl.java:994)
> 12:40:18,468 ERROR [stderr] (default task-1) at
> org.apache.beam.repackaged.sql.org.apache.calcite.sql.validate.SqlValidatorImpl.validateQuery(SqlValidatorImpl.java:954)
> 12:40:18,468 ERROR [stderr] (default task-1) at
> org.apache.beam.repackaged.sql.org.apache.calcite.sql.validate.SqlValidatorImpl.validateFrom(SqlValidatorImpl.java:3087)
> 12:40:18,468 ERROR [stderr] (default task-1) at
> org.apache.beam.repackaged.sql.org.apache.calcite.sql.validate.SqlValidatorImpl.validateFrom(SqlValidatorImpl.java:3069)
> 12:40:18,469 ERROR [stderr] (default task-1) at
> org.apache.beam.repackaged.sql.org.apache.calcite.sql.validate.SqlValidatorImpl.validateSelect(SqlValidatorImpl.java:3339)
> 12:40:18,469 ERROR [stderr] (default task-1) at
> org.apache.beam.repackaged.sql.org.apache.calcite.sql.validate.SelectNamespace.validateImpl(SelectNamespace.java:60)
> 12:40:18,469 ERROR [stderr] (default task-1) at
> org.apache.beam.repackaged.sql.org.apache.calcite.sql.validate.AbstractNamespace.validate(AbstractNamespace.java:84)
> 12:40:18,469 ERROR [stderr] (default task-1) at
> org.apache.beam.repackaged.sql.org.apache.calcite.sql.validate.SqlValidatorImpl.validateNamespace(SqlValidatorImpl.java:994)
> 12:40:18,470 ERROR [stderr] (default task-1) at
> org.apache.beam.repackaged.sql.org.apache.calcite.sql.validate.SqlValidatorImpl.validateQuery(SqlValidatorImpl.java:954)
> 12:40:18,470 ERROR [stderr] (default task-1) at
> org.apache.beam.repackaged.sql.org.apache.calcite.sql.SqlSelect.validate(SqlSelect.java:216)
> 12:40:18,470 ERROR [stderr] (default task-1) at
> org.apache.beam.repackaged.sql.org.apache.calcite.sql.validate.SqlValidatorImpl.validateScopedExpression(SqlValidatorImpl.java:929)
> 12:40:18,470 ERROR [stderr] (default task-1) at
> org.apache.beam.repackaged.sql.org.apache.calcite.sql.validate.SqlValidatorImpl.validate(SqlValidatorImpl.java:633)
> 12:40:18,470 ERROR [stderr] (default task-1) at
> org.apache.beam.repackaged.sql.org.apache.calcite.prepare.PlannerImpl.validate(PlannerImpl.java:188)
> 12:40:18,471 ERROR [stderr] (default task-1) ... 101 more
--
This message was sent by Atlassian Jira
(v8.3.4#803005)
