This is an automated email from the ASF dual-hosted git repository. reuvenlax pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/beam.git.
from 316d667 Merge pull request #5050: Use WorkerBinary override in Go Dataflow add e185d1b Rename RowType -> Schema and move to new schemas package. add c38ca34 Add classes for SchemaRegistry and SchemaProvider. add b72dd7c Add SchemaCoder, some more detail in SchemaProvider/Registery, and FieldAccessDescriptor. add a252dbf Move Schema classes into SDK package. add d110d29 Integrate Schemas into PCollection Coder inference. add 66909ca Add setSchema convenience method. add e22ee16 Fix compilation errors, tests, and CheckStyle. add f6c259d Add package-info.java for new schemas package. add 96737be Refactor Schema: Schema now is a fully-specified based on primitive types. Nested Schemas and arrays are also supported. add cc9a877 Make sure Schema class is serializable. add b5716b8 Add BigEndingShortCoder to encode int16s. add 520bc58 Cleanup types in Beam Schema. Add a metadata field to TypeDescriptor, so that Calcite can distinguish multiple types that all map to the same Beam schema type. add e9f3610 Make all SQL fields nullable for now. add 38c4c97 Add better type validation and conversion. Introduce FloatCoder. add b1053f0 Use Beam's built-in BooleanCoder. add 6e1bb38 Add coder mapping for DECIMAL. add d0a8351 Fix BigEndianShortCoder add e1f8bbf Fix Array inference and capture. add dcf257c Don't use byte[] as the key of a map - Java arrays use object equality. add 202f2ba Add equals and hashCode methods. add 234f297 Remove unused code. add ab9b3b3 Fix handling of null values - Row was dropping them. add 8b603c7 Allow field description to be null. add d0ba633 Fix array verification in Row. add f2bdd55 Move BeamSQL over to use joda time. add d19c966 Fix full outer join test. add abdce7f Add more unit tests for Row, Schema, and RowCoder. Fix a few issues found. add 59af8f4 Revert components that we are not ready to merge yet. These will continue to be developed in another branch. add 2faeb99 Sync ParDoTranslation to master. add c7cd301 Add license header. add d810475 Fix row size estimation. add 81e517b Address code-review comments. add 7ae147e Add RowSqlTypes. add dd9fb94 Make variables final and fix unused imports. add 592b0c4 Fix RowCoderTest. add 2e5de8f Delete bogus test. add f0cd6d7 Fix timezone test. new c2d6744 Merge pull request #4964: [BEAM-3437] Introduce Schema class, and use it in BeamSQL The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: ...DefaultCoderCloudObjectTranslatorRegistrar.java | 2 + .../beam/sdk/coders/BigEndianIntegerCoder.java | 3 +- ...nIntegerCoder.java => BigEndianShortCoder.java} | 36 +- .../org/apache/beam/sdk/coders/CoderRegistry.java | 2 + .../coders/{DoubleCoder.java => FloatCoder.java} | 36 +- .../java/org/apache/beam/sdk/coders/RowCoder.java | 130 ++++-- .../main/java/org/apache/beam/sdk/io/FileIO.java | 4 +- .../main/java/org/apache/beam/sdk/io/TextIO.java | 2 +- .../java/org/apache/beam/sdk/schemas/Schema.java | 491 +++++++++++++++++++++ .../sdk/{io/range => schemas}/package-info.java | 8 +- .../org/apache/beam/sdk/values/PCollection.java | 4 +- .../main/java/org/apache/beam/sdk/values/Row.java | 379 +++++++++++----- .../java/org/apache/beam/sdk/values/RowType.java | 140 ------ .../org/apache/beam/sdk/values/TypeDescriptor.java | 2 +- .../sdk/values/reflect/DefaultRowTypeFactory.java | 87 ++-- .../values/reflect/ReflectionGetterFactory.java | 4 +- .../apache/beam/sdk/values/reflect/RowFactory.java | 16 +- .../beam/sdk/values/reflect/RowTypeFactory.java | 8 +- .../beam/sdk/values/reflect/RowTypeGetters.java | 16 +- .../org/apache/beam/sdk/coders/RowCoderTest.java | 126 ++++++ .../org/apache/beam/sdk/schemas/SchemaTest.java | 175 ++++++++ .../java/org/apache/beam/sdk/values/RowTest.java | 163 +++++-- .../org/apache/beam/sdk/values/RowTypeTest.java | 85 ---- ...toryTest.java => DefaultSchemaFactoryTest.java} | 34 +- .../beam/sdk/values/reflect/RowFactoryTest.java | 4 +- ...TypeGettersTest.java => SchemaGettersTest.java} | 10 +- .../beam/sdk/extensions/sql/BeamSqlTable.java | 4 +- .../apache/beam/sdk/extensions/sql/RowHelper.java | 192 -------- .../apache/beam/sdk/extensions/sql/RowSqlType.java | 131 ------ .../beam/sdk/extensions/sql/RowSqlTypes.java | 175 ++++++++ .../beam/sdk/extensions/sql/SqlRowTypeFactory.java | 90 ---- .../beam/sdk/extensions/sql/SqlTypeCoder.java | 234 ---------- .../beam/sdk/extensions/sql/SqlTypeCoders.java | 90 ---- .../sdk/extensions/sql/example/BeamSqlExample.java | 12 +- .../beam/sdk/extensions/sql/impl/BeamSqlEnv.java | 25 +- .../sql/impl/interpreter/BeamSqlFnExecutor.java | 7 +- .../operator/BeamSqlCastExpression.java | 32 +- .../interpreter/operator/BeamSqlDotExpression.java | 15 +- .../operator/BeamSqlInputRefExpression.java | 2 +- .../interpreter/operator/BeamSqlPrimitive.java | 29 +- .../operator/BeamSqlWindowEndExpression.java | 7 +- .../operator/BeamSqlWindowExpression.java | 6 +- .../operator/BeamSqlWindowStartExpression.java | 7 +- .../date/BeamSqlCurrentDateExpression.java | 4 +- .../date/BeamSqlCurrentTimeExpression.java | 8 +- .../date/BeamSqlCurrentTimestampExpression.java | 4 +- .../operator/date/BeamSqlDateCeilExpression.java | 9 +- .../operator/date/BeamSqlDateFloorExpression.java | 9 +- .../date/BeamSqlDatetimePlusExpression.java | 2 +- .../operator/date/BeamSqlExtractExpression.java | 8 +- .../BeamSqlTimestampMinusIntervalExpression.java | 5 +- .../DatetimeReinterpretConversions.java | 11 +- .../extensions/sql/impl/parser/ParserUtils.java | 7 +- .../sql/impl/rel/BeamAggregationRel.java | 49 +- .../sdk/extensions/sql/impl/rel/BeamFilterRel.java | 2 +- .../extensions/sql/impl/rel/BeamIOSourceRel.java | 2 +- .../sdk/extensions/sql/impl/rel/BeamJoinRel.java | 30 +- .../extensions/sql/impl/rel/BeamProjectRel.java | 4 +- .../sdk/extensions/sql/impl/rel/BeamSortRel.java | 11 +- .../sdk/extensions/sql/impl/rel/BeamValuesRel.java | 14 +- .../extensions/sql/impl/schema/BaseBeamTable.java | 12 +- .../sql/impl/schema/BeamPCollectionTable.java | 10 +- .../extensions/sql/impl/schema/BeamTableUtils.java | 80 ++-- .../impl/transform/BeamAggregationTransforms.java | 106 ++--- .../impl/transform/BeamBuiltinAggregations.java | 10 +- .../sql/impl/transform/BeamJoinTransforms.java | 57 +-- .../sql/impl/transform/BeamSqlProjectFn.java | 15 +- .../sql/impl/utils/BigDecimalConverter.java | 32 +- .../extensions/sql/impl/utils/CalciteUtils.java | 212 +++++---- .../beam/sdk/extensions/sql/meta/Column.java | 8 +- .../extensions/sql/meta/provider/MetaUtils.java | 15 +- .../sql/meta/provider/kafka/BeamKafkaCSVTable.java | 28 +- .../sql/meta/provider/kafka/BeamKafkaTable.java | 14 +- .../meta/provider/kafka/KafkaTableProvider.java | 6 +- .../sql/meta/provider/text/BeamTextCSVTable.java | 14 +- .../provider/text/BeamTextCSVTableIOReader.java | 10 +- .../provider/text/BeamTextCSVTableIOWriter.java | 8 +- .../sql/meta/provider/text/BeamTextTable.java | 6 +- .../sql/meta/provider/text/TextTableProvider.java | 6 +- .../sdk/extensions/sql/BeamSqlApiSurfaceTest.java | 63 --- .../beam/sdk/extensions/sql/BeamSqlArrayTest.java | 162 +++---- .../sql/BeamSqlDslAggregationCovarianceTest.java | 8 +- .../extensions/sql/BeamSqlDslAggregationTest.java | 124 +++--- .../sql/BeamSqlDslAggregationVarianceTest.java | 10 +- .../beam/sdk/extensions/sql/BeamSqlDslBase.java | 37 +- .../sdk/extensions/sql/BeamSqlDslJoinTest.java | 22 +- .../extensions/sql/BeamSqlDslNestedRowsTest.java | 137 +++--- .../sdk/extensions/sql/BeamSqlDslProjectTest.java | 22 +- .../sdk/extensions/sql/BeamSqlDslUdfUdafTest.java | 22 +- ...eFactoryTest.java => SqlSchemaFactoryTest.java} | 47 +- .../apache/beam/sdk/extensions/sql/TestUtils.java | 45 +- .../interpreter/BeamSqlFnExecutorTestBase.java | 2 +- .../operator/BeamSqlCastExpressionTest.java | 32 +- .../operator/BeamSqlDotExpressionTest.java | 16 +- .../operator/BeamSqlInputRefExpressionTest.java | 6 +- .../operator/BeamSqlReinterpretExpressionTest.java | 11 +- .../date/BeamSqlDateExpressionTestBase.java | 29 +- .../date/BeamSqlDatetimeMinusExpressionTest.java | 5 +- .../date/BeamSqlDatetimePlusExpressionTest.java | 19 +- .../date/BeamSqlExtractExpressionTest.java | 4 +- ...eamSqlTimestampMinusIntervalExpressionTest.java | 5 +- ...amSqlTimestampMinusTimestampExpressionTest.java | 19 +- .../DatetimeReinterpretConversionsTest.java | 11 +- .../operator/reinterpret/ReinterpreterTest.java | 4 +- .../row/BeamSqlFieldAccessExpressionTest.java | 10 +- .../sql/impl/parser/BeamSqlParserTest.java | 8 +- .../sql/impl/rel/BeamIntersectRelTest.java | 26 +- .../impl/rel/BeamJoinRelBoundedVsBoundedTest.java | 62 +-- .../rel/BeamJoinRelUnboundedVsBoundedTest.java | 62 +-- .../rel/BeamJoinRelUnboundedVsUnboundedTest.java | 48 +- .../extensions/sql/impl/rel/BeamMinusRelTest.java | 26 +- .../sql/impl/rel/BeamSetOperatorRelBaseTest.java | 20 +- .../extensions/sql/impl/rel/BeamSortRelTest.java | 108 ++--- .../extensions/sql/impl/rel/BeamUnionRelTest.java | 20 +- .../extensions/sql/impl/rel/BeamValuesRelTest.java | 22 +- .../sql/impl/schema/BeamSqlRowCoderTest.java | 18 +- .../transform/BeamAggregationTransformTest.java | 47 +- .../schema/transform/BeamTransformBaseTest.java | 26 +- .../sql/impl/utils/BigDecimalConverterTest.java | 8 +- ...BeamSqlBuiltinFunctionsIntegrationTestBase.java | 61 ++- .../BeamSqlComparisonOperatorsIntegrationTest.java | 6 +- .../BeamSqlDateFunctionsIntegrationTest.java | 14 +- .../meta/provider/kafka/BeamKafkaCSVTableTest.java | 10 +- .../provider/kafka/KafkaTableProviderTest.java | 17 +- .../meta/provider/text/BeamTextCSVTableTest.java | 20 +- .../meta/provider/text/TextTableProviderTest.java | 17 +- .../sql/meta/store/InMemoryMetaStoreTest.java | 23 +- .../extensions/sql/mock/MockedBoundedTable.java | 10 +- .../beam/sdk/extensions/sql/mock/MockedTable.java | 6 +- .../extensions/sql/mock/MockedUnboundedTable.java | 12 +- .../beam/sdk/extensions/sql/utils/RowAsserts.java | 2 +- .../beam/sdk/io/gcp/bigquery/BigQueryIO.java | 2 +- .../beam/sdk/io/gcp/bigtable/BigtableWriteIT.java | 4 +- .../apache/beam/sdk/nexmark/model/sql/RowSize.java | 63 +-- .../apache/beam/sdk/nexmark/model/sql/ToRow.java | 2 +- .../model/sql/adapter/ModelAdaptersMapping.java | 60 +-- .../model/sql/adapter/ModelFieldsAdapter.java | 14 +- .../beam/sdk/nexmark/queries/sql/SqlQuery0.java | 2 +- .../beam/sdk/nexmark/queries/sql/SqlQuery1.java | 2 +- .../beam/sdk/nexmark/queries/sql/SqlQuery2.java | 2 +- .../beam/sdk/nexmark/queries/sql/SqlQuery3.java | 6 +- .../beam/sdk/nexmark/queries/sql/SqlQuery5.java | 2 +- .../beam/sdk/nexmark/queries/sql/SqlQuery7.java | 2 +- .../beam/sdk/nexmark/model/sql/RowSizeTest.java | 19 +- .../beam/sdk/nexmark/model/sql/ToRowTest.java | 2 +- .../sql/adapter/ModelAdaptersMappingTest.java | 32 +- .../sdk/nexmark/queries/sql/SqlQuery5Test.java | 2 +- 147 files changed, 2888 insertions(+), 2752 deletions(-) copy sdks/java/core/src/main/java/org/apache/beam/sdk/coders/{BigEndianIntegerCoder.java => BigEndianShortCoder.java} (65%) copy sdks/java/core/src/main/java/org/apache/beam/sdk/coders/{DoubleCoder.java => FloatCoder.java} (69%) create mode 100644 sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/Schema.java copy sdks/java/core/src/main/java/org/apache/beam/sdk/{io/range => schemas}/package-info.java (78%) delete mode 100644 sdks/java/core/src/main/java/org/apache/beam/sdk/values/RowType.java create mode 100644 sdks/java/core/src/test/java/org/apache/beam/sdk/coders/org/apache/beam/sdk/coders/RowCoderTest.java create mode 100644 sdks/java/core/src/test/java/org/apache/beam/sdk/schemas/SchemaTest.java delete mode 100644 sdks/java/core/src/test/java/org/apache/beam/sdk/values/RowTypeTest.java rename sdks/java/core/src/test/java/org/apache/beam/sdk/values/reflect/{DefaultRowTypeFactoryTest.java => DefaultSchemaFactoryTest.java} (72%) rename sdks/java/core/src/test/java/org/apache/beam/sdk/values/reflect/{RowTypeGettersTest.java => SchemaGettersTest.java} (81%) delete mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/RowHelper.java delete mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/RowSqlType.java create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/RowSqlTypes.java delete mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/SqlRowTypeFactory.java delete mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/SqlTypeCoder.java delete mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/SqlTypeCoders.java delete mode 100644 sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/BeamSqlApiSurfaceTest.java rename sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/{SqlRowTypeFactoryTest.java => SqlSchemaFactoryTest.java} (68%) -- To stop receiving notification emails like this one, please contact reuven...@apache.org.