lostluck commented on a change in pull request #16642: URL: https://github.com/apache/beam/pull/16642#discussion_r794995888
########## File path: sdks/go/pkg/beam/io/xlang/debeziumio/debezium.go ########## @@ -0,0 +1,124 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package debeziumio contains cross-language functionality for using Debezium +// (http://kafka.apache.org/). These transforms only work on runners that +// support cross-language transforms. +// +// Setup +// +// Transforms specified here are cross-language transforms implemented in a +// different SDK (listed below). During pipeline construction, the Go SDK will +// need to connect to an expansion service containing information on these +// transforms in their native SDK. +// +// To use an expansion service, it must be run as a separate process accessible +// during pipeline construction. The address of that process must be passed to +// the transforms in this package. +// +// The version of the expansion service should match the version of the Beam SDK +// being used. For numbered releases of Beam, these expansions services are +// released to the Maven repository as modules. For development versions of +// Beam, it is recommended to build and run it from source using Gradle. +// +// Current supported SDKs including expansion service modules +// * Java +// - Vendored Module: beam-sdks-java-io-debezium-expansion-service +// - Run via Gradle: ./gradlew :sdks:java:io:debezium:expansion-service:shadowJar +// java -jar <path-to-debezium-jar> <port> + +package debeziumio + +import ( + "reflect" + + "github.com/apache/beam/sdks/v2/go/pkg/beam" + "github.com/apache/beam/sdks/v2/go/pkg/beam/core/typex" +) + +// DriverClassName is the type for valid and supported Database connectors for Debezium IO. +type DriverClassName string + +const ( + // MYSQL connector for Debezium + MYSQL DriverClassName = "MySQL" + // POSTGRESQL connector for Debezium + POSTGRESQL = "PostgreSQL" + // ORACLE connector for Debezium + ORACLE = "Oracle" + // DB2 connector for Debezium + DB2 = "Db2" +) + +const readURN = "beam:transform:org.apache.beam:debezium_read:v1" + +// readFromDebeziumSchema is config schema that matches exactly with the Java's Debezium IO +// for cross language payload. +type readFromDebeziumSchema struct { + ConnectorClass string + Username string + Password string + Host string + Port string + MaxNumberOfRecords *int64 + ConnectionProperties []string +} + +// readOption facilitates additional parameters to debeziumio.Read() Ptransform. +type readOption func(*readFromDebeziumSchema) + +// Read is an external PTransform which reads from Debezium and returns a +// JSON string. It requires the address of an expansion service for Debezium IO. +// +// Example: +// expansionAddr := "localhost:9000" +// username := "debezium" +// password := "dbz" +// host := "localhost" +// port := "5432" +// connectorClass := debeziumIO.POSTGRESQL +// maxrecords := 1 +// debeziumio.Read(s.Scope("Read from debezium"), expansionAddr, username, password, host, port, connectorClass, reflectx.String, debeziumio.MaxRecord(maxrecords)) +func Read(s beam.Scope, addr, username, password, host, port string, connectorClass DriverClassName, t reflect.Type, opts ...readOption) beam.PCollection { Review comment: Don't require users to users pass in an expansion address. Instead have it as a read option. This will let us replace the default with Jack's auto starting expansion service work instead, and not be left with an unused parameter. ########## File path: sdks/go/pkg/beam/io/xlang/debeziumio/debezium.go ########## @@ -0,0 +1,124 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package debeziumio contains cross-language functionality for using Debezium +// (http://kafka.apache.org/). These transforms only work on runners that +// support cross-language transforms. +// +// Setup +// +// Transforms specified here are cross-language transforms implemented in a +// different SDK (listed below). During pipeline construction, the Go SDK will +// need to connect to an expansion service containing information on these +// transforms in their native SDK. +// +// To use an expansion service, it must be run as a separate process accessible +// during pipeline construction. The address of that process must be passed to +// the transforms in this package. +// +// The version of the expansion service should match the version of the Beam SDK +// being used. For numbered releases of Beam, these expansions services are +// released to the Maven repository as modules. For development versions of +// Beam, it is recommended to build and run it from source using Gradle. +// +// Current supported SDKs including expansion service modules +// * Java +// - Vendored Module: beam-sdks-java-io-debezium-expansion-service +// - Run via Gradle: ./gradlew :sdks:java:io:debezium:expansion-service:shadowJar +// java -jar <path-to-debezium-jar> <port> + Review comment: rm spare line. This blank line prevents the comment from being used in GoDoc. And From the looks of things I missed this same error in the jdbcio ########## File path: sdks/go/pkg/beam/io/xlang/debeziumio/debezium.go ########## @@ -0,0 +1,124 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package debeziumio contains cross-language functionality for using Debezium +// (http://kafka.apache.org/). These transforms only work on runners that +// support cross-language transforms. +// +// Setup +// +// Transforms specified here are cross-language transforms implemented in a +// different SDK (listed below). During pipeline construction, the Go SDK will +// need to connect to an expansion service containing information on these +// transforms in their native SDK. +// +// To use an expansion service, it must be run as a separate process accessible +// during pipeline construction. The address of that process must be passed to +// the transforms in this package. +// +// The version of the expansion service should match the version of the Beam SDK +// being used. For numbered releases of Beam, these expansions services are +// released to the Maven repository as modules. For development versions of +// Beam, it is recommended to build and run it from source using Gradle. +// +// Current supported SDKs including expansion service modules +// * Java +// - Vendored Module: beam-sdks-java-io-debezium-expansion-service +// - Run via Gradle: ./gradlew :sdks:java:io:debezium:expansion-service:shadowJar +// java -jar <path-to-debezium-jar> <port> + +package debeziumio + +import ( + "reflect" + + "github.com/apache/beam/sdks/v2/go/pkg/beam" + "github.com/apache/beam/sdks/v2/go/pkg/beam/core/typex" +) + +// DriverClassName is the type for valid and supported Database connectors for Debezium IO. +type DriverClassName string + +const ( + // MYSQL connector for Debezium + MYSQL DriverClassName = "MySQL" + // POSTGRESQL connector for Debezium + POSTGRESQL = "PostgreSQL" + // ORACLE connector for Debezium + ORACLE = "Oracle" + // DB2 connector for Debezium + DB2 = "Db2" +) + +const readURN = "beam:transform:org.apache.beam:debezium_read:v1" + +// readFromDebeziumSchema is config schema that matches exactly with the Java's Debezium IO +// for cross language payload. +type readFromDebeziumSchema struct { + ConnectorClass string + Username string + Password string + Host string + Port string + MaxNumberOfRecords *int64 + ConnectionProperties []string +} + +// readOption facilitates additional parameters to debeziumio.Read() Ptransform. +type readOption func(*readFromDebeziumSchema) + +// Read is an external PTransform which reads from Debezium and returns a +// JSON string. It requires the address of an expansion service for Debezium IO. +// +// Example: +// expansionAddr := "localhost:9000" +// username := "debezium" +// password := "dbz" +// host := "localhost" +// port := "5432" +// connectorClass := debeziumIO.POSTGRESQL +// maxrecords := 1 +// debeziumio.Read(s.Scope("Read from debezium"), expansionAddr, username, password, host, port, connectorClass, reflectx.String, debeziumio.MaxRecord(maxrecords)) Review comment: If you'd like these to show up as code in the godoc, they need to be indented slightly further. ########## File path: sdks/go/pkg/beam/io/xlang/debeziumio/debezium.go ########## @@ -0,0 +1,124 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package debeziumio contains cross-language functionality for using Debezium +// (http://kafka.apache.org/). These transforms only work on runners that +// support cross-language transforms. +// +// Setup +// +// Transforms specified here are cross-language transforms implemented in a +// different SDK (listed below). During pipeline construction, the Go SDK will +// need to connect to an expansion service containing information on these +// transforms in their native SDK. +// +// To use an expansion service, it must be run as a separate process accessible +// during pipeline construction. The address of that process must be passed to +// the transforms in this package. +// +// The version of the expansion service should match the version of the Beam SDK +// being used. For numbered releases of Beam, these expansions services are +// released to the Maven repository as modules. For development versions of +// Beam, it is recommended to build and run it from source using Gradle. +// +// Current supported SDKs including expansion service modules +// * Java +// - Vendored Module: beam-sdks-java-io-debezium-expansion-service +// - Run via Gradle: ./gradlew :sdks:java:io:debezium:expansion-service:shadowJar +// java -jar <path-to-debezium-jar> <port> + +package debeziumio + +import ( + "reflect" + + "github.com/apache/beam/sdks/v2/go/pkg/beam" + "github.com/apache/beam/sdks/v2/go/pkg/beam/core/typex" +) + +// DriverClassName is the type for valid and supported Database connectors for Debezium IO. +type DriverClassName string + +const ( + // MYSQL connector for Debezium + MYSQL DriverClassName = "MySQL" + // POSTGRESQL connector for Debezium + POSTGRESQL = "PostgreSQL" + // ORACLE connector for Debezium + ORACLE = "Oracle" + // DB2 connector for Debezium + DB2 = "Db2" +) + +const readURN = "beam:transform:org.apache.beam:debezium_read:v1" + +// readFromDebeziumSchema is config schema that matches exactly with the Java's Debezium IO +// for cross language payload. Review comment: Please include the link to the Java Debezium IO configuration call so it's possible for later people to find, incase this needs updating. ########## File path: sdks/go/pkg/beam/io/xlang/debeziumio/debezium.go ########## @@ -0,0 +1,124 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package debeziumio contains cross-language functionality for using Debezium +// (http://kafka.apache.org/). These transforms only work on runners that +// support cross-language transforms. +// +// Setup +// +// Transforms specified here are cross-language transforms implemented in a +// different SDK (listed below). During pipeline construction, the Go SDK will +// need to connect to an expansion service containing information on these +// transforms in their native SDK. +// +// To use an expansion service, it must be run as a separate process accessible +// during pipeline construction. The address of that process must be passed to +// the transforms in this package. +// +// The version of the expansion service should match the version of the Beam SDK +// being used. For numbered releases of Beam, these expansions services are +// released to the Maven repository as modules. For development versions of +// Beam, it is recommended to build and run it from source using Gradle. +// +// Current supported SDKs including expansion service modules +// * Java +// - Vendored Module: beam-sdks-java-io-debezium-expansion-service +// - Run via Gradle: ./gradlew :sdks:java:io:debezium:expansion-service:shadowJar +// java -jar <path-to-debezium-jar> <port> + +package debeziumio + +import ( + "reflect" + + "github.com/apache/beam/sdks/v2/go/pkg/beam" + "github.com/apache/beam/sdks/v2/go/pkg/beam/core/typex" +) + +// DriverClassName is the type for valid and supported Database connectors for Debezium IO. +type DriverClassName string + +const ( + // MYSQL connector for Debezium + MYSQL DriverClassName = "MySQL" Review comment: Go doesn't mandate that constants are capitalized. Please use the standard capitalization for the identifiers (allowing for the first letter to be Capitalized, so the value is exported of course). So MySQL, PostgreSQL, Oracle, Db2. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
