This is an automated email from the ASF dual-hosted git repository.
tvalentyn pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new c786cd004b7 [yaml]: uncomment WriteToKafka portion of test (#35277)
c786cd004b7 is described below
commit c786cd004b78c5690613623ab9658ff9febffbca
Author: Derrick Williams <[email protected]>
AuthorDate: Fri Jun 20 12:53:19 2025 -0400
[yaml]: uncomment WriteToKafka portion of test (#35277)
* uncomment kafka test and see if it runs with self hosted runner
* try again but with streaming mode enabled
* comment out the read portion
---
.../beam_PostCommit_Yaml_Xlang_Direct.json | 2 +-
.../yaml/extended_tests/messaging/kafka.yaml | 42 ++++++++++++----------
2 files changed, 24 insertions(+), 20 deletions(-)
diff --git a/.github/trigger_files/beam_PostCommit_Yaml_Xlang_Direct.json
b/.github/trigger_files/beam_PostCommit_Yaml_Xlang_Direct.json
index 8ed972c9f57..9cc78c7d1c6 100644
--- a/.github/trigger_files/beam_PostCommit_Yaml_Xlang_Direct.json
+++ b/.github/trigger_files/beam_PostCommit_Yaml_Xlang_Direct.json
@@ -1,4 +1,4 @@
{
"comment": "Modify this file in a trivial way to cause this test suite to
run",
- "revision": 3
+ "revision": 4
}
diff --git a/sdks/python/apache_beam/yaml/extended_tests/messaging/kafka.yaml
b/sdks/python/apache_beam/yaml/extended_tests/messaging/kafka.yaml
index 790f7e70a5f..5793fd111a6 100644
--- a/sdks/python/apache_beam/yaml/extended_tests/messaging/kafka.yaml
+++ b/sdks/python/apache_beam/yaml/extended_tests/messaging/kafka.yaml
@@ -15,9 +15,9 @@
# limitations under the License.
#
-# fixtures:
-# - name: TEMP_BOOTSTAP_SERVER
-# type: "apache_beam.yaml.integration_tests.temp_kafka_server"
+fixtures:
+ - name: TEMP_BOOTSTAP_SERVER
+ type: "apache_beam.yaml.integration_tests.temp_kafka_server"
pipelines:
# Kafka write pipeline
@@ -30,23 +30,24 @@ pipelines:
- {value: 123}
- {value: 456}
- {value: 789}
-# TODO(#35272) - fix kafka test
- # - type: MapToFields
- # config:
- # language: python
- # fields:
- # value:
- # callable: |
- # lambda row: str(row.value).encode('utf-8')
- # output_type: bytes
- # - type: WriteToKafka
- # config:
- # format: "RAW"
- # topic: "silly_topic"
- # bootstrap_servers: "{TEMP_BOOTSTAP_SERVER}"
- # producer_config_updates:
- # linger.ms: "0"
+ - type: MapToFields
+ config:
+ language: python
+ fields:
+ value:
+ callable: |
+ lambda row: str(row.value).encode('utf-8')
+ output_type: bytes
+ - type: WriteToKafka
+ config:
+ format: "RAW"
+ topic: "silly_topic"
+ bootstrap_servers: "{TEMP_BOOTSTAP_SERVER}"
+ producer_config_updates:
+ linger.ms: "0"
+ # TODO(#35272): Figure out why this times out when running on github.
+ # Locally runs fine.
# Kafka read pipeline
# Need a separate read pipeline to make sure the write pipeline is flushed
# - pipeline:
@@ -77,5 +78,8 @@ pipelines:
# - {value: "456"}
# - {value: "789"}
+ options:
+ streaming: true
+
# TODO: Error handling hard to trigger upon initial investigations. Need to
# investigate more.
