This is an automated email from the ASF dual-hosted git repository.
damccorm pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 5733cc8115d Fix flaky BigQuery file loads by safely handling
concurrent mkdirs (#38426)
5733cc8115d is described below
commit 5733cc8115d223457bd5c4c0fe04bfb7d1be89fa
Author: Shunping Huang <[email protected]>
AuthorDate: Mon May 11 08:55:13 2026 -0400
Fix flaky BigQuery file loads by safely handling concurrent mkdirs (#38426)
---
sdks/python/apache_beam/io/gcp/bigquery_file_loads.py | 8 +++++++-
1 file changed, 7 insertions(+), 1 deletion(-)
diff --git a/sdks/python/apache_beam/io/gcp/bigquery_file_loads.py
b/sdks/python/apache_beam/io/gcp/bigquery_file_loads.py
index 738ace67a5f..4e45d0324ee 100644
--- a/sdks/python/apache_beam/io/gcp/bigquery_file_loads.py
+++ b/sdks/python/apache_beam/io/gcp/bigquery_file_loads.py
@@ -134,7 +134,13 @@ def _make_new_file_writer(
directory = fs.FileSystems.join(file_prefix, destination)
if not fs.FileSystems.exists(directory):
- fs.FileSystems.mkdirs(directory)
+ try:
+ fs.FileSystems.mkdirs(directory)
+ except IOError:
+ # Concurrent workers may race to create the same directory.
+ # Ignore the IOError if another worker successfully created it.
+ if not fs.FileSystems.exists(directory):
+ raise
file_name = str(uuid.uuid4())
file_path = fs.FileSystems.join(file_prefix, destination, file_name)
