This is an automated email from the ASF dual-hosted git repository.
jrmccluskey pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 367f46d4014 [Python] Create temporary dataset with a 24 hour ttl.
(#39615)
367f46d4014 is described below
commit 367f46d4014818f10e57dc29f4405a725d228bd4
Author: Ali Ebrahim <[email protected]>
AuthorDate: Mon Aug 10 17:04:30 2026 -0700
[Python] Create temporary dataset with a 24 hour ttl. (#39615)
* Create temporary dataset with a 24 hour ttl.
* Update sdks/python/apache_beam/io/gcp/bigquery_tools.py
Co-authored-by: Jack McCluskey
<[email protected]>
---------
Co-authored-by: Jack McCluskey
<[email protected]>
---
sdks/python/apache_beam/io/gcp/bigquery_tools.py | 8 ++++++++
1 file changed, 8 insertions(+)
diff --git a/sdks/python/apache_beam/io/gcp/bigquery_tools.py
b/sdks/python/apache_beam/io/gcp/bigquery_tools.py
index 8dd58cd55a0..0d62ec5233c 100644
--- a/sdks/python/apache_beam/io/gcp/bigquery_tools.py
+++ b/sdks/python/apache_beam/io/gcp/bigquery_tools.py
@@ -125,6 +125,13 @@ BIGQUERY_TYPE_TO_PYTHON_TYPE = {
"GEOGRAPHY": str,
}
+# Duplicated logic with io/gcp/bigquery_change_history.py
+# Default table expiration for auto-created temp datasets: 24 hours in ms.
+# Tables created in the dataset auto-expire after this duration if not
+# explicitly deleted, acting as a safety net for orphaned temp tables
+# (e.g. pipeline crash before cleanup runs).
+_DEFAULT_TABLE_EXPIRATION_MS = 24 * 60 * 60 * 1000
+
class FileFormat(object):
CSV = 'CSV'
@@ -952,6 +959,7 @@ class BigQueryWrapper(object):
project_id,
self.temp_dataset_id,
location=location,
+ default_table_expiration_ms=_DEFAULT_TABLE_EXPIRATION_MS,
labels=labels,
kms_key=kms_key)
