damccorm commented on code in PR #41059: URL: https://github.com/apache/beam/pull/41059#discussion_r4513558487
########## sdks/python/apache_beam/io/gcp/bigquery.py: ########## @@ -1522,6 +1523,25 @@ def BigQuerySink(*args, validate=False, **kwargs): _KNOWN_TABLES = set() +_PRIMITIVE_SIZEOF_TYPES = (str, int, float, bool, bytes, type(None)) + + +def _fast_row_and_insert_id_size(row_and_insert_id): + if type(row_and_insert_id) is tuple and len(row_and_insert_id) == 2: + row, insert_id = row_and_insert_id + if type(row) is dict and type(insert_id) in _PRIMITIVE_SIZEOF_TYPES: + getsizeof = sys.getsizeof + total = getsizeof(row_and_insert_id) + getsizeof(row) + if insert_id is not None: + total += getsizeof(insert_id) + for k, v in row.items(): + if not (type(k) in _PRIMITIVE_SIZEOF_TYPES and + type(v) in _PRIMITIVE_SIZEOF_TYPES): + return get_deep_size(row_and_insert_id) + total += getsizeof(k) + ( Review Comment: Fixed in 0f74680ebac: `_check_row_size` now falls back to `get_deep_size(row_and_insert_id)` when `row_byte_size >= self._max_insert_payload_size`. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
