tvalentyn commented on code in PR #40699:
URL: https://github.com/apache/beam/pull/40699#discussion_r4407298013
##########
sdks/python/apache_beam/runners/dataflow/internal/apiclient.py:
##########
@@ -626,31 +626,31 @@ def _stage_resources(self, pipeline, options):
type_payload.path,
type_payload.sha256)
remote_name = staged_hashes[type_payload.sha256]
- if is_staged_role:
- # We should not be overriding this, as dep.role_payload.staged_name
- # refers to the desired name on the worker, whereas staged_name
- # refers to its placement in a distributed filesystem.
- # TODO(heejong): Clean this up.
- dep.role_payload =
beam_runner_api_pb2.ArtifactStagingToRolePayload(
- staged_name=remote_name).SerializeToString()
elif type_payload.path and type_payload.path in staged_paths:
_LOGGER.info(
'Found duplicated artifact path: %s (%s)',
type_payload.path,
type_payload.sha256)
remote_name = staged_paths[type_payload.path]
- if is_staged_role:
- # We should not be overriding this, as dep.role_payload.staged_name
- # refers to the desired name on the worker, whereas staged_name
- # refers to its placement in a distributed filesystem.
- # TODO(heejong): Clean this up.
- dep.role_payload =
beam_runner_api_pb2.ArtifactStagingToRolePayload(
- staged_name=remote_name).SerializeToString()
else:
+ if staged_names.get(remote_name,
+ type_payload.sha256) != (type_payload.sha256):
+ # Different content already staged under this name (e.g. each
+ # Python environment stages its own requirements.txt). Stage this
+ # one as <name>-<sha256><ext> instead of overwriting it. The
+ # worker-local name (role_payload.staged_name) is unaffected.
+ stem, ext = os.path.splitext(remote_name)
+ remote_name = '%s-%s%s' % (stem, type_payload.sha256, ext)
Review Comment:
IIUC this (submission side deduplication) is still useful and can be
submitted irrespective of the runtime deduplication decision.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
