ahmedabu98 commented on code in PR #29834:
URL: https://github.com/apache/beam/pull/29834#discussion_r1486898981
##########
sdks/python/setup.py:
##########
@@ -205,6 +204,51 @@ def generate_protos_first():
err.stderr)
+def generate_external_transform_wrappers():
+ try:
+ sdks_dir = os.path.realpath(
+ os.path.join(os.path.realpath(__file__), '..', '..'))
+ script_exists = os.path.exists(
+ os.path.join(sdks_dir, 'python', 'gen_xlang_wrappers.py'))
+ config_exists = os.path.exists(
+ os.path.join(sdks_dir, 'standard_external_transforms.yaml'))
+ # we need both the script and the standard transforms config file.
+ # at build time, we don't have access to apache_beam to discover and
+ # retrieve external transforms, so the config file has to already exist
+ if not script_exists or not config_exists:
+ generated_transforms_dir = os.path.join(
+ sdks_dir, 'python', 'apache_beam',
+ 'transforms', '_external_transforms')
+
+ # if exists, this directory will have at least its __init__.py file
+ if (not os.path.exists(generated_transforms_dir) or
+ len(os.listdir(generated_transforms_dir)) <= 1):
+ message = 'External transform wrappers have not been generated '
+ if not script_exists:
+ message += 'and the generation script `gen_xlang_wrappers.py`'
+ if not config_exists:
+ message += 'and the standard external transforms config'
+ message += ' could not be found'
+ warnings.warn(message)
+ else:
+ warnings.warn(
+ 'Skipping external transform wrapper generation as they '
+ 'are already generated.')
+ return
+ out = subprocess.run([
Review Comment:
> This will make pretty much every run of setup.py slow, right?
Not anymore, since we decided to decouple the generation steps. `setup.py`
expects the transform config to already exist, so no expansion + discovery is
done here. All it does is generate files based on the existing config.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
