This is an automated email from the ASF dual-hosted git repository.
heejong pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new f012bf9 [BEAM-11843] Fix empty artifact info generation by copy
resource file names from generator
new 820f3cd Merge pull request #14029 from y1chi/BEAM-11843
f012bf9 is described below
commit f012bf9dd3fcb4480600f5b70413b0227c2a280d
Author: Yichi Zhang <[email protected]>
AuthorDate: Fri Feb 19 21:01:04 2021 -0800
[BEAM-11843] Fix empty artifact info generation by copy resource file names
from generator
---
sdks/python/apache_beam/examples/wordcount_it_test.py | 9 ++++++++-
.../apache_beam/runners/portability/sdk_container_builder.py | 8 +++++---
2 files changed, 13 insertions(+), 4 deletions(-)
diff --git a/sdks/python/apache_beam/examples/wordcount_it_test.py
b/sdks/python/apache_beam/examples/wordcount_it_test.py
index 5511ca6..317b5c6 100644
--- a/sdks/python/apache_beam/examples/wordcount_it_test.py
+++ b/sdks/python/apache_beam/examples/wordcount_it_test.py
@@ -57,12 +57,19 @@ class WordCountIT(unittest.TestCase):
self._run_wordcount_it(wordcount.run, experiment='beam_fn_api')
@attr('ValidatesContainer')
- def test_wordcount_it_with_prebuilt_sdk_container(self):
+ def test_wordcount_it_with_prebuilt_sdk_container_local_docker(self):
self._run_wordcount_it(
wordcount.run,
experiment='beam_fn_api',
prebuild_sdk_container_engine='local_docker')
+ @attr('ValidatesContainer')
+ def test_wordcount_it_with_prebuilt_sdk_container_cloud_build(self):
+ self._run_wordcount_it(
+ wordcount.run,
+ experiment='beam_fn_api',
+ prebuild_sdk_container_engine='cloud_build')
+
def _run_wordcount_it(self, run_wordcount, **opts):
test_pipeline = TestPipeline(is_integration_test=True)
extra_opts = {}
diff --git
a/sdks/python/apache_beam/runners/portability/sdk_container_builder.py
b/sdks/python/apache_beam/runners/portability/sdk_container_builder.py
index e769d55..cc54416 100644
--- a/sdks/python/apache_beam/runners/portability/sdk_container_builder.py
+++ b/sdks/python/apache_beam/runners/portability/sdk_container_builder.py
@@ -99,8 +99,10 @@ class SdkContainerImageBuilder(plugin.BeamPlugin):
artifacts = Stager.create_job_resources(self._options, tmp)
resources = Stager.extract_staging_tuple_iter(artifacts)
# make a copy of the staged artifacts into the temp source folder.
+ file_names = []
for path, name in resources:
shutil.copyfile(path, os.path.join(self._temp_src_dir, name))
+ file_names.append(name)
with open(os.path.join(self._temp_src_dir, 'Dockerfile'), 'w') as file:
file.write(
DOCKERFILE_TEMPLATE.format(
@@ -109,7 +111,7 @@ class SdkContainerImageBuilder(plugin.BeamPlugin):
manifest_file=ARTIFACTS_MANIFEST_FILE,
entrypoint=SDK_CONTAINER_ENTRYPOINT))
self._generate_artifacts_manifests_json_file(
- resources, self._temp_src_dir)
+ file_names, self._temp_src_dir)
def _invoke_docker_build_and_push(self, container_image_name):
raise NotImplementedError
@@ -119,9 +121,9 @@ class SdkContainerImageBuilder(plugin.BeamPlugin):
return f'{cls.__module__}.{cls.__name__}'
@staticmethod
- def _generate_artifacts_manifests_json_file(resources, temp_dir):
+ def _generate_artifacts_manifests_json_file(file_names, temp_dir):
infos = []
- for _, name in resources:
+ for name in file_names:
info = beam_runner_api_pb2.ArtifactInformation(
type_urn=common_urns.StandardArtifacts.Types.FILE.urn,
type_payload=beam_runner_api_pb2.ArtifactFilePayload(
