benWize commented on code in PR #23546:
URL: https://github.com/apache/beam/pull/23546#discussion_r1001078567
##########
sdks/python/apache_beam/examples/complete/distribopt_test.py:
##########
@@ -20,17 +20,38 @@
# pytype: skip-file
import logging
-import os
-import tempfile
import unittest
+import uuid
from ast import literal_eval as make_tuple
import numpy as np
import pytest
from mock import MagicMock
from mock import patch
-from apache_beam.testing.util import open_shards
+from apache_beam.testing.test_pipeline import TestPipeline
+
+# Protect against environments where gcsio library is not available.
+try:
+ from apache_beam.io.gcp import gcsio
+except ImportError:
+ gcsio = None
+
+
+def read_gcs_output_file(file_pattern):
+ gcs = gcsio.GcsIO()
+ file_names = gcs.list_prefix(file_pattern).keys()
+ output = []
+ for file_name in file_names:
+ output.append(gcs.open(file_name).read().decode('utf-8').strip())
+ return '\n'.join(output)
+
+
+def create_content_input_file(path, contents):
+ logging.info('Creating file: %s', path)
+ gcs = gcsio.GcsIO()
+ with gcs.open(path, 'w') as f:
+ f.write(str.encode(contents, 'utf-8'))
Review Comment:
Thanks, @TheNeuralBit, I applied some of your suggestions
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
