sanjibansg commented on code in PR #12701:
URL: https://github.com/apache/arrow/pull/12701#discussion_r857445075
##########
python/pyarrow/tests/util.py:
##########
@@ -350,3 +353,98 @@ def signal_wakeup_fd(*, warn_on_full_buffer=False):
signal.set_wakeup_fd(old_fd)
r.close()
w.close()
+
+
+def _ensure_minio_component_version(component, minimum_year):
+ full_args = [component, '--version']
+ proc = subprocess.Popen(full_args, stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE, encoding='utf-8')
+ if proc.wait(10) != 0:
+ return False
+ stdout = proc.stdout.read()
+ pattern = component + r' version RELEASE\.(\d+)-.*'
+ version_match = re.search(pattern, stdout)
+ if version_match:
+ version_year = version_match.group(1)
+ return int(version_year) >= minimum_year
+ else:
+ return False
+
+
+def _wait_for_minio_startup(mcdir, address, access_key, secret_key):
+ start = time.time()
+ while time.time() - start < 10:
+ try:
+ _run_mc_command(mcdir, 'alias', 'set', 'myminio',
+ f'http://{address}', access_key, secret_key)
+ return
+ except ChildProcessError:
+ time.sleep(1)
+ raise Exception("mc command could not connect to local minio")
+
+
+def _run_mc_command(mcdir, *args):
+ full_args = ['mc', '-C', mcdir] + list(args)
+ proc = subprocess.Popen(full_args, stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE, encoding='utf-8')
+ retval = proc.wait(10)
+ cmd_str = ' '.join(full_args)
+ print(f'Cmd: {cmd_str}')
+ print(f' Return: {retval}')
+ print(f' Stdout: {proc.stdout.read()}')
+ print(f' Stderr: {proc.stderr.read()}')
+ if retval != 0:
+ raise ChildProcessError("Could not run mc")
+
+
+def _configure_s3_limited_user(s3_server, policy):
+ """
+ Attempts to use the mc command to configure the minio server
+ with a special user limited:limited123 which does not have
+ permission to create buckets. This mirrors some real life S3
+ configurations where users are given strict permissions.
+
+ Arrow S3 operations should still work in such a configuration
+ (e.g. see ARROW-13685)
+ """
+
+ if sys.platform == 'win32':
+ # Can't rely on FileNotFound check because
+ # there is sometimes an mc command on Windows
+ # which is unrelated to the minio mc
+ pytest.skip('The mc command is not installed on Windows')
+ tempdir = s3_server['tempdir']
+ host, port, access_key, secret_key = s3_server['connection']
+ address = '{}:{}'.format(host, port)
+
+ try:
+ if not _ensure_minio_component_version('mc', 2021):
+ # mc version is too old for the capabilities we need
+ return False
+ if not _ensure_minio_component_version('minio', 2021):
+ # minio version is too old for the capabilities we need
+ return False
+ mcdir = os.path.join(tempdir, 'mc')
+ if os.path.exists(mcdir):
+ shutil.rmtree(mcdir)
+ os.mkdir(mcdir)
+ policy_path = os.path.join(tempdir, 'limited-buckets-policy.json')
+ with open(policy_path, mode='w') as policy_file:
+ policy_file.write(policy)
+ # The s3_server fixture starts the minio process but
+ # it takes a few moments for the process to become available
+ _wait_for_minio_startup(mcdir, address, access_key, secret_key)
+ # These commands create a limited user with a specific
Review Comment:
Starting the try block from `_wait_for_minio_startup`. Made the change,
thanks!
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
