kszucs commented on a change in pull request #11136:
URL: https://github.com/apache/arrow/pull/11136#discussion_r713611373
##########
File path: python/pyarrow/tests/test_fs.py
##########
@@ -298,6 +300,99 @@ def subtree_s3fs(request, s3fs):
)
+__minio_limited_policy = """{
+ "Version": "2012-10-17",
+ "Statement": [
+ {
+ "Effect": "Allow",
+ "Action": [
+ "s3:ListAllMyBuckets",
+ "s3:PutObject",
+ "s3:GetObject",
+ "s3:ListBucket",
+ "s3:PutObjectTagging",
+ "s3:DeleteObject",
+ "s3:GetObjectVersion"
+ ],
+ "Resource": [
+ "arn:aws:s3:::*"
+ ]
+ }
+ ]
+}"""
+
+
+def __run_mc_command(mcdir, *args):
+ full_args = ['mc', '-C', mcdir] + list(args)
+ proc = subprocess.Popen(full_args, stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE, encoding='utf-8')
+ retval = proc.wait(10)
+ cmd_str = ' '.join(full_args)
+ print(f'Cmd: {cmd_str}')
+ print(f' Return: {retval}')
+ print(f' Stdout: {proc.stdout.read()}')
+ print(f' Stderr: {proc.stderr.read()}')
+ if retval != 0:
+ raise ChildProcessError("Could not run mc")
+
+
+def __wait_for_minio_startup(mcdir, address, access_key, secret_key):
+ start = time.time()
+ while time.time() - start < 10:
+ try:
+ __run_mc_command(mcdir, 'alias', 'set', 'myminio',
+ f'http://{address}', access_key, secret_key)
+ return
+ except ChildProcessError:
+ time.sleep(1)
+ raise Exception("mc command could not connect to local minio")
+
+
+def __configure_limited_user(tmpdir, address, access_key, secret_key):
+ """
+ Attempts to use the mc command to configure the minio server
+ with a special user limited:limited123 which does not have
+ permission to create buckets. This mirrors some real life S3
+ configurations where users are given strict permissions.
+
+ Arrow S3 operations should still work in such a configuration
+ (e.g. see ARROW-13685)
+ """
+ try:
+ mcdir = os.path.join(tmpdir, 'mc')
+ os.mkdir(mcdir)
+ policy_path = os.path.join(tmpdir, 'limited-buckets-policy.json')
+ with open(policy_path, mode='w') as policy_file:
+ policy_file.write(__minio_limited_policy)
+ # The s3_server fixture starts the minio process but
+ # it takes a few moments for the process to become available
+ __wait_for_minio_startup(mcdir, address, access_key, secret_key)
+ # These commands create a limited user with a specific
+ # policy and creates a sample bucket for that user to
+ # write to
+ __run_mc_command(mcdir, 'admin', 'policy', 'add',
Review comment:
I wonder, could we use the minio python client instead of subprocess
calls to `mc`?
https://docs.min.io/docs/python-client-api-reference.html
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: github-unsubscr...@arrow.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
