SteNicholas commented on code in PR #3604:
URL: https://github.com/apache/celeborn/pull/3604#discussion_r2827061809
##########
multipart-uploader/multipart-uploader-s3/src/main/java/org/apache/celeborn/S3MultipartUploadHandler.java:
##########
@@ -43,75 +43,89 @@
import org.slf4j.LoggerFactory;
import org.apache.celeborn.server.common.service.mpu.MultipartUploadHandler;
+import
org.apache.celeborn.server.common.service.mpu.MultipartUploadHandlerSharedState;
public class S3MultipartUploadHandler implements MultipartUploadHandler {
private static final Logger logger =
LoggerFactory.getLogger(S3MultipartUploadHandler.class);
private String uploadId;
+ private final String key;
- private final AmazonS3 s3Client;
+ private final S3MultipartUploadHandlerSharedState sharedState;
- private final String key;
+ public static class S3MultipartUploadHandlerSharedState
+ implements MultipartUploadHandlerSharedState {
+
+ private final AmazonS3 s3Client;
+ private final String bucketName;
+ private final int s3MultiplePartUploadMaxRetries;
+ private final int baseDelay;
+ private final int maxBackoff;
+
+ public S3MultipartUploadHandlerSharedState(
+ FileSystem hadoopFs,
+ String bucketName,
+ Integer s3MultiplePartUploadMaxRetries,
+ Integer baseDelay,
+ Integer maxBackoff)
+ throws IOException, URISyntaxException {
+ this.bucketName = bucketName;
+ this.s3MultiplePartUploadMaxRetries = s3MultiplePartUploadMaxRetries;
+ this.baseDelay = baseDelay;
+ this.maxBackoff = maxBackoff;
+ Configuration conf = hadoopFs.getConf();
+ URI binding = new URI(String.format("s3a://%s", bucketName));
- private final String bucketName;
-
- private final Integer s3MultiplePartUploadMaxRetries;
- private final Integer baseDelay;
- private final Integer maxBackoff;
-
- public S3MultipartUploadHandler(
- FileSystem hadoopFs,
- String bucketName,
- String key,
- Integer s3MultiplePartUploadMaxRetries,
- Integer baseDelay,
- Integer maxBackoff)
- throws IOException, URISyntaxException {
- this.bucketName = bucketName;
- this.s3MultiplePartUploadMaxRetries = s3MultiplePartUploadMaxRetries;
- this.baseDelay = baseDelay;
- this.maxBackoff = maxBackoff;
-
- Configuration conf = hadoopFs.getConf();
- URI binding = new URI(String.format("s3a://%s", bucketName));
-
- RetryPolicy retryPolicy =
- new RetryPolicy(
- PredefinedRetryPolicies.DEFAULT_RETRY_CONDITION,
- new PredefinedBackoffStrategies.SDKDefaultBackoffStrategy(
- baseDelay, baseDelay, maxBackoff),
- s3MultiplePartUploadMaxRetries,
- false);
- ClientConfiguration clientConfig =
- new ClientConfiguration()
- .withRetryPolicy(retryPolicy)
- .withMaxErrorRetry(s3MultiplePartUploadMaxRetries);
- AmazonS3ClientBuilder builder =
- AmazonS3ClientBuilder.standard()
- .withCredentials(getCredentialsProvider(binding, conf))
- .withClientConfiguration(clientConfig);
- // for MinIO
- String endpoint = conf.get("fs.s3a.endpoint");
- if (endpoint != null && !endpoint.isEmpty()) {
- builder =
- builder
- .withEndpointConfiguration(
- new AwsClientBuilder.EndpointConfiguration(
- endpoint, conf.get(Constants.AWS_REGION)))
-
.withPathStyleAccessEnabled(conf.getBoolean("fs.s3a.path.style.access", false));
- } else {
- builder = builder.withRegion(conf.get(Constants.AWS_REGION));
+ RetryPolicy retryPolicy =
+ new RetryPolicy(
+ PredefinedRetryPolicies.DEFAULT_RETRY_CONDITION,
+ new PredefinedBackoffStrategies.SDKDefaultBackoffStrategy(
+ baseDelay, baseDelay, maxBackoff),
+ s3MultiplePartUploadMaxRetries,
+ false);
+ ClientConfiguration clientConfig =
+ new ClientConfiguration()
+ .withRetryPolicy(retryPolicy)
+ .withMaxErrorRetry(s3MultiplePartUploadMaxRetries);
+ AmazonS3ClientBuilder builder =
+ AmazonS3ClientBuilder.standard()
+ .withCredentials(getCredentialsProvider(binding, conf))
+ .withClientConfiguration(clientConfig);
+ // for MinIO
+ String endpoint = conf.get("fs.s3a.endpoint");
Review Comment:
```suggestion
String endpoint = conf.get(Constants.ENDPOINT);
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
