ccaominh commented on a change in pull request #8903: S3 input source
URL: https://github.com/apache/incubator-druid/pull/8903#discussion_r349744118
##########
File path:
extensions-core/s3-extensions/src/main/java/org/apache/druid/data/input/s3/S3InputSource.java
##########
@@ -46,43 +47,64 @@
import java.util.stream.Stream;
import java.util.stream.StreamSupport;
-public class S3InputSource extends AbstractInputSource implements
SplittableInputSource<URI>
+public class S3InputSource extends AbstractInputSource implements
SplittableInputSource<CloudObjectLocation>
{
private static final int MAX_LISTING_LENGTH = 1024;
private final ServerSideEncryptingAmazonS3 s3Client;
private final List<URI> uris;
private final List<URI> prefixes;
+ private final List<CloudObjectLocation> objects;
@JsonCreator
public S3InputSource(
@JacksonInject ServerSideEncryptingAmazonS3 s3Client,
@JsonProperty("uris") @Nullable List<URI> uris,
- @JsonProperty("prefixes") @Nullable List<URI> prefixes
+ @JsonProperty("prefixes") @Nullable List<URI> prefixes,
+ @JsonProperty("objects") @Nullable List<CloudObjectLocation> objects
)
{
this.s3Client = Preconditions.checkNotNull(s3Client, "s3Client");
this.uris = uris == null ? new ArrayList<>() : uris;
this.prefixes = prefixes == null ? new ArrayList<>() : prefixes;
- if (!this.uris.isEmpty() && !this.prefixes.isEmpty()) {
- throw new IAE("uris and prefixes cannot be used together");
- }
-
- if (this.uris.isEmpty() && this.prefixes.isEmpty()) {
- throw new IAE("uris or prefixes must be specified");
- }
-
- for (final URI inputURI : this.uris) {
- Preconditions.checkArgument("s3".equals(inputURI.getScheme()), "input
uri scheme == s3 (%s)", inputURI);
+ if (objects != null) {
+ this.objects = objects;
+ if (!this.uris.isEmpty()) {
+ throw new IAE("uris cannot be used with object");
+ }
+ if (!this.prefixes.isEmpty()) {
+ throw new IAE("prefixes cannot be used with object");
+ }
+ } else {
+ this.objects = null;
+ if (!this.uris.isEmpty() && !this.prefixes.isEmpty()) {
+ throw new IAE("uris and prefixes cannot be used together");
+ }
+
+ if (this.uris.isEmpty() && this.prefixes.isEmpty()) {
+ throw new IAE("uris or prefixes must be specified");
+ }
+
+ for (final URI inputURI : this.uris) {
+ Preconditions.checkArgument("s3".equals(inputURI.getScheme()), "input
uri scheme == s3 (%s)", inputURI);
+ }
+
+ for (final URI inputURI : this.prefixes) {
+ Preconditions.checkArgument("s3".equals(inputURI.getScheme()), "input
uri scheme == s3 (%s)", inputURI);
+ }
}
+ }
- for (final URI inputURI : this.prefixes) {
- Preconditions.checkArgument("s3".equals(inputURI.getScheme()), "input
uri scheme == s3 (%s)", inputURI);
- }
+ private S3InputSource(ServerSideEncryptingAmazonS3 s3Client,
CloudObjectLocation inputSplit)
+ {
+ this.s3Client = Preconditions.checkNotNull(s3Client, "s3Client");
+ this.objects = ImmutableList.of(Preconditions.checkNotNull(inputSplit,
"object"));
+ this.uris = new ArrayList<>();
+ this.prefixes = new ArrayList<>();
}
- @JsonProperty
+ @JsonProperty("uris")
Review comment:
Is the `("uris")` needed?
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
