This is an automated email from the ASF dual-hosted git repository.
tballison pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tika.git
The following commit(s) were added to refs/heads/main by this push:
new 7d79e73640 TIKA-5006 follow-up: default S3 credentials provider (ECS
task roles, EKS Pod Identity); one credentials mapping for the S3 fetcher,
emitter and iterator (#3562)
7d79e73640 is described below
commit 7d79e73640eb53125b5b9aaffa754ea37fe0988a
Author: Tim Allison <[email protected]>
AuthorDate: Fri Nov 6 16:27:56 2026 -0500
TIKA-5006 follow-up: default S3 credentials provider (ECS task roles, EKS
Pod Identity); one credentials mapping for the S3 fetcher, emitter and iterator
(#3562)
Co-authored-by: Claude Opus 5.5 <[email protected]>
---
CHANGES.txt | 5 +-
docs/modules/ROOT/pages/pipes/plugins/s3.adoc | 9 +-
.../apache/tika/pipes/emitter/s3/S3Emitter.java | 24 +--
.../tika/pipes/emitter/s3/S3EmitterConfig.java | 13 +-
.../apache/tika/pipes/fetcher/s3/S3Fetcher.java | 24 +--
.../tika/pipes/iterator/s3/S3PipesIterator.java | 28 +--
.../org/apache/tika/pipes/s3/S3Credentials.java | 81 +++++++++
.../tika/pipes/emitter/s3/S3EmitterConfigTest.java | 5 +
.../apache/tika/pipes/s3/S3CredentialsTest.java | 200 +++++++++++++++++++++
9 files changed, 309 insertions(+), 80 deletions(-)
diff --git a/CHANGES.txt b/CHANGES.txt
index cfb61d1e4a..6b61b28444 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -80,8 +80,9 @@ Release 4.3.0 - ???
instead of showing empty boxes or nothing (TIKA-5005).
* The S3 fetcher, emitter and pipes iterator accept credentialsProvider
"web_identity"
- for EKS IAM roles for service accounts (IRSA); the plugin now bundles the
AWS STS
- module it needs (TIKA-5006).
+ for EKS IAM roles for service accounts (IRSA), and "default" for the AWS
SDK default
+ chain, which covers ECS task roles and EKS Pod Identity; the plugin now
bundles the
+ AWS STS module these need (TIKA-5006).
* tika-grpc applies parse-context.timeout-limits and exception-reporting in
the server
process too, so its backstop for a hung worker no longer falls back to
the built-in
diff --git a/docs/modules/ROOT/pages/pipes/plugins/s3.adoc
b/docs/modules/ROOT/pages/pipes/plugins/s3.adoc
index c6b791fa3e..e85d51565b 100644
--- a/docs/modules/ROOT/pages/pipes/plugins/s3.adoc
+++ b/docs/modules/ROOT/pages/pipes/plugins/s3.adoc
@@ -47,11 +47,10 @@ All three components share the same `credentialsProvider`
selector:
* `instance` — uses the IAM role of the EC2 instance the host runs on, read
from the instance metadata service. No additional fields needed.
* `web_identity` — assumes a role with a web identity token, as EKS IAM roles
for service accounts (IRSA) provide. Reads the role from `AWS_ROLE_ARN` and the
token from `AWS_WEB_IDENTITY_TOKEN_FILE`, which EKS sets on the pod. No
additional fields needed.
* `key_secret` — reads `accessKey` and `secretKey` from the config. Avoid
checking these into source control; prefer environment-variable substitution or
one of the other providers.
+* `default` — the AWS SDK default credentials chain: Java system properties,
environment variables, web identity, the shared profile files, container
credentials (ECS task roles, EKS Pod Identity), then the EC2 instance role. The
first source that has credentials wins. Use it for ECS task roles and EKS Pod
Identity, or when the credential source differs between environments. No
additional fields needed.
All three components reject any other value when they are created. If
`credentialsProvider` is unset, the emitter's `validate()` rejects it, but the
fetcher and iterator fall back to `instance`.
-ECS task roles and EKS Pod Identity are not supported — neither `instance` nor
`web_identity` picks them up.
-
[#s3-fetcher]
== S3 Fetcher (`s3-fetcher`)
@@ -82,7 +81,7 @@ include::example$pipes-s3-fetcher.json[]
|`credentialsProvider`
|`instance`
-|One of `profile`, `instance`, `web_identity`, `key_secret`. See
<<credentials>>.
+|One of `profile`, `instance`, `web_identity`, `key_secret`, `default`. See
<<credentials>>.
|`profile` / `accessKey` / `secretKey`
|_conditional_
@@ -144,7 +143,7 @@ include::example$pipes-s3-emitter.json[]
|`credentialsProvider`
|_required_
-|One of `profile`, `instance`, `web_identity`, `key_secret` (validated). See
<<credentials>>.
+|One of `profile`, `instance`, `web_identity`, `key_secret`, `default`
(validated). See <<credentials>>.
|`profile` / `accessKey` / `secretKey`
|_conditional_
@@ -207,7 +206,7 @@ In addition to the required `fetcherId` / `emitterId` (see
xref:pipes/iterators.
|`credentialsProvider`
|`instance`
-|One of `profile`, `instance`, `web_identity`, `key_secret`. See
<<credentials>>.
+|One of `profile`, `instance`, `web_identity`, `key_secret`, `default`. See
<<credentials>>.
|`profile` / `accessKey` / `secretKey` / `endpointConfigurationService`
|_conditional_
diff --git
a/tika-pipes/tika-pipes-plugins/tika-pipes-s3/src/main/java/org/apache/tika/pipes/emitter/s3/S3Emitter.java
b/tika-pipes/tika-pipes-plugins/tika-pipes-s3/src/main/java/org/apache/tika/pipes/emitter/s3/S3Emitter.java
index 8ceadde02e..ccd441aba0 100644
---
a/tika-pipes/tika-pipes-plugins/tika-pipes-s3/src/main/java/org/apache/tika/pipes/emitter/s3/S3Emitter.java
+++
b/tika-pipes/tika-pipes-plugins/tika-pipes-s3/src/main/java/org/apache/tika/pipes/emitter/s3/S3Emitter.java
@@ -34,12 +34,7 @@ import java.util.Map;
import org.apache.commons.io.output.UnsynchronizedByteArrayOutputStream;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
-import
software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider;
-import software.amazon.awssdk.auth.credentials.ProfileCredentialsProvider;
-import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
-import
software.amazon.awssdk.auth.credentials.WebIdentityTokenFileCredentialsProvider;
import software.amazon.awssdk.core.checksums.RequestChecksumCalculation;
import software.amazon.awssdk.core.exception.SdkClientException;
import software.amazon.awssdk.core.sync.RequestBody;
@@ -58,6 +53,7 @@ import org.apache.tika.io.TikaInputStream;
import org.apache.tika.metadata.Metadata;
import org.apache.tika.parser.ParseContext;
import org.apache.tika.pipes.api.emitter.AbstractStreamEmitter;
+import org.apache.tika.pipes.s3.S3Credentials;
import org.apache.tika.plugins.ExtensionConfig;
import org.apache.tika.serialization.JsonMetadataList;
import org.apache.tika.utils.StringUtils;
@@ -138,22 +134,8 @@ public class S3Emitter extends AbstractStreamEmitter {
}
private static AwsCredentialsProvider
buildCredentialsProvider(S3EmitterConfig config) throws TikaConfigException {
- switch (config.credentialsProvider()) {
- case "instance":
- return InstanceProfileCredentialsProvider.builder().build();
- case "web_identity":
- return
WebIdentityTokenFileCredentialsProvider.builder().build();
- case "profile":
- return ProfileCredentialsProvider.builder()
- .profileName(config.profile())
- .build();
- case "key_secret":
- AwsBasicCredentials awsCreds = AwsBasicCredentials.create(
- config.accessKey(), config.secretKey());
- return StaticCredentialsProvider.create(awsCreds);
- default:
- throw new TikaConfigException("credentialsProvider must be
'instance', 'web_identity', 'profile' or 'key_secret'");
- }
+ return S3Credentials.build(config.credentialsProvider(),
config.profile(), config.accessKey(),
+ config.secretKey());
}
@Override
diff --git
a/tika-pipes/tika-pipes-plugins/tika-pipes-s3/src/main/java/org/apache/tika/pipes/emitter/s3/S3EmitterConfig.java
b/tika-pipes/tika-pipes-plugins/tika-pipes-s3/src/main/java/org/apache/tika/pipes/emitter/s3/S3EmitterConfig.java
index 0301abdf30..bff637ec97 100644
---
a/tika-pipes/tika-pipes-plugins/tika-pipes-s3/src/main/java/org/apache/tika/pipes/emitter/s3/S3EmitterConfig.java
+++
b/tika-pipes/tika-pipes-plugins/tika-pipes-s3/src/main/java/org/apache/tika/pipes/emitter/s3/S3EmitterConfig.java
@@ -17,6 +17,7 @@
package org.apache.tika.pipes.emitter.s3;
import org.apache.tika.exception.TikaConfigException;
+import org.apache.tika.pipes.s3.S3Credentials;
import org.apache.tika.plugins.PluginJson;
public record S3EmitterConfig(
@@ -67,15 +68,11 @@ public record S3EmitterConfig(
throw new TikaConfigException("'region' must not be empty");
}
if (credentialsProvider == null || credentialsProvider.isBlank()) {
- throw new TikaConfigException("'credentialsProvider' must be set
to 'profile', 'instance', 'web_identity' or 'key_secret'");
+ throw new TikaConfigException("'credentialsProvider' must be set
to " + S3Credentials.describeNames());
}
- if (!credentialsProvider.equals("profile")
- && !credentialsProvider.equals("instance")
- && !credentialsProvider.equals("web_identity")
- && !credentialsProvider.equals("key_secret")) {
- throw new TikaConfigException(
- "credentialsProvider must be 'profile', 'instance',
'web_identity' or 'key_secret', but was: "
- + credentialsProvider);
+ if (!S3Credentials.isKnown(credentialsProvider)) {
+ throw new TikaConfigException("credentialsProvider must be " +
S3Credentials.describeNames()
+ + ", but was: " + credentialsProvider);
}
if (credentialsProvider.equals("profile") && (profile == null ||
profile.isBlank())) {
throw new TikaConfigException("'profile' must be set when
credentialsProvider is 'profile'");
diff --git
a/tika-pipes/tika-pipes-plugins/tika-pipes-s3/src/main/java/org/apache/tika/pipes/fetcher/s3/S3Fetcher.java
b/tika-pipes/tika-pipes-plugins/tika-pipes-s3/src/main/java/org/apache/tika/pipes/fetcher/s3/S3Fetcher.java
index 2d2a175b13..d0aec762fa 100644
---
a/tika-pipes/tika-pipes-plugins/tika-pipes-s3/src/main/java/org/apache/tika/pipes/fetcher/s3/S3Fetcher.java
+++
b/tika-pipes/tika-pipes-plugins/tika-pipes-s3/src/main/java/org/apache/tika/pipes/fetcher/s3/S3Fetcher.java
@@ -29,12 +29,7 @@ import java.util.Set;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
-import
software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider;
-import software.amazon.awssdk.auth.credentials.ProfileCredentialsProvider;
-import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
-import
software.amazon.awssdk.auth.credentials.WebIdentityTokenFileCredentialsProvider;
import software.amazon.awssdk.awscore.exception.AwsServiceException;
import software.amazon.awssdk.core.ResponseInputStream;
import software.amazon.awssdk.core.exception.SdkClientException;
@@ -61,6 +56,7 @@ import org.apache.tika.parser.ParseContext;
import org.apache.tika.pipes.api.fetcher.Fetcher;
import org.apache.tika.pipes.api.fetcher.RangeFetcher;
import org.apache.tika.pipes.fetcher.s3.config.S3FetcherConfig;
+import org.apache.tika.pipes.s3.S3Credentials;
import org.apache.tika.plugins.AbstractTikaExtension;
import org.apache.tika.plugins.ExtensionConfig;
import org.apache.tika.utils.StringUtils;
@@ -139,22 +135,10 @@ public class S3Fetcher extends AbstractTikaExtension
implements Fetcher, RangeFe
static AwsCredentialsProvider buildCredentialsProvider(S3FetcherConfig
config) throws TikaConfigException {
String credentialsProvider = config.getCredentialsProvider();
if (credentialsProvider == null) {
- credentialsProvider = "instance";
- }
- switch (credentialsProvider) {
- case "instance":
- return InstanceProfileCredentialsProvider.builder().build();
- case "web_identity":
- return
WebIdentityTokenFileCredentialsProvider.builder().build();
- case "profile":
- return
ProfileCredentialsProvider.builder().profileName(config.getProfile()).build();
- case "key_secret":
- AwsBasicCredentials awsCreds =
AwsBasicCredentials.create(config.getAccessKey(), config.getSecretKey());
- return StaticCredentialsProvider.create(awsCreds);
- default:
- throw new TikaConfigException("credentialsProvider must be set
and must be either "
- + "'instance', 'web_identity', 'profile' or
'key_secret'");
+ credentialsProvider = S3Credentials.INSTANCE;
}
+ return S3Credentials.build(credentialsProvider, config.getProfile(),
config.getAccessKey(),
+ config.getSecretKey());
}
@Override
diff --git
a/tika-pipes/tika-pipes-plugins/tika-pipes-s3/src/main/java/org/apache/tika/pipes/iterator/s3/S3PipesIterator.java
b/tika-pipes/tika-pipes-plugins/tika-pipes-s3/src/main/java/org/apache/tika/pipes/iterator/s3/S3PipesIterator.java
index 55507fee39..ab247fb69b 100644
---
a/tika-pipes/tika-pipes-plugins/tika-pipes-s3/src/main/java/org/apache/tika/pipes/iterator/s3/S3PipesIterator.java
+++
b/tika-pipes/tika-pipes-plugins/tika-pipes-s3/src/main/java/org/apache/tika/pipes/iterator/s3/S3PipesIterator.java
@@ -26,12 +26,7 @@ import java.util.regex.Pattern;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
-import
software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider;
-import software.amazon.awssdk.auth.credentials.ProfileCredentialsProvider;
-import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
-import
software.amazon.awssdk.auth.credentials.WebIdentityTokenFileCredentialsProvider;
import software.amazon.awssdk.core.exception.SdkClientException;
import software.amazon.awssdk.http.SdkHttpClient;
import software.amazon.awssdk.http.apache.ApacheHttpClient;
@@ -50,6 +45,7 @@ import org.apache.tika.pipes.api.FetchEmitTuple;
import org.apache.tika.pipes.api.emitter.EmitKey;
import org.apache.tika.pipes.api.fetcher.FetchKey;
import org.apache.tika.pipes.pipesiterator.PipesIteratorBase;
+import org.apache.tika.pipes.s3.S3Credentials;
import org.apache.tika.plugins.ExtensionConfig;
import org.apache.tika.utils.StringUtils;
@@ -78,26 +74,10 @@ public class S3PipesIterator extends PipesIteratorBase {
// Initialize S3 client
String credentialsProvider = config.getCredentialsProvider();
if (credentialsProvider == null) {
- credentialsProvider = "instance";
- }
- AwsCredentialsProvider provider;
- switch (credentialsProvider) {
- case "instance":
- provider =
InstanceProfileCredentialsProvider.builder().build();
- break;
- case "web_identity":
- provider =
WebIdentityTokenFileCredentialsProvider.builder().build();
- break;
- case "profile":
- provider =
ProfileCredentialsProvider.builder().profileName(config.getProfile()).build();
- break;
- case "key_secret":
- AwsBasicCredentials awsCreds =
AwsBasicCredentials.create(config.getAccessKey(), config.getSecretKey());
- provider = StaticCredentialsProvider.create(awsCreds);
- break;
- default:
- throw new TikaConfigException("credentialsProvider must be set
and must be either 'instance', 'web_identity', 'profile' or 'key_secret'");
+ credentialsProvider = S3Credentials.INSTANCE;
}
+ AwsCredentialsProvider provider =
S3Credentials.build(credentialsProvider, config.getProfile(),
+ config.getAccessKey(), config.getSecretKey());
SdkHttpClient httpClient =
ApacheHttpClient.builder().maxConnections(config.getMaxConnections()).build();
S3Configuration clientConfig =
S3Configuration.builder().pathStyleAccessEnabled(config.isPathStyleAccessEnabled()).build();
diff --git
a/tika-pipes/tika-pipes-plugins/tika-pipes-s3/src/main/java/org/apache/tika/pipes/s3/S3Credentials.java
b/tika-pipes/tika-pipes-plugins/tika-pipes-s3/src/main/java/org/apache/tika/pipes/s3/S3Credentials.java
new file mode 100644
index 0000000000..12a99a9ce2
--- /dev/null
+++
b/tika-pipes/tika-pipes-plugins/tika-pipes-s3/src/main/java/org/apache/tika/pipes/s3/S3Credentials.java
@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.pipes.s3;
+
+import java.util.List;
+
+import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
+import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
+import software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider;
+import
software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider;
+import software.amazon.awssdk.auth.credentials.ProfileCredentialsProvider;
+import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
+import
software.amazon.awssdk.auth.credentials.WebIdentityTokenFileCredentialsProvider;
+
+import org.apache.tika.exception.TikaConfigException;
+
+/**
+ * Maps the {@code credentialsProvider} setting shared by the S3 fetcher,
emitter and
+ * pipes iterator to an AWS SDK credentials provider.
+ */
+public final class S3Credentials {
+
+ public static final String INSTANCE = "instance";
+ public static final String WEB_IDENTITY = "web_identity";
+ public static final String PROFILE = "profile";
+ public static final String KEY_SECRET = "key_secret";
+ public static final String DEFAULT = "default";
+
+ public static final List<String> NAMES = List.of(INSTANCE, WEB_IDENTITY,
PROFILE, KEY_SECRET, DEFAULT);
+
+ private static final String DESCRIBE_NAMES =
+ "'instance', 'web_identity', 'profile', 'key_secret' or 'default'";
+
+ private S3Credentials() {
+ }
+
+ public static boolean isKnown(String credentialsProvider) {
+ return NAMES.contains(credentialsProvider);
+ }
+
+ public static String describeNames() {
+ return DESCRIBE_NAMES;
+ }
+
+ public static AwsCredentialsProvider build(String credentialsProvider,
String profile,
+ String accessKey, String
secretKey)
+ throws TikaConfigException {
+ if (credentialsProvider == null) {
+ throw new TikaConfigException("credentialsProvider must be set to
" + DESCRIBE_NAMES);
+ }
+ switch (credentialsProvider) {
+ case INSTANCE:
+ return InstanceProfileCredentialsProvider.builder().build();
+ case WEB_IDENTITY:
+ return
WebIdentityTokenFileCredentialsProvider.builder().build();
+ case PROFILE:
+ return
ProfileCredentialsProvider.builder().profileName(profile).build();
+ case KEY_SECRET:
+ return
StaticCredentialsProvider.create(AwsBasicCredentials.create(accessKey,
secretKey));
+ case DEFAULT:
+ return DefaultCredentialsProvider.builder().build();
+ default:
+ throw new TikaConfigException("credentialsProvider must be " +
DESCRIBE_NAMES
+ + ", but was: " + credentialsProvider);
+ }
+ }
+}
diff --git
a/tika-pipes/tika-pipes-plugins/tika-pipes-s3/src/test/java/org/apache/tika/pipes/emitter/s3/S3EmitterConfigTest.java
b/tika-pipes/tika-pipes-plugins/tika-pipes-s3/src/test/java/org/apache/tika/pipes/emitter/s3/S3EmitterConfigTest.java
index cc91d27f9d..2a7c1681d6 100644
---
a/tika-pipes/tika-pipes-plugins/tika-pipes-s3/src/test/java/org/apache/tika/pipes/emitter/s3/S3EmitterConfigTest.java
+++
b/tika-pipes/tika-pipes-plugins/tika-pipes-s3/src/test/java/org/apache/tika/pipes/emitter/s3/S3EmitterConfigTest.java
@@ -32,6 +32,11 @@ public class S3EmitterConfigTest {
assertDoesNotThrow(config::validate);
}
+ @Test
+ public void testDefaultIsValid() throws Exception {
+ assertDoesNotThrow(load("default")::validate);
+ }
+
@Test
public void testUnknownProviderIsInvalid() throws Exception {
S3EmitterConfig config = load("bogus");
diff --git
a/tika-pipes/tika-pipes-plugins/tika-pipes-s3/src/test/java/org/apache/tika/pipes/s3/S3CredentialsTest.java
b/tika-pipes/tika-pipes-plugins/tika-pipes-s3/src/test/java/org/apache/tika/pipes/s3/S3CredentialsTest.java
new file mode 100644
index 0000000000..bcbbca8a9d
--- /dev/null
+++
b/tika-pipes/tika-pipes-plugins/tika-pipes-s3/src/test/java/org/apache/tika/pipes/s3/S3CredentialsTest.java
@@ -0,0 +1,200 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.pipes.s3;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertInstanceOf;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.junit.jupiter.api.Assumptions.assumeTrue;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.InetAddress;
+import java.net.InetSocketAddress;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.List;
+import java.util.concurrent.CopyOnWriteArrayList;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.sun.net.httpserver.HttpExchange;
+import com.sun.net.httpserver.HttpServer;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
+import software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider;
+import
software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider;
+import software.amazon.awssdk.auth.credentials.ProfileCredentialsProvider;
+import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
+import
software.amazon.awssdk.auth.credentials.WebIdentityTokenFileCredentialsProvider;
+
+import org.apache.tika.exception.TikaConfigException;
+import org.apache.tika.io.TikaInputStream;
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.parser.ParseContext;
+import org.apache.tika.pipes.fetcher.s3.S3Fetcher;
+import org.apache.tika.plugins.ExtensionConfig;
+import org.apache.tika.utils.SuppressForbidden;
+
+// jdk.httpserver stands in for STS, the container credentials endpoint and S3
+@SuppressForbidden
+public class S3CredentialsTest {
+
+ private static final String ISSUED_KEY_ID = "ASIAISSUEDKEY";
+ private static final String ROLE_ARN =
"arn:aws:iam::123456789012:role/tika";
+ private static final String CONTAINER_PATH = "/v1/credentials";
+ private static final String CONTAINER_TOKEN = "pod-identity-token";
+ private static final List<String> SYS_PROPS =
List.of("aws.webIdentityTokenFile", "aws.roleArn",
+ "aws.region", "aws.endpointUrlSts",
"aws.containerCredentialsFullUri",
+ "aws.containerAuthorizationTokenFile",
"aws.sharedCredentialsFile", "aws.configFile");
+
+ private HttpServer server;
+ private String endpoint;
+ private final List<String> stsBodies = new CopyOnWriteArrayList<>();
+ private final List<String> containerAuthorizations = new
CopyOnWriteArrayList<>();
+ private final List<String> s3Authorizations = new CopyOnWriteArrayList<>();
+
+ @BeforeEach
+ public void setUp(@TempDir Path tmp) throws IOException {
+ server = HttpServer.create(new
InetSocketAddress(InetAddress.getLoopbackAddress(), 0), 0);
+ server.createContext("/", this::handle);
+ server.start();
+ endpoint = "http://127.0.0.1:"; + server.getAddress().getPort();
+ // keep the default chain off the developer's own ~/.aws files
+ System.setProperty("aws.sharedCredentialsFile",
tmp.resolve("no-credentials").toString());
+ System.setProperty("aws.configFile",
tmp.resolve("no-config").toString());
+ }
+
+ @AfterEach
+ public void tearDown() {
+ server.stop(0);
+ SYS_PROPS.forEach(System::clearProperty);
+ }
+
+ @Test
+ public void testProviderPerName() throws Exception {
+ assertInstanceOf(InstanceProfileCredentialsProvider.class,
build(S3Credentials.INSTANCE));
+ assertInstanceOf(WebIdentityTokenFileCredentialsProvider.class,
build(S3Credentials.WEB_IDENTITY));
+ assertInstanceOf(ProfileCredentialsProvider.class,
build(S3Credentials.PROFILE));
+ assertInstanceOf(StaticCredentialsProvider.class,
build(S3Credentials.KEY_SECRET));
+ assertInstanceOf(DefaultCredentialsProvider.class,
build(S3Credentials.DEFAULT));
+ }
+
+ @Test
+ public void testUnknownProvider() {
+ TikaConfigException e = assertThrows(TikaConfigException.class, () ->
build("irsa"));
+ assertTrue(e.getMessage().contains("'default'"), e.getMessage());
+ assertThrows(TikaConfigException.class, () -> build(null));
+ }
+
+ @ParameterizedTest
+ @ValueSource(strings = {S3Credentials.WEB_IDENTITY, S3Credentials.DEFAULT})
+ public void testFetcherSignsWithWebIdentityCredentials(String
credentialsProvider,
+ @TempDir Path tmp)
throws Exception {
+ if (S3Credentials.DEFAULT.equals(credentialsProvider)) {
+ // env credentials come before web identity in the default chain
+ assumeTrue(System.getenv("AWS_ACCESS_KEY_ID") == null);
+ }
+ Path token = tmp.resolve("token");
+ Files.writeString(token, "eyJ.test.token", StandardCharsets.UTF_8);
+ System.setProperty("aws.webIdentityTokenFile", token.toString());
+ System.setProperty("aws.roleArn", ROLE_ARN);
+ System.setProperty("aws.region", "us-east-1");
+ System.setProperty("aws.endpointUrlSts", endpoint);
+
+ assertFetch(credentialsProvider);
+ assertEquals(1, stsBodies.size());
+
assertTrue(stsBodies.get(0).contains("Action=AssumeRoleWithWebIdentity"),
stsBodies.get(0));
+
assertTrue(stsBodies.get(0).contains("WebIdentityToken=eyJ.test.token"),
stsBodies.get(0));
+ }
+
+ @Test
+ public void testDefaultSignsWithContainerCredentials(@TempDir Path tmp)
throws Exception {
+ // EKS Pod Identity and ECS task roles: an earlier link in the chain
would win
+ assumeTrue(System.getenv("AWS_ACCESS_KEY_ID") == null);
+ assumeTrue(System.getenv("AWS_WEB_IDENTITY_TOKEN_FILE") == null);
+ assumeTrue(System.getenv("AWS_PROFILE") == null);
+ Path token = tmp.resolve("token");
+ Files.writeString(token, CONTAINER_TOKEN, StandardCharsets.UTF_8);
+ System.setProperty("aws.containerCredentialsFullUri", endpoint +
CONTAINER_PATH);
+ System.setProperty("aws.containerAuthorizationTokenFile",
token.toString());
+
+ assertFetch(S3Credentials.DEFAULT);
+ assertEquals(List.of(CONTAINER_TOKEN), containerAuthorizations);
+ assertTrue(stsBodies.isEmpty());
+ }
+
+ private void assertFetch(String credentialsProvider) throws Exception {
+ ObjectNode json = new ObjectMapper().createObjectNode();
+ json.put("bucket", "bucket");
+ json.put("region", "us-east-1");
+ json.put("credentialsProvider", credentialsProvider);
+ json.put("endpointConfigurationService", endpoint);
+ json.put("pathStyleAccessEnabled", true);
+ json.put("spoolToTemp", false);
+ S3Fetcher fetcher = S3Fetcher.build(new ExtensionConfig("s3f",
"s3-fetcher", json.toString()));
+
+ try (TikaInputStream tis = fetcher.fetch("key", new Metadata(), new
ParseContext())) {
+ assertEquals("hello", new String(tis.readAllBytes(),
StandardCharsets.UTF_8));
+ }
+ assertEquals(1, s3Authorizations.size());
+ assertTrue(s3Authorizations.get(0).contains("Credential=" +
ISSUED_KEY_ID + "/"),
+ s3Authorizations.get(0));
+ }
+
+ private static Object build(String name) throws TikaConfigException {
+ return S3Credentials.build(name, "my-profile", "access", "secret");
+ }
+
+ private void handle(HttpExchange exchange) throws IOException {
+ byte[] body;
+ if ("POST".equals(exchange.getRequestMethod())) {
+ try (InputStream is = exchange.getRequestBody()) {
+ stsBodies.add(new String(is.readAllBytes(),
StandardCharsets.UTF_8));
+ }
+ body = ("<AssumeRoleWithWebIdentityResponse
xmlns=\"https://sts.amazonaws.com/doc/2011-06-15/\";>"
+ + "<AssumeRoleWithWebIdentityResult><Credentials>"
+ + "<AccessKeyId>" + ISSUED_KEY_ID + "</AccessKeyId>"
+ + "<SecretAccessKey>secret</SecretAccessKey>"
+ + "<SessionToken>session</SessionToken>"
+ + "<Expiration>2099-01-01T00:00:00Z</Expiration>"
+ + "</Credentials></AssumeRoleWithWebIdentityResult>"
+ +
"</AssumeRoleWithWebIdentityResponse>").getBytes(StandardCharsets.UTF_8);
+ exchange.getResponseHeaders().add("Content-Type", "text/xml");
+ } else if (CONTAINER_PATH.equals(exchange.getRequestURI().getPath())) {
+
containerAuthorizations.add(exchange.getRequestHeaders().getFirst("Authorization"));
+ body = ("{\"AccessKeyId\":\"" + ISSUED_KEY_ID +
"\",\"SecretAccessKey\":\"secret\","
+ +
"\"Token\":\"session\",\"Expiration\":\"2099-01-01T00:00:00Z\"}")
+ .getBytes(StandardCharsets.UTF_8);
+ exchange.getResponseHeaders().add("Content-Type",
"application/json");
+ } else {
+
s3Authorizations.add(exchange.getRequestHeaders().getFirst("Authorization"));
+ body = "hello".getBytes(StandardCharsets.UTF_8);
+ }
+ exchange.sendResponseHeaders(200, body.length);
+ try (OutputStream os = exchange.getResponseBody()) {
+ os.write(body);
+ }
+ }
+}
