This is an automated email from the ASF dual-hosted git repository.
rishabhdaim pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git
The following commit(s) were added to refs/heads/trunk by this push:
new 77d365073c OAK-12241 : support for S3 tests for oak-blob-cloud to run
on S3Mock (#2964)
77d365073c is described below
commit 77d365073c5ca41d967e6dbc698852368d0bb83c
Author: Rishabh Kumar <[email protected]>
AuthorDate: Sat Jun 20 20:50:09 2026 +0530
OAK-12241 : support for S3 tests for oak-blob-cloud to run on S3Mock (#2964)
* OAK-12241 : updated S3 tests for oak-blob-cloud to run on S3Mock using
docker
* OAK-12241 : fixed test cases
* OAK-12241 : skip S3Mock small cache concurrency test
* OAK-12241 : skip S3Mock incompatible oak-it checks
* OAK-12241 : skip S3Mock record timestamp check
* OAK-12241 : fixed unit cases for oak-run
* OAK-12241 : remove S3-GCP emulator tests plan from PR
* OAK-12241 : addressed review comments by AI
* OAK-12241 : used executorcloser to close timeout executor
* OAK-12241 : fixed the error where we were reusing already closed
blobStore for validation
* OAK-12241 : removed timeout executor from production code
* OAK-12241: make blob fixture close idempotent
* OAK-12241: add blob wrapper regression test
* OAK-12241 : fixed sonar warnings
---
oak-blob-cloud/pom.xml | 5 +
.../jackrabbit/oak/blob/cloud/s3/S3Backend.java | 19 ++-
.../jackrabbit/oak/blob/cloud/s3/S3Constants.java | 8 +
.../apache/jackrabbit/oak/blob/cloud/s3/Utils.java | 32 +++-
.../cloud/s3/S3DataRecordAccessProviderIT.java | 8 +-
.../cloud/s3/S3DataRecordAccessProviderTest.java | 161 ++++++++++++++++++-
.../oak/blob/cloud/s3/S3DataStoreUtils.java | 128 +++++++++++----
.../oak/blob/cloud/s3/S3EmulatorSupport.java | 128 +++++++++++++++
.../oak/blob/cloud/s3/S3EmulatorSupportTest.java | 94 +++++++++++
.../jackrabbit/oak/blob/cloud/s3/S3MockRule.java | 109 +++++++++++++
.../oak/blob/cloud/s3/TestS3DSWithSmallCache.java | 10 ++
.../jackrabbit/oak/blob/cloud/s3/TestS3Ds.java | 25 +++
.../oak/blob/cloud/s3/TestS3DsCacheOff.java | 24 +++
.../jackrabbit/oak/blob/cloud/s3/UtilsTest.java | 171 ++++++++++++++++++++-
oak-doc/src/site/markdown/osgi_config.md | 4 +
.../plugins/document/S3SharedBlobStoreGCTest.java | 10 ++
.../document/SharedS3DataStoreUtilsTest.java | 18 +++
.../oak/run/cli/BlobStoreFixtureProvider.java | 22 ++-
.../oak/run/cli/ReadOnlyBlobStoreWrapper.java | 16 +-
.../oak/run/cli/ReadOnlyBlobStoreWrapperTest.java | 25 ++-
.../jackrabbit/oak/run/DataStoreCommandTest.java | 53 ++++++-
21 files changed, 1015 insertions(+), 55 deletions(-)
diff --git a/oak-blob-cloud/pom.xml b/oak-blob-cloud/pom.xml
index 9bccf0d064..5b70e6453f 100644
--- a/oak-blob-cloud/pom.xml
+++ b/oak-blob-cloud/pom.xml
@@ -487,6 +487,11 @@
<artifactId>org.osgi.service.cm</artifactId>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>org.testcontainers</groupId>
+ <artifactId>testcontainers</artifactId>
+ <scope>test</scope>
+ </dependency>
</dependencies>
</project>
diff --git
a/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3Backend.java
b/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3Backend.java
index b91119cc72..01cb97cb5f 100644
---
a/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3Backend.java
+++
b/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3Backend.java
@@ -133,6 +133,7 @@ public class S3Backend extends AbstractSharedBackend {
private static final int MAX_ALLOWABLE_UPLOAD_URIS = 10000; // AWS
limitation
private S3Client s3Client;
+ private S3Client s3PresignClient;
private S3AsyncClient s3AsyncClient;
// needed only in case of transfer acceleration is enabled for presigned
URIs
@@ -286,8 +287,10 @@ public class S3Backend extends AbstractSharedBackend {
}
void setBinaryTransferAccelerationEnabled(final boolean enabled) {
+ closePresignClient();
if (!enabled) {
- s3PresignService = Utils.createPresigner(s3Client, properties);
+ s3PresignClient = s3Client;
+ s3PresignService = Utils.createPresigner(s3Client, properties,
false);
return;
}
@@ -295,15 +298,24 @@ public class S3Backend extends AbstractSharedBackend {
b -> b.bucket(bucket).build());
if (Objects.equals(BucketAccelerateStatus.ENABLED,
accelerateConfig.status())) {
- s3PresignService =
Utils.createPresigner(Utils.openService(properties, true), properties);
+ s3PresignClient = Utils.openService(properties, true);
+ s3PresignService = Utils.createPresigner(s3PresignClient,
properties, true);
LOG.info("S3 Transfer Acceleration enabled for presigned URIs.");
} else {
LOG.warn("S3 Transfer Acceleration is not enabled on the bucket
{}. Will create normal, non-accelerated presigned URIs. To enable set {}",
bucket, S3Constants.PRESIGNED_URI_ENABLE_ACCELERATION);
- s3PresignService = Utils.createPresigner(s3Client, properties);
+ s3PresignClient = s3Client;
+ s3PresignService = Utils.createPresigner(s3Client, properties,
false);
}
}
+ private void closePresignClient() {
+ if (s3PresignClient != null && s3PresignClient != s3Client) {
+ s3PresignClient.close();
+ }
+ s3PresignClient = null;
+ }
+
/**
* It uploads file to Amazon S3. If file size is greater than 5MB, this
* method uses parallel concurrent connections to upload.
@@ -481,6 +493,7 @@ public class S3Backend extends AbstractSharedBackend {
if (s3PresignService != null) {
s3PresignService.close();
}
+ closePresignClient();
s3Client.close();
}
LOG.info("S3Backend closed.");
diff --git
a/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3Constants.java
b/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3Constants.java
index 7e70b6f13d..5f7238e3b2 100644
---
a/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3Constants.java
+++
b/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3Constants.java
@@ -169,6 +169,14 @@ public final class S3Constants {
*/
public static final String PRESIGNED_HTTP_DOWNLOAD_URI_VERIFY_EXISTS =
"presignedHttpDownloadURIVerifyExists";
+ /**
+ * Boolean property to enable path-style access for S3 clients ({@code
pathStyleAccess=true}).
+ * When enabled, requests use {@code http://endpoint/bucket/key} instead of
+ * {@code https://bucket.endpoint/key}. Defaults to {@code false}.
+ * Always enabled automatically when the remote storage mode is GCP.
+ */
+ public static final String PATH_STYLE_ACCESS = "pathStyleAccess";
+
/**
* private constructor so that class cannot initialized from outside.
*/
diff --git
a/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/Utils.java
b/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/Utils.java
index 6c479076a1..5686b8bad9 100644
---
a/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/Utils.java
+++
b/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/Utils.java
@@ -116,7 +116,6 @@ public final class Utils {
* @return {@link S3Client}
*/
public static S3Client openService(final Properties prop, boolean accReq) {
-
S3ClientBuilder builder = S3Client.builder();
configureBuilder(builder, prop, accReq);
@@ -165,15 +164,24 @@ public final class Utils {
* @return a configured {@link S3Presigner} instance
*/
public static S3Presigner createPresigner(final S3Client s3Client, final
Properties props) {
+ return createPresigner(s3Client, props, false);
+ }
+
+ static S3Presigner createPresigner(final S3Client s3Client, final
Properties props, final boolean accReq) {
final boolean isGCP = Objects.equals(RemoteStorageMode.GCP,
props.get(S3Constants.MODE));
- return S3Presigner.builder().s3Client(s3Client)
+ String region = Utils.getRegion(props);
+ S3Presigner.Builder builder = S3Presigner.builder().s3Client(s3Client)
.credentialsProvider(Utils.getAwsCredentials(props))
- .region(Region.of(Utils.getRegion(props)))
+ .region(Region.of(region))
.serviceConfiguration(S3Configuration.builder()
- .pathStyleAccessEnabled(isGCP)
+
.pathStyleAccessEnabled(isPathStyleAccessEnabled(props, isGCP))
.chunkedEncodingEnabled(!isGCP)
- .build())
- .build();
+ .build());
+
+ if (accReq || hasCustomEndpoint(props)) {
+ builder.endpointOverride(getEndPointUri(props, accReq, region));
+ }
+ return builder.build();
}
/**
@@ -419,7 +427,6 @@ public final class Utils {
int apiTimeout = Math.max(connectionTimeOut * 10, 300000); // At least
5 minutes
ClientOverrideConfiguration.Builder builder =
ClientOverrideConfiguration.builder();
-
builder.retryStrategy(b -> b.maxAttempts(maxErrorRetry));
builder.apiCallTimeout(Duration.ofMillis(apiTimeout)); // Long timeout
for large uploads
builder.apiCallAttemptTimeout(Duration.ofMillis(connectionTimeOut));
// Per-attempt timeout
@@ -584,12 +591,21 @@ public final class Utils {
builder.serviceConfiguration(
S3Configuration.builder()
- .pathStyleAccessEnabled(isGCP) // enable for GCP
+ .pathStyleAccessEnabled(isPathStyleAccessEnabled(prop,
isGCP)) // always on for GCP; opt-in for S3 via pathStyleAccess property
.chunkedEncodingEnabled(!isGCP) // Disable for GCP
.useArnRegionEnabled(!isGCP) // Disable for GCP
.build());
}
+ private static boolean isPathStyleAccessEnabled(Properties prop, boolean
isGCP) {
+ return isGCP ||
Boolean.parseBoolean(prop.getProperty(S3Constants.PATH_STYLE_ACCESS, "false"));
+ }
+
+ private static boolean hasCustomEndpoint(Properties prop) {
+ String endpoint = prop.getProperty(S3Constants.S3_END_POINT);
+ return endpoint != null && !endpoint.isEmpty();
+ }
+
// Helper class to hold common Http config
private static class HttpClientConfig {
final int connectionTimeout;
diff --git
a/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3DataRecordAccessProviderIT.java
b/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3DataRecordAccessProviderIT.java
index fa36a39a29..0958b3d941 100644
---
a/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3DataRecordAccessProviderIT.java
+++
b/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3DataRecordAccessProviderIT.java
@@ -20,6 +20,7 @@ package org.apache.jackrabbit.oak.blob.cloud.s3;
import java.io.File;
import java.io.IOException;
+import java.io.InputStream;
import java.net.URI;
import javax.net.ssl.HttpsURLConnection;
@@ -80,6 +81,11 @@ public class S3DataRecordAccessProviderIT extends
AbstractDataRecordAccessProvid
@Override
protected HttpsURLConnection getHttpsConnection(long length, URI uri)
throws IOException {
- return S3DataStoreUtils.getHttpsConnection(length, uri);
+ throw new UnsupportedOperationException("Upload is handled by
doHttpsUpload; see S3DataStoreUtils.doHttpUpload");
+ }
+
+ @Override
+ protected void doHttpsUpload(InputStream in, long contentLength, URI uri)
throws IOException {
+ S3DataStoreUtils.doHttpUpload(in, contentLength, uri);
}
}
diff --git
a/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3DataRecordAccessProviderTest.java
b/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3DataRecordAccessProviderTest.java
index cf59f79645..fc095532ad 100644
---
a/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3DataRecordAccessProviderTest.java
+++
b/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3DataRecordAccessProviderTest.java
@@ -22,6 +22,8 @@ import static
org.apache.jackrabbit.oak.blob.cloud.s3.S3DataStoreUtils.getFixtur
import static
org.apache.jackrabbit.oak.blob.cloud.s3.S3DataStoreUtils.getS3Config;
import static
org.apache.jackrabbit.oak.blob.cloud.s3.S3DataStoreUtils.getS3DataStore;
import static
org.apache.jackrabbit.oak.blob.cloud.s3.S3DataStoreUtils.isS3Configured;
+import static
org.apache.jackrabbit.oak.plugins.blob.datastore.DataStoreUtils.randomStream;
+import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.junit.Assume.assumeTrue;
@@ -29,13 +31,19 @@ import static org.junit.Assume.assumeTrue;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
+import java.net.HttpURLConnection;
import java.net.URI;
+import java.nio.charset.StandardCharsets;
+import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Properties;
import javax.net.ssl.HttpsURLConnection;
+import org.apache.jackrabbit.oak.api.blob.BlobDownloadOptions;
+import
org.apache.jackrabbit.oak.plugins.blob.datastore.directaccess.DataRecordAccessProvider;
+import
org.apache.jackrabbit.oak.plugins.blob.datastore.directaccess.DataRecordDownloadOptions;
import org.apache.jackrabbit.oak.spi.blob.data.DataIdentifier;
import org.apache.jackrabbit.oak.spi.blob.data.DataRecord;
import org.apache.jackrabbit.oak.spi.blob.data.DataStore;
@@ -131,7 +139,12 @@ public class S3DataRecordAccessProviderTest extends
AbstractDataRecordAccessProv
@Override
protected HttpsURLConnection getHttpsConnection(long length, URI uri)
throws IOException {
- return S3DataStoreUtils.getHttpsConnection(length, uri);
+ throw new UnsupportedOperationException("S3Mock uses HTTP; upload is
handled by doHttpsUpload");
+ }
+
+ @Override
+ protected void doHttpsUpload(InputStream in, long contentLength, URI uri)
throws IOException {
+ S3DataStoreUtils.doHttpUpload(in, contentLength, uri);
}
@Test
@@ -168,4 +181,150 @@ public class S3DataRecordAccessProviderTest extends
AbstractDataRecordAccessProv
upload = ds.initiateDataRecordUpload(uploadSize, -1);
assertEquals(expectedNumURIs, upload.getUploadURIs().size());
}
+
+ @Override
+ @Test
+ public void testGetDownloadURIIT() throws DataStoreException, IOException {
+ assumeTrue("SSE-C doesn't support presigned GET URLs",
!isSSECustomerKeyEncryption());
+ DataRecord dataRecord = null;
+ DataRecordAccessProvider store = getDataStore();
+ try {
+ InputStream testStream = randomStream(0, 256);
+ dataRecord = doSynchronousAddRecord((DataStore) store, testStream);
+ URI uri = store.getDownloadURI(dataRecord.getIdentifier(),
DataRecordDownloadOptions.DEFAULT);
+ HttpURLConnection conn = (HttpURLConnection)
uri.toURL().openConnection();
+ try {
+ conn.setRequestMethod("GET");
+ assertEquals(200, conn.getResponseCode());
+
+ testStream.reset();
+ try (InputStream responseStream = conn.getInputStream()) {
+ assertArrayEquals(testStream.readAllBytes(),
responseStream.readAllBytes());
+ }
+ } finally {
+ conn.disconnect();
+ }
+ } finally {
+ if (null != dataRecord) {
+ doDeleteRecord((DataStore) store, dataRecord.getIdentifier());
+ }
+ }
+ }
+
+ @Override
+ @Test
+ public void testGetDownloadURIWithCustomHeadersIT() throws
DataStoreException, IOException {
+ assumeTrue("SSE-C doesn't support presigned GET URLs",
!isSSECustomerKeyEncryption());
+ String umlautFilename = "Umläutfile.png";
+ String umlautfilenameIso88591 = new String(
+ StandardCharsets.ISO_8859_1.encode(umlautFilename).array(),
+ StandardCharsets.ISO_8859_1
+ );
+ List<String> fileNames = List.of(
+ "image.png",
+ "beautiful landscape.png",
+ "\"filename-with-double-quotes\".png",
+ "filename-with-one\"double-quote.jpg",
+ umlautFilename
+ );
+ List<String> iso88591FileNames = List.of(
+ "image.png",
+ "beautiful landscape.png",
+ "\\\"filename-with-double-quotes\\\".png",
+ "filename-with-one\\\"double-quote.jpg",
+ umlautfilenameIso88591
+ );
+ List<String> rfc8187FileNames = List.of(
+ "image.png",
+ "beautiful%20landscape.png",
+ "%22filename-with-double-quotes%22.png",
+ "filename-with-one%22double-quote.jpg",
+ "Uml%C3%A4utfile.png"
+ );
+
+ DataRecord dataRecord = null;
+ DataRecordAccessProvider store = getDataStore();
+ try {
+ InputStream testStream = randomStream(0, 256);
+ dataRecord = doSynchronousAddRecord((DataStore) store, testStream);
+ String mimeType = "image/png";
+ String dispositionType = "inline";
+ for (int i = 0; i < fileNames.size(); i++) {
+ String fileName = fileNames.get(i);
+ String iso88591FileName = iso88591FileNames.get(i);
+ String encodedFileName = rfc8187FileNames.get(i);
+ DataRecordDownloadOptions downloadOptions =
+ DataRecordDownloadOptions.fromBlobDownloadOptions(
+ new BlobDownloadOptions(mimeType, null,
fileName, dispositionType)
+ );
+ URI uri = store.getDownloadURI(dataRecord.getIdentifier(),
downloadOptions);
+
+ HttpURLConnection conn = (HttpURLConnection)
uri.toURL().openConnection();
+ try {
+ conn.setRequestMethod("GET");
+ assertEquals(200, conn.getResponseCode());
+
+ assertEquals(mimeType,
conn.getHeaderField("Content-Type"));
+ assertEquals(
+ String.format("%s; filename=\"%s\";
filename*=UTF-8''%s",
+ dispositionType, iso88591FileName,
encodedFileName),
+ conn.getHeaderField("Content-Disposition")
+ );
+
+ testStream.reset();
+ try (InputStream responseStream = conn.getInputStream()) {
+ assertArrayEquals(testStream.readAllBytes(),
responseStream.readAllBytes());
+ }
+ } finally {
+ conn.disconnect();
+ }
+ }
+ } finally {
+ if (null != dataRecord) {
+ doDeleteRecord((DataStore) store, dataRecord.getIdentifier());
+ }
+ }
+ }
+
+ @Override
+ @Test
+ public void testSinglePutDirectUploadIT() throws
DataRecordUploadException, DataStoreException, IOException {
+ super.testSinglePutDirectUploadIT();
+ }
+
+ @Override
+ @Test
+ public void testCompleteAlreadyUploadedBinaryReturnsSameBinaryIT() throws
DataStoreException, DataRecordUploadException, IOException {
+ super.testCompleteAlreadyUploadedBinaryReturnsSameBinaryIT();
+ }
+
+ @Override
+ @Test
+ public void testGetExpiredReadURIFailsIT() throws DataStoreException,
IOException {
+ assumeTrue("SSE-C doesn't support presigned GET URLs",
!isSSECustomerKeyEncryption());
+ assumeTrue("S3Mock does not enforce presigned URL expiry",
!S3EmulatorSupport.isAvailable());
+ DataRecord dataRecord = null;
+ ConfigurableDataRecordAccessProvider store = getDataStore();
+ try {
+ store.setDirectDownloadURIExpirySeconds(2);
+ dataRecord = doSynchronousAddRecord((DataStore) store,
randomStream(0, 256));
+ URI uri = store.getDownloadURI(dataRecord.getIdentifier(),
DataRecordDownloadOptions.DEFAULT);
+ try {
+ Thread.sleep(5 * 1000);
+ } catch (InterruptedException ignored) {
+ }
+ HttpURLConnection conn = (HttpURLConnection)
uri.toURL().openConnection();
+ try {
+ conn.setRequestMethod("GET");
+ assertEquals(403, conn.getResponseCode());
+ } finally {
+ conn.disconnect();
+ }
+ } finally {
+ if (null != dataRecord) {
+ doDeleteRecord((DataStore) store, dataRecord.getIdentifier());
+ }
+ store.setDirectDownloadURIExpirySeconds(expirySeconds);
+ }
+ }
}
diff --git
a/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3DataStoreUtils.java
b/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3DataStoreUtils.java
index 5b695ebe84..4cb3b6626a 100644
---
a/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3DataStoreUtils.java
+++
b/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3DataStoreUtils.java
@@ -22,15 +22,17 @@ import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.HttpURLConnection;
import java.net.URI;
+import java.net.URLConnection;
+import java.nio.charset.StandardCharsets;
import java.util.Date;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Properties;
-import javax.net.ssl.HttpsURLConnection;
-
import org.apache.commons.io.IOUtils;
import org.apache.jackrabbit.oak.spi.blob.data.DataStore;
import org.apache.jackrabbit.oak.commons.PropertiesUtil;
@@ -80,6 +82,14 @@ public class S3DataStoreUtils extends DataStoreUtils {
return true;
}
+ public static boolean isS3EmulatorConfigured() {
+ Properties props = getS3Config();
+ return Objects.equals(S3EmulatorSupport.ACCESS_KEY,
props.getProperty(S3Constants.ACCESS_KEY))
+ && Objects.equals(S3EmulatorSupport.SECRET_KEY,
props.getProperty(S3Constants.SECRET_KEY))
+ &&
"true".equals(props.getProperty(S3Constants.PATH_STYLE_ACCESS))
+ && props.getProperty(S3Constants.S3_END_POINT,
"").startsWith("http://127.0.0.1:";);
+ }
+
/**
*
* @return true if SSE_C encryption is configured
@@ -125,9 +135,21 @@ public class S3DataStoreUtils extends DataStoreUtils {
props = new Properties();
props.putAll(filtered);
}
+ // Fall back to emulator when no real credentials are present.
+ if (!hasRealCredentials(props) && S3EmulatorSupport.isAvailable()) {
+ return S3EmulatorSupport.getEmulatorProperties();
+ }
return props;
}
+ private static boolean hasRealCredentials(Properties props) {
+ String accessKey = props.getProperty(S3Constants.ACCESS_KEY,
"").trim();
+ String secretKey = props.getProperty(S3Constants.SECRET_KEY,
"").trim();
+ String region = props.getProperty(S3Constants.S3_REGION, "").trim();
+ String endpoint = props.getProperty(S3Constants.S3_END_POINT,
"").trim();
+ return !accessKey.isEmpty() && !secretKey.isEmpty() &&
(!region.isEmpty() || !endpoint.isEmpty());
+ }
+
public static DataStore getS3DataStore(String className, Properties props,
String homeDir) throws Exception {
DataStore ds =
Class.forName(className).asSubclass(DataStore.class).newInstance();
PropertiesUtil.populate(ds, Utils.asMap(props), false);
@@ -150,38 +172,78 @@ public class S3DataStoreUtils extends DataStoreUtils {
S3BackendHelper.deleteBucketAndAbortMultipartUploads(bucket, date,
props);
}
- protected static HttpsURLConnection getHttpsConnection(long length, URI
uri) throws IOException {
- HttpsURLConnection conn = (HttpsURLConnection)
uri.toURL().openConnection();
- conn.setDoOutput(true);
- conn.setRequestMethod("PUT");
- conn.setRequestProperty("Content-Length", String.valueOf(length));
-
- final Properties props = getS3Config();
- DataEncryption encryption = Utils.getDataEncryption(props);
-
- switch (encryption) {
- case SSE_S3:
- conn.setRequestProperty("x-amz-server-side-encryption",
"AES256");
- System.out.println("Added SSE-S3 header: AES256");
- break;
- case SSE_KMS:
- conn.setRequestProperty("x-amz-server-side-encryption",
"aws:kms");
- System.out.println("Added SSE-KMS header: aws:kms");
- if
(Objects.nonNull(props.getProperty(S3Constants.S3_SSE_KMS_KEYID))) {
-
conn.setRequestProperty("x-amz-server-side-encryption-aws-kms-key-id",
props.getProperty(S3Constants.S3_SSE_KMS_KEYID));
- System.out.println("Added KMS Key ID: " +
props.getProperty(S3Constants.S3_SSE_KMS_KEYID));
- }
- break;
- case SSE_C:
- String customerKey =
props.getProperty(S3Constants.S3_SSE_C_KEY);
- String customerKeyMD5 = Utils.calculateMD5(customerKey);
-
conn.setRequestProperty("x-amz-server-side-encryption-customer-algorithm",
"AES256");
-
conn.setRequestProperty("x-amz-server-side-encryption-customer-key",
customerKey);
-
conn.setRequestProperty("x-amz-server-side-encryption-customer-key-MD5",
customerKeyMD5);
- break;
- case NONE:
- break;
+ protected static HttpURLConnection getHttpConnection(long length, URI uri)
throws IOException {
+ URLConnection raw = uri.toURL().openConnection();
+ if (!(raw instanceof HttpURLConnection)) {
+ throw new IOException("Expected HTTP(S) connection for URI: " +
uri);
+ }
+ HttpURLConnection conn = (HttpURLConnection) raw;
+ try {
+ conn.setDoOutput(true);
+ conn.setRequestMethod("PUT");
+ conn.setRequestProperty("Content-Length", String.valueOf(length));
+
+ final Properties props = getS3Config();
+ DataEncryption encryption = Utils.getDataEncryption(props);
+
+ switch (encryption) {
+ case SSE_S3:
+ conn.setRequestProperty("x-amz-server-side-encryption",
"AES256");
+ System.out.println("Added SSE-S3 header: AES256");
+ break;
+ case SSE_KMS:
+ conn.setRequestProperty("x-amz-server-side-encryption",
"aws:kms");
+ System.out.println("Added SSE-KMS header: aws:kms");
+ if
(Objects.nonNull(props.getProperty(S3Constants.S3_SSE_KMS_KEYID))) {
+
conn.setRequestProperty("x-amz-server-side-encryption-aws-kms-key-id",
props.getProperty(S3Constants.S3_SSE_KMS_KEYID));
+ System.out.println("Added KMS Key ID: " +
props.getProperty(S3Constants.S3_SSE_KMS_KEYID));
+ }
+ break;
+ case SSE_C:
+ String customerKey =
props.getProperty(S3Constants.S3_SSE_C_KEY);
+ String customerKeyMD5 = Utils.calculateMD5(customerKey);
+
conn.setRequestProperty("x-amz-server-side-encryption-customer-algorithm",
"AES256");
+
conn.setRequestProperty("x-amz-server-side-encryption-customer-key",
customerKey);
+
conn.setRequestProperty("x-amz-server-side-encryption-customer-key-MD5",
customerKeyMD5);
+ break;
+ case NONE:
+ break;
+ }
+ } catch (RuntimeException e) {
+ conn.disconnect();
+ throw e;
}
return conn;
}
+
+ protected static void doHttpUpload(InputStream in, long contentLength, URI
uri) throws IOException {
+ HttpURLConnection conn = getHttpConnection(contentLength, uri);
+ try {
+ try (OutputStream out = conn.getOutputStream()) {
+ IOUtils.copy(in, out);
+ }
+ int responseCode = conn.getResponseCode();
+ if (responseCode >= 400) {
+ String errorBody = getErrorBody(conn);
+ org.junit.Assert.fail(conn.getResponseMessage()
+ + (StringUtils.isEmpty(errorBody) ? "" : ": " +
errorBody));
+ }
+ } finally {
+ conn.disconnect();
+ }
+ }
+
+ private static String getErrorBody(HttpURLConnection conn) {
+ InputStream err = conn.getErrorStream();
+ if (err == null) {
+ return "";
+ }
+ try {
+ return IOUtils.toString(err, StandardCharsets.UTF_8);
+ } catch (IOException ignored) {
+ return "";
+ } finally {
+ IOUtils.closeQuietly(err);
+ }
+ }
}
diff --git
a/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3EmulatorSupport.java
b/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3EmulatorSupport.java
new file mode 100644
index 0000000000..d8fc9cdc84
--- /dev/null
+++
b/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3EmulatorSupport.java
@@ -0,0 +1,128 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.jackrabbit.oak.blob.cloud.s3;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Properties;
+
+/**
+ * Lazy singleton that starts an {@link S3MockRule} Docker container on first
use and exposes
+ * S3 emulator properties for use in {@link S3DataStoreUtils#getS3Config()}.
+ *
+ * <p>GCP emulator coverage is intentionally out of scope because S3Mock is
not compatible
+ * with Oak's GCP client configuration.</p>
+ *
+ * <p>If Docker is unavailable, {@link #isAvailable()} returns {@code false}
and tests
+ * skip rather than fail.</p>
+ */
+public final class S3EmulatorSupport {
+
+ private static final Logger log =
LoggerFactory.getLogger(S3EmulatorSupport.class);
+
+ /** System property reserved for selecting emulator mode; only S3 is
currently supported. */
+ static final String S3_TEST_MODE_PROP = "s3.test.mode";
+
+ /** Dummy access key accepted by S3Mock. */
+ static final String ACCESS_KEY = "foo";
+
+ /** Dummy secret key accepted by S3Mock. */
+ static final String SECRET_KEY = "bar";
+
+ /** Default bucket name used when no test-specific bucket is configured. */
+ static final String DEFAULT_BUCKET = "s3mock-default-test-bucket";
+
+ private static volatile S3MockRule instance;
+ private static volatile boolean startupAttempted = false;
+
+ private S3EmulatorSupport() {}
+
+ /**
+ * Returns {@code true} if the S3Mock container started successfully and
is available for tests.
+ */
+ public static boolean isAvailable() {
+ if (isUnsupportedMode()) {
+ return false;
+ }
+ ensureStarted();
+ return instance != null;
+ }
+
+ /**
+ * Returns a {@link Properties} instance configured for the S3 emulator.
+ * Returns an empty {@link Properties} if the container is not available.
+ */
+ public static Properties getEmulatorProperties() {
+ if (isUnsupportedMode()) {
+ return new Properties();
+ }
+ ensureStarted();
+ S3MockRule rule = instance;
+ if (rule == null) {
+ return new Properties();
+ }
+ String endpoint = rule.getHttpEndpoint();
+ Properties props = new Properties();
+ props.setProperty(S3Constants.ACCESS_KEY, ACCESS_KEY);
+ props.setProperty(S3Constants.SECRET_KEY, SECRET_KEY);
+ props.setProperty(S3Constants.S3_BUCKET, DEFAULT_BUCKET);
+ props.setProperty(S3Constants.S3_CONN_PROTOCOL, "http");
+ props.setProperty(S3Constants.PATH_STYLE_ACCESS, "true");
+ props.setProperty(S3Constants.S3_ENCRYPTION,
S3Constants.S3_ENCRYPTION_NONE);
+ props.setProperty(S3Constants.S3_END_POINT, endpoint);
+ props.setProperty(S3Constants.S3_REGION, "us-east-1");
+ props.setProperty(S3Constants.S3_MAX_ERR_RETRY, "3");
+ props.setProperty(S3Constants.S3_CONN_TIMEOUT, "10000");
+ props.setProperty(S3Constants.S3_SOCK_TIMEOUT, "30000");
+ props.setProperty(S3Constants.S3_MAX_CONNS, "10");
+ return props;
+ }
+
+ private static void ensureStarted() {
+ if (isUnsupportedMode() || instance != null) {
+ return;
+ }
+ synchronized (S3EmulatorSupport.class) {
+ if (instance != null || startupAttempted) {
+ return;
+ }
+ startupAttempted = true;
+ S3MockRule rule = new S3MockRule();
+ try {
+ rule.before();
+ instance = rule;
+ Runtime.getRuntime().addShutdownHook(new Thread(() -> {
+ try {
+ rule.after();
+ } catch (Exception ignored) {
+ }
+ }));
+ } catch (Throwable e) {
+ try {
+ rule.after();
+ } catch (Exception ignored) {
+ }
+ log.warn("S3Mock container failed to start — emulator tests
will be skipped", e);
+ }
+ }
+ }
+
+ private static boolean isUnsupportedMode() {
+ return !"S3".equalsIgnoreCase(System.getProperty(S3_TEST_MODE_PROP,
"S3"));
+ }
+}
diff --git
a/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3EmulatorSupportTest.java
b/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3EmulatorSupportTest.java
new file mode 100644
index 0000000000..0e58146efc
--- /dev/null
+++
b/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3EmulatorSupportTest.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.jackrabbit.oak.blob.cloud.s3;
+
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Assume;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.util.Properties;
+
+/**
+ * Tests for {@link S3EmulatorSupport}.
+ */
+public class S3EmulatorSupportTest {
+
+ private String previousMode;
+
+ @Before
+ public void setUp() {
+ previousMode = System.getProperty(S3EmulatorSupport.S3_TEST_MODE_PROP);
+ }
+
+ @After
+ public void tearDown() {
+ if (previousMode == null) {
+ System.clearProperty(S3EmulatorSupport.S3_TEST_MODE_PROP);
+ } else {
+ System.setProperty(S3EmulatorSupport.S3_TEST_MODE_PROP,
previousMode);
+ }
+ }
+
+ @Test
+ public void isAvailableReturnsFalseForUnsupportedGcpMode() {
+ System.setProperty(S3EmulatorSupport.S3_TEST_MODE_PROP, "GCP");
+ Assert.assertFalse(S3EmulatorSupport.isAvailable());
+ }
+
+ @Test
+ public void isAvailableReturnsTrueInS3ModeWhenDockerAvailable() {
+ System.clearProperty(S3EmulatorSupport.S3_TEST_MODE_PROP);
+ Assume.assumeTrue("Docker is not available",
S3EmulatorSupport.isAvailable());
+ Assert.assertTrue(S3EmulatorSupport.isAvailable());
+ }
+
+ @Test
+ public void getEmulatorPropertiesReturnsEmptyForUnsupportedGcpMode() {
+ System.setProperty(S3EmulatorSupport.S3_TEST_MODE_PROP, "GCP");
+ Assert.assertTrue(S3EmulatorSupport.getEmulatorProperties().isEmpty());
+ }
+
+ @Test
+ public void getEmulatorPropertiesReturnsEmptyWhenUnavailable() {
+ // When Docker is not available, isAvailable() is false and
getEmulatorProperties()
+ // must return empty properties rather than throwing.
+ Assume.assumeFalse("Docker is available",
S3EmulatorSupport.isAvailable());
+ Properties props = S3EmulatorSupport.getEmulatorProperties();
+ Assert.assertTrue("Expected empty properties when emulator is
unavailable", props.isEmpty());
+ }
+
+ @Test
+ public void getEmulatorPropertiesInS3ModeContainsRequiredKeys() {
+ System.clearProperty(S3EmulatorSupport.S3_TEST_MODE_PROP);
+ Assume.assumeTrue("Docker is not available",
S3EmulatorSupport.isAvailable());
+
+ Properties props = S3EmulatorSupport.getEmulatorProperties();
+ Assert.assertNotNull(props.getProperty(S3Constants.ACCESS_KEY));
+ Assert.assertNotNull(props.getProperty(S3Constants.SECRET_KEY));
+ Assert.assertNotNull(props.getProperty(S3Constants.S3_END_POINT));
+ Assert.assertNotNull(props.getProperty(S3Constants.S3_REGION));
+ Assert.assertNotNull(props.getProperty(S3Constants.S3_BUCKET));
+ Assert.assertEquals("true",
props.getProperty(S3Constants.PATH_STYLE_ACCESS));
+ Assert.assertNotNull(props.getProperty(S3Constants.S3_MAX_ERR_RETRY));
+ Assert.assertNotNull(props.getProperty(S3Constants.S3_CONN_TIMEOUT));
+ Assert.assertEquals("http",
props.getProperty(S3Constants.S3_CONN_PROTOCOL));
+ Assert.assertEquals(S3Constants.S3_ENCRYPTION_NONE,
props.getProperty(S3Constants.S3_ENCRYPTION));
+ Assert.assertNull("S3 mode must not set MODE",
props.get(S3Constants.MODE));
+ }
+}
diff --git
a/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3MockRule.java
b/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3MockRule.java
new file mode 100644
index 0000000000..d1878c7749
--- /dev/null
+++
b/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3MockRule.java
@@ -0,0 +1,109 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.jackrabbit.oak.blob.cloud.s3;
+
+import org.junit.Assume;
+import org.junit.rules.ExternalResource;
+import org.junit.runner.Description;
+import org.junit.runners.model.MultipleFailureException;
+import org.junit.runners.model.Statement;
+import org.testcontainers.containers.GenericContainer;
+import org.testcontainers.utility.DockerImageName;
+
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicReference;
+
+/**
+ * JUnit rule that starts an Adobe S3Mock Docker container for emulator-based
S3 integration tests.
+ * Mirrors the pattern used by {@code AzuriteDockerRule} in
oak-blob-cloud-azure.
+ *
+ * <p>On Docker failure, the rule calls {@link Assume#assumeNoException} so
tests are skipped
+ * rather than failed, preserving the same behaviour as when no credentials
are configured.</p>
+ */
+public class S3MockRule extends ExternalResource {
+
+ private static final DockerImageName DOCKER_IMAGE =
DockerImageName.parse("adobe/s3mock:5.0.0");
+ static final int HTTP_PORT = 9090;
+ private final AtomicReference<Exception> startupException = new
AtomicReference<>();
+
+ private GenericContainer<?> container;
+
+ @Override
+ protected void before() throws Throwable {
+ startupException.set(null);
+ container = new GenericContainer<>(DOCKER_IMAGE)
+ .withExposedPorts(HTTP_PORT)
+ .withStartupTimeout(Duration.ofSeconds(60));
+ try {
+ container.start();
+ } catch (Exception e) {
+ startupException.set(e);
+ throw e;
+ }
+ }
+
+ @Override
+ protected void after() {
+ if (container != null) {
+ container.stop();
+ }
+ }
+
+ @Override
+ public Statement apply(Statement base, Description description) {
+ return new Statement() {
+ @Override
+ public void evaluate() throws Throwable {
+ try {
+ before();
+ } catch (Exception e) {
+ Assume.assumeNoException(e);
+ throw e;
+ }
+ List<Throwable> errors = new ArrayList<>();
+ try {
+ base.evaluate();
+ } catch (Throwable t) {
+ errors.add(t);
+ } finally {
+ try {
+ after();
+ } catch (Throwable t) {
+ errors.add(t);
+ }
+ }
+ MultipleFailureException.assertEmpty(errors);
+ }
+ };
+ }
+
+ /**
+ * Returns the mapped HTTP port on the host for the running S3Mock
container.
+ */
+ public int getHttpPort() {
+ return container.getMappedPort(HTTP_PORT);
+ }
+
+ /**
+ * Returns the HTTP endpoint URL for the running S3Mock container, e.g.
{@code http://127.0.0.1:PORT}.
+ */
+ public String getHttpEndpoint() {
+ return "http://127.0.0.1:"; + getHttpPort();
+ }
+}
diff --git
a/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/TestS3DSWithSmallCache.java
b/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/TestS3DSWithSmallCache.java
index 227e7019bc..bab5189dea 100644
---
a/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/TestS3DSWithSmallCache.java
+++
b/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/TestS3DSWithSmallCache.java
@@ -18,7 +18,9 @@ package org.apache.jackrabbit.oak.blob.cloud.s3;
import org.apache.jackrabbit.oak.spi.blob.data.CachingDataStore;
import org.apache.jackrabbit.oak.spi.blob.data.LocalCache;
+import org.junit.Assume;
import org.junit.Before;
+import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -41,4 +43,12 @@ public class TestS3DSWithSmallCache extends TestS3Ds {
props.setProperty("cacheSize", String.valueOf(dataLength * 10));
super.setUp();
}
+
+ @Override
+ @Test
+ public void testMultiThreaded() {
+ Assume.assumeTrue("S3Mock is not reliable with concurrent reads
through the very small local cache",
+ !S3DataStoreUtils.isS3EmulatorConfigured());
+ super.testMultiThreaded();
+ }
}
diff --git
a/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/TestS3Ds.java
b/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/TestS3Ds.java
index 00e79a73c2..5826a0de23 100644
---
a/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/TestS3Ds.java
+++
b/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/TestS3Ds.java
@@ -70,6 +70,7 @@ import static
org.apache.jackrabbit.oak.blob.cloud.s3.S3DataStoreUtils.getFixtur
import static
org.apache.jackrabbit.oak.blob.cloud.s3.S3DataStoreUtils.getS3Config;
import static
org.apache.jackrabbit.oak.blob.cloud.s3.S3DataStoreUtils.getS3DataStore;
import static
org.apache.jackrabbit.oak.blob.cloud.s3.S3DataStoreUtils.isS3Configured;
+import static
org.apache.jackrabbit.oak.blob.cloud.s3.S3DataStoreUtils.isS3EmulatorConfigured;
import static
org.apache.jackrabbit.oak.blob.cloud.s3.S3DataStoreUtils.isSseCustomerKeyEncrypted;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.fail;
@@ -315,6 +316,16 @@ public class TestS3Ds extends AbstractDataStoreTest {
public void testDeleteAllOlderThan() {
}
+ // deleteRecord removes the object from S3 but leaves the local file cache
intact.
+ // getRecordIfStored then returns the cached copy, causing a spurious
failure.
+ // The cache-off subclass (TestS3DsCacheOff) re-enables this test where it
is valid.
+ @Override
+ public void testDeleteRecord() {
+ Assume.assumeFalse("S3 local cache masks deletions; not verifiable
against the emulator",
+ isS3EmulatorConfigured());
+ super.testDeleteRecord();
+ }
+
// helper methods
private PutObjectResponse httpPut(@Nullable DataRecordUpload
uploadContext, InputStream inputstream, long length) {
@@ -381,6 +392,9 @@ public class TestS3Ds extends AbstractDataStoreTest {
private static String extractBucketFromUri(S3Client s3Client, URI uri) {
LOG.info("Extracting bucket from URI {}", uri);
+ if (isLocalPathStyleUri(uri)) {
+ return uri.getPath().substring(1).split("/", 2)[0];
+ }
S3Utilities s3Utilities = s3Client.utilities();
S3Uri s3Uri = s3Utilities.parseUri(uri);
@@ -390,10 +404,21 @@ public class TestS3Ds extends AbstractDataStoreTest {
private static String extractKeyFromUri(S3Client s3Client, URI uri) {
LOG.info("Extracting key from URI {}", uri);
+ if (isLocalPathStyleUri(uri)) {
+ return uri.getPath().substring(1).split("/", 2)[1];
+ }
S3Utilities s3Utilities = s3Client.utilities();
S3Uri s3Uri = s3Utilities.parseUri(uri);
return s3Uri.key().orElse(null);
}
+
+ private static boolean isLocalPathStyleUri(URI uri) {
+ String host = uri.getHost();
+ String path = uri.getPath();
+ return ("127.0.0.1".equals(host) || "localhost".equals(host))
+ && path != null
+ && path.substring(1).contains("/");
+ }
}
diff --git
a/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/TestS3DsCacheOff.java
b/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/TestS3DsCacheOff.java
index fb62f1ee8e..00969076ca 100644
---
a/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/TestS3DsCacheOff.java
+++
b/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/TestS3DsCacheOff.java
@@ -17,7 +17,9 @@
package org.apache.jackrabbit.oak.blob.cloud.s3;
import org.apache.jackrabbit.oak.spi.blob.data.CachingDataStore;
+import org.junit.Assume;
import org.junit.Before;
+import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -40,4 +42,26 @@ public class TestS3DsCacheOff extends TestS3Ds {
props.setProperty("cacheSize", "0");
super.setUp();
}
+
+ // Re-enable: with cache off, deleteRecord is immediately visible via S3
so the
+ // assertion that getRecordIfStored returns null after deletion is valid
here.
+ @Override
+ @Test
+ public void testDeleteRecord() {
+ try {
+ doDeleteRecordTest();
+ } catch (Exception e) {
+ throw new AssertionError("Failed to delete S3 record with cache
disabled", e);
+ }
+ }
+
+ // S3Backend updates duplicate records via CopyObject (copy-to-self).
S3Mock does not
+ // support that operation, so this test cannot run against the emulator.
+ @Override
+ @Test
+ public void testAddDuplicateRecord() {
+ Assume.assumeTrue("S3Mock does not support CopyObject used by
S3Backend for duplicate record updates",
+ !S3DataStoreUtils.isS3EmulatorConfigured());
+ super.testAddDuplicateRecord();
+ }
}
diff --git
a/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/UtilsTest.java
b/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/UtilsTest.java
index 580c111e3c..6c3eba2b20 100644
---
a/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/UtilsTest.java
+++
b/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/UtilsTest.java
@@ -18,11 +18,23 @@
package org.apache.jackrabbit.oak.blob.cloud.s3;
import org.junit.Assert;
+import org.junit.Assume;
import org.junit.Test;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
import java.net.URI;
+import java.time.Duration;
import java.util.Properties;
+import software.amazon.awssdk.services.s3.S3Client;
+import software.amazon.awssdk.services.s3.model.GetObjectRequest;
+import software.amazon.awssdk.services.s3.presigner.S3Presigner;
+import
software.amazon.awssdk.services.s3.presigner.model.GetObjectPresignRequest;
+import
software.amazon.awssdk.services.s3.presigner.model.PresignedGetObjectRequest;
+
/**
* Unit cases for Utils class
*/
@@ -201,4 +213,161 @@ public class UtilsTest {
Assert.assertEquals(S3Backend.RemoteStorageMode.GCP,
props.get(S3Constants.MODE));
}
-}
\ No newline at end of file
+ @Test
+ public void testSetRemoteStorageModeDefaultsPreSetGCPToS3() {
+ Properties props = new Properties();
+ props.setProperty("s3EndPoint", "http://127.0.0.1:9090";);
+ props.put(S3Constants.MODE, S3Backend.RemoteStorageMode.GCP);
+ Utils.setRemoteStorageMode(props);
+ Assert.assertEquals(S3Backend.RemoteStorageMode.S3,
props.get(S3Constants.MODE));
+ }
+
+ @Test
+ public void testSetRemoteStorageModeDefaultsBlankModeToS3() {
+ Properties props = new Properties();
+ props.setProperty("s3EndPoint", "https://s3.amazonaws.com";);
+ props.setProperty(S3Constants.MODE, "");
+ Utils.setRemoteStorageMode(props);
+ Assert.assertEquals(S3Backend.RemoteStorageMode.S3,
props.get(S3Constants.MODE));
+ }
+
+ @Test
+ public void testPathStyleAccessConstantHasExpectedStringValue() {
+ Assert.assertEquals("pathStyleAccess", S3Constants.PATH_STYLE_ACCESS);
+ }
+
+ @Test
+ public void testPathStyleAccessDefaultsToFalseWhenPropertyAbsent() {
+ Properties props = new Properties();
+
Assert.assertFalse(Boolean.parseBoolean(props.getProperty(S3Constants.PATH_STYLE_ACCESS,
"false")));
+ }
+
+ @Test
+ public void testPathStyleAccessTrueWhenPropertySetToTrue() {
+ Properties props = new Properties();
+ props.setProperty(S3Constants.PATH_STYLE_ACCESS, "true");
+
Assert.assertTrue(Boolean.parseBoolean(props.getProperty(S3Constants.PATH_STYLE_ACCESS,
"false")));
+ }
+
+ @Test
+ public void createPresignerUsesConfiguredEndpointOverride() {
+ Properties props = clientProperties();
+ props.setProperty(S3Constants.S3_END_POINT, "http://127.0.0.1:9090";);
+ props.setProperty(S3Constants.PATH_STYLE_ACCESS, "true");
+
+ try (S3Client client = Utils.openService(props, false);
+ S3Presigner presigner = Utils.createPresigner(client, props)) {
+ GetObjectRequest getObject =
GetObjectRequest.builder().bucket("bucket").key("key").build();
+ GetObjectPresignRequest presignRequest =
GetObjectPresignRequest.builder()
+ .signatureDuration(Duration.ofMinutes(5))
+ .getObjectRequest(getObject)
+ .build();
+ PresignedGetObjectRequest presigned =
presigner.presignGetObject(presignRequest);
+
+ Assert.assertEquals("127.0.0.1", presigned.url().getHost());
+ Assert.assertEquals(9090, presigned.url().getPort());
+ Assert.assertEquals("/bucket/key", presigned.url().getPath());
+ }
+ }
+
+ @Test
+ public void createPresignerUsesAccelerationEndpointWhenRequested() {
+ Properties props = clientProperties();
+
+ try (S3Client client = Utils.openService(props, true);
+ S3Presigner presigner = Utils.createPresigner(client, props,
true)) {
+ GetObjectRequest getObject =
GetObjectRequest.builder().bucket("bucket").key("key").build();
+ GetObjectPresignRequest presignRequest =
GetObjectPresignRequest.builder()
+ .signatureDuration(Duration.ofMinutes(5))
+ .getObjectRequest(getObject)
+ .build();
+ PresignedGetObjectRequest presigned =
presigner.presignGetObject(presignRequest);
+
+
Assert.assertTrue(presigned.url().getHost().contains("s3-accelerate"));
+ }
+ }
+
+ @Test
+ public void
isS3ConfiguredReturnsFalseForPartialCredentialsWithNoRegionOrEndpoint() throws
IOException {
+ // accessKey + secretKey without region or endpoint must not count as
real credentials
+ File tmp = File.createTempFile("s3test-partial-creds", ".properties");
+ String previousConfig = System.getProperty("s3.config");
+ try {
+ Properties partial = new Properties();
+ partial.setProperty(S3Constants.ACCESS_KEY, "somekey");
+ partial.setProperty(S3Constants.SECRET_KEY, "somesecret");
+ try (OutputStream out = new FileOutputStream(tmp)) {
+ partial.store(out, null);
+ }
+ System.setProperty("s3.config", tmp.getAbsolutePath());
+ // Skip when emulator is available — getS3Config() would fall back
to it, making isS3Configured() true
+ Assume.assumeFalse("Emulator available — would override partial
creds", S3EmulatorSupport.isAvailable());
+ Assert.assertFalse("Partial credentials (no region/endpoint) must
not be treated as real",
+ S3DataStoreUtils.isS3Configured());
+ } finally {
+ if (previousConfig == null) {
+ System.clearProperty("s3.config");
+ } else {
+ System.setProperty("s3.config", previousConfig);
+ }
+ tmp.delete();
+ }
+ }
+
+ @Test
+ public void isS3EmulatorConfiguredReturnsTrueForEmulatorProperties()
throws IOException {
+ Properties props = clientProperties();
+ props.setProperty(S3Constants.ACCESS_KEY,
S3EmulatorSupport.ACCESS_KEY);
+ props.setProperty(S3Constants.SECRET_KEY,
S3EmulatorSupport.SECRET_KEY);
+ props.setProperty(S3Constants.S3_END_POINT, "http://127.0.0.1:9090";);
+ props.setProperty(S3Constants.PATH_STYLE_ACCESS, "true");
+
+ withS3Config(props, () ->
Assert.assertTrue(S3DataStoreUtils.isS3EmulatorConfigured()));
+ }
+
+ @Test
+ public void isS3EmulatorConfiguredReturnsFalseForRealS3Properties() throws
IOException {
+ Properties props = clientProperties();
+ props.setProperty(S3Constants.S3_END_POINT,
"https://s3.amazonaws.com";);
+
+ withS3Config(props, () ->
Assert.assertFalse(S3DataStoreUtils.isS3EmulatorConfigured()));
+ }
+
+ private static Properties clientProperties() {
+ Properties props = new Properties();
+ props.setProperty(S3Constants.ACCESS_KEY, "accessKey");
+ props.setProperty(S3Constants.SECRET_KEY, "secretKey");
+ props.setProperty(S3Constants.S3_REGION, "us-east-1");
+ props.setProperty(S3Constants.S3_MAX_ERR_RETRY, "3");
+ props.setProperty(S3Constants.S3_CONN_TIMEOUT, "1000");
+ props.setProperty(S3Constants.S3_SOCK_TIMEOUT, "1000");
+ props.setProperty(S3Constants.S3_MAX_CONNS, "2");
+ props.setProperty(S3Constants.S3_CONN_PROTOCOL, "http");
+ props.setProperty(S3Constants.S3_CROSS_REGION_ACCESS, "false");
+ return props;
+ }
+
+ private static void withS3Config(Properties props, ConfigAssertion
assertion) throws IOException {
+ File tmp = File.createTempFile("s3test-config", ".properties");
+ String previousConfig = System.getProperty("s3.config");
+ try {
+ try (OutputStream out = new FileOutputStream(tmp)) {
+ props.store(out, null);
+ }
+ System.setProperty("s3.config", tmp.getAbsolutePath());
+ assertion.run();
+ } finally {
+ if (previousConfig == null) {
+ System.clearProperty("s3.config");
+ } else {
+ System.setProperty("s3.config", previousConfig);
+ }
+ tmp.delete();
+ }
+ }
+
+ private interface ConfigAssertion {
+ void run();
+ }
+
+}
diff --git a/oak-doc/src/site/markdown/osgi_config.md
b/oak-doc/src/site/markdown/osgi_config.md
index 67c471500a..0b8ae52598 100644
--- a/oak-doc/src/site/markdown/osgi_config.md
+++ b/oak-doc/src/site/markdown/osgi_config.md
@@ -367,6 +367,10 @@ s3Region
s3EndPoint
: S3 rest API endpoint. Can help reduce latency of redirection from standard
endpoint if a different region configured.
+pathStyleAccess
+: Default - `false`
+: Set to `true` to use path-style S3 requests (`http://endpoint/bucket/key`)
instead of virtual-host-style requests. This is useful for S3-compatible
endpoints that do not support bucket names in the host. Note: always enabled
automatically when the remote storage mode is GCP; the property has no effect
in that case.
+
connectionTimeout
: S3 connection timeout. See [AWS S3
documentation](https://docs.aws.amazon.com/sdk-for-java/v1/developer-guide/section-client-configuration.html).
diff --git
a/oak-it/src/test/java/org/apache/jackrabbit/oak/plugins/document/S3SharedBlobStoreGCTest.java
b/oak-it/src/test/java/org/apache/jackrabbit/oak/plugins/document/S3SharedBlobStoreGCTest.java
index 91f5e1be73..aea3ac203b 100644
---
a/oak-it/src/test/java/org/apache/jackrabbit/oak/plugins/document/S3SharedBlobStoreGCTest.java
+++
b/oak-it/src/test/java/org/apache/jackrabbit/oak/plugins/document/S3SharedBlobStoreGCTest.java
@@ -27,9 +27,11 @@ import
org.apache.jackrabbit.oak.blob.cloud.s3.S3DataStoreUtils;
import org.apache.jackrabbit.oak.plugins.blob.datastore.DataStoreBlobStore;
import org.junit.After;
import org.junit.BeforeClass;
+import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
+import static org.junit.Assume.assumeFalse;
import static org.junit.Assume.assumeTrue;
/**
@@ -71,6 +73,14 @@ public class S3SharedBlobStoreGCTest extends
SharedBlobStoreGCTest {
S3DataStoreUtils.getS3DataStore(s3Class, props,
rootFolder.getAbsolutePath()));
}
+ @Override
+ @Test
+ public void testGCWithNodeSpecialChars() throws Exception {
+ assumeFalse("S3Mock is not reliable for this shared-blob GC
special-character write path",
+ S3DataStoreUtils.isS3EmulatorConfigured());
+ super.testGCWithNodeSpecialChars();
+ }
+
@Override
protected void sleep() throws InterruptedException {
if (S3DataStoreUtils.isS3Configured()) {
diff --git
a/oak-it/src/test/java/org/apache/jackrabbit/oak/plugins/document/SharedS3DataStoreUtilsTest.java
b/oak-it/src/test/java/org/apache/jackrabbit/oak/plugins/document/SharedS3DataStoreUtilsTest.java
index dcc76af645..7a087039a2 100644
---
a/oak-it/src/test/java/org/apache/jackrabbit/oak/plugins/document/SharedS3DataStoreUtilsTest.java
+++
b/oak-it/src/test/java/org/apache/jackrabbit/oak/plugins/document/SharedS3DataStoreUtilsTest.java
@@ -30,12 +30,14 @@ import
org.apache.jackrabbit.oak.plugins.blob.SharedDataStoreUtilsTest;
import org.apache.jackrabbit.oak.plugins.blob.datastore.DataStoreBlobStore;
import org.junit.After;
import org.junit.BeforeClass;
+import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import static
org.apache.jackrabbit.oak.blob.cloud.s3.S3DataStoreUtils.deleteBucket;
+import static org.junit.Assume.assumeFalse;
import static org.junit.Assume.assumeTrue;
/**
@@ -71,6 +73,22 @@ public class SharedS3DataStoreUtilsTest extends
SharedDataStoreUtilsTest {
S3DataStoreUtils.getS3DataStore(s3Class, props,
rootFolder.getAbsolutePath()));
}
+ @Override
+ @Test
+ public void test() throws Exception {
+ assumeFalse("S3Mock returns metadata lastModified values with second
precision",
+ S3DataStoreUtils.isS3EmulatorConfigured());
+ super.test();
+ }
+
+ @Override
+ @Test
+ public void testStreamFromGetAllRecords() throws Exception {
+ assumeFalse("S3Mock returns getAllRecords lastModified values with
second precision",
+ S3DataStoreUtils.isS3EmulatorConfigured());
+ super.testStreamFromGetAllRecords();
+ }
+
@After
public void close() throws IOException {
try {
diff --git
a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/run/cli/BlobStoreFixtureProvider.java
b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/run/cli/BlobStoreFixtureProvider.java
index 0f1a3d2e0f..083754c8a5 100644
---
a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/run/cli/BlobStoreFixtureProvider.java
+++
b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/run/cli/BlobStoreFixtureProvider.java
@@ -29,6 +29,7 @@ import java.util.Enumeration;
import java.util.HashMap;
import java.util.Map;
import java.util.Properties;
+import java.util.concurrent.atomic.AtomicBoolean;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.FilenameUtils;
@@ -100,7 +101,8 @@ public class BlobStoreFixtureProvider {
}
delegate.init(null);
}
- DataStoreBlobStore blobStore = new DataStoreBlobStore(delegate);
+ // The command fixture and DocumentNodeStore can both tear down the
same blob store instance.
+ DataStoreBlobStore blobStore = new
IdempotentDataStoreBlobStore(delegate);
return new DataStoreFixture(blobStore, closer,
(!options.getCommonOpts().isReadWrite() && !bsopts.isReadWrite()));
}
@@ -140,8 +142,8 @@ public class BlobStoreFixtureProvider {
@Override
public void close() throws IOException {
- closer.close();
try {
+ closer.close();
blobStore.close();
} catch (DataStoreException e) {
throw new IOException(e);
@@ -149,6 +151,22 @@ public class BlobStoreFixtureProvider {
}
}
+ private static final class IdempotentDataStoreBlobStore extends
DataStoreBlobStore {
+ private final AtomicBoolean closed = new AtomicBoolean();
+
+ private IdempotentDataStoreBlobStore(DataStore delegate) {
+ super(delegate);
+ }
+
+ @Override
+ public void close() throws DataStoreException {
+ // DocumentNodeStore and the outer fixture can both own teardown
of the same blob store.
+ if (closed.compareAndSet(false, true)) {
+ super.close();
+ }
+ }
+ }
+
private static Properties loadAndTransformProps(String cfgPath) throws
IOException {
Dictionary dict = ConfigurationHandler.read(new
FileInputStream(cfgPath));
Properties props = new Properties();
diff --git
a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/run/cli/ReadOnlyBlobStoreWrapper.java
b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/run/cli/ReadOnlyBlobStoreWrapper.java
index 782d0be246..f2a9414a89 100644
---
a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/run/cli/ReadOnlyBlobStoreWrapper.java
+++
b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/run/cli/ReadOnlyBlobStoreWrapper.java
@@ -21,6 +21,7 @@ package org.apache.jackrabbit.oak.run.cli;
import java.lang.reflect.InvocationHandler;
import java.lang.reflect.Method;
import java.lang.reflect.Proxy;
+import java.util.LinkedHashSet;
import java.util.Set;
import org.apache.jackrabbit.oak.spi.blob.BlobStore;
@@ -28,12 +29,25 @@ import org.apache.jackrabbit.oak.spi.blob.BlobStore;
class ReadOnlyBlobStoreWrapper {
public static BlobStore wrap(BlobStore delegate){
- Class[] interfaces = delegate.getClass().getInterfaces();
+ Class<?>[] interfaces = getAllInterfaces(delegate);
return (BlobStore)
Proxy.newProxyInstance(ReadOnlyBlobStoreWrapper.class.getClassLoader(),
interfaces,
new ReadOnlyHandler(delegate));
}
+ private static Class<?>[] getAllInterfaces(BlobStore delegate) {
+ Set<Class<?>> interfaces = new LinkedHashSet<>();
+ Class<?> type = delegate.getClass();
+ while (type != null) {
+ for (Class<?> iface : type.getInterfaces()) {
+ interfaces.add(iface);
+ }
+ type = type.getSuperclass();
+ }
+ interfaces.add(BlobStore.class);
+ return interfaces.toArray(new Class<?>[0]);
+ }
+
private static class ReadOnlyHandler implements InvocationHandler {
private final Set<String> writableMethods = Set.of(
"writeBlob", //BlobStore
diff --git
a/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/run/cli/ReadOnlyBlobStoreWrapperTest.java
b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/run/cli/ReadOnlyBlobStoreWrapperTest.java
index 1d6f19689c..7f9c51a060 100644
---
a/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/run/cli/ReadOnlyBlobStoreWrapperTest.java
+++
b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/run/cli/ReadOnlyBlobStoreWrapperTest.java
@@ -23,9 +23,12 @@ import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.InputStream;
+import org.apache.jackrabbit.oak.plugins.blob.BlobTrackingStore;
import org.apache.jackrabbit.oak.spi.blob.data.FileDataStore;
import org.apache.jackrabbit.oak.plugins.blob.datastore.DataStoreBlobStore;
+import org.apache.jackrabbit.oak.plugins.blob.datastore.TypedDataStore;
import org.apache.jackrabbit.oak.spi.blob.BlobStore;
+import org.apache.jackrabbit.oak.spi.blob.GarbageCollectableBlobStore;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
@@ -62,4 +65,24 @@ public class ReadOnlyBlobStoreWrapperTest {
}
-}
\ No newline at end of file
+ @Test
+ public void readOnlySubclassPreservesBlobStoreInterfaces() {
+ FileDataStore fds = new FileDataStore();
+ fds.setPath(temporaryFolder.getRoot().getAbsolutePath());
+ fds.init(null);
+
+ BlobStore readOnly = ReadOnlyBlobStoreWrapper.wrap(new
DerivedDataStoreBlobStore(fds));
+
+ assertTrue(readOnly instanceof BlobStore);
+ assertTrue(readOnly instanceof GarbageCollectableBlobStore);
+ assertTrue(readOnly instanceof TypedDataStore);
+ assertTrue(readOnly instanceof BlobTrackingStore);
+ }
+
+ private static final class DerivedDataStoreBlobStore extends
DataStoreBlobStore {
+ private DerivedDataStoreBlobStore(FileDataStore delegate) {
+ super(delegate);
+ }
+ }
+
+}
diff --git
a/oak-run/src/test/java/org/apache/jackrabbit/oak/run/DataStoreCommandTest.java
b/oak-run/src/test/java/org/apache/jackrabbit/oak/run/DataStoreCommandTest.java
index 0a477cdfa0..9d29d89322 100644
---
a/oak-run/src/test/java/org/apache/jackrabbit/oak/run/DataStoreCommandTest.java
+++
b/oak-run/src/test/java/org/apache/jackrabbit/oak/run/DataStoreCommandTest.java
@@ -27,6 +27,8 @@ import java.io.InputStream;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Date;
+import java.util.Dictionary;
+import java.util.Enumeration;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
@@ -843,10 +845,16 @@ public class DataStoreCommandTest {
}
Set<String> blobsBeforeGc = SetUtils.difference(data.added,
data.missingDataStore);
- if (maxAge <= 0) {
- assertEquals(SetUtils.difference(blobsBeforeGc, data.deleted),
blobs(setupDataStore));
- } else {
- assertEquals(blobsBeforeGc, blobs(setupDataStore));
+ // storeFixture.close() may dispose the original blob store, so
validate through a fresh handle.
+ DataStoreBlobStore validationBlobStore =
blobFixture.openDataStore(temporaryFolder.newFolder().getAbsolutePath());
+ try {
+ if (maxAge <= 0) {
+ assertEquals(SetUtils.difference(blobsBeforeGc, data.deleted),
blobs(validationBlobStore));
+ } else {
+ assertEquals(blobsBeforeGc, blobs(validationBlobStore));
+ }
+ } finally {
+ validationBlobStore.close();
}
}
@@ -967,6 +975,21 @@ public class DataStoreCommandTest {
return cfgFile.getAbsolutePath();
}
+ // Reuse the same OSGi config format as the command under test when
reopening a datastore.
+ private static Properties readConfigProperties(String cfgFilePath) throws
IOException {
+ Dictionary<?, ?> dict;
+ try (FileInputStream fis = new FileInputStream(cfgFilePath)) {
+ dict = ConfigurationHandler.read(fis);
+ }
+ Properties props = new Properties();
+ Enumeration<?> keys = dict.keys();
+ while (keys.hasMoreElements()) {
+ String key = (String) keys.nextElement();
+ props.put(key, dict.get(key));
+ }
+ return props;
+ }
+
private static Set<String> encodedIdsAndPath(Set<String> ids, Type
dsOption, Map<String, String> idToNodes,
boolean encodeId) {
return SetUtils.toSet(IteratorUtils.transform(ids.iterator(), input ->
String.join(",", encodeId ? encodeId(input, dsOption) : input,
idToNodes.get(input))));
@@ -1244,6 +1267,8 @@ public class DataStoreCommandTest {
DataStoreBlobStore getDataStore();
+ DataStoreBlobStore openDataStore(String homeDir) throws Exception;
+
String getConfigPath();
Type getType();
@@ -1281,6 +1306,12 @@ public class DataStoreCommandTest {
return blobStore;
}
+ @Override public DataStoreBlobStore openDataStore(String homeDir)
throws Exception {
+ Properties props = readConfigProperties(cfgFilePath);
+ DataStore ds =
S3DataStoreUtils.getS3DataStore(S3DataStoreUtils.getFixtures().get(0), props,
homeDir);
+ return new DataStoreBlobStore(ds);
+ }
+
@Override public String getConfigPath() {
return cfgFilePath;
}
@@ -1324,6 +1355,12 @@ public class DataStoreCommandTest {
return blobStore;
}
+ @Override public DataStoreBlobStore openDataStore(String homeDir)
throws Exception {
+ Properties props = readConfigProperties(cfgFilePath);
+ DataStore ds = AzureDataStoreUtils.getAzureDataStore(props,
homeDir);
+ return new DataStoreBlobStore(ds);
+ }
+
@Override public String getConfigPath() {
return cfgFilePath;
}
@@ -1370,6 +1407,14 @@ public class DataStoreCommandTest {
return blobStore;
}
+ @Override public DataStoreBlobStore openDataStore(String homeDir)
throws Exception {
+ OakFileDataStore delegate = new OakFileDataStore();
+ Properties props = readConfigProperties(cfgFilePath);
+ delegate.setPath(props.getProperty("path"));
+ delegate.init(null);
+ return new DataStoreBlobStore(delegate);
+ }
+
@Override public String getConfigPath() {
return cfgFilePath;
}
