This is an automated email from the ASF dual-hosted git repository. rishabhdaim pushed a commit to branch OAK-12241 in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git
commit c0946b9a8f9b9707a941a18ade1fac38dfcedf63 Author: rishabhdaim <[email protected]> AuthorDate: Fri Jun 19 00:02:06 2026 +0530 OAK-12241 : updated S3 tests for oak-blob-cloud to run on S3Mock using docker --- oak-blob-cloud/pom.xml | 5 + .../jackrabbit/oak/blob/cloud/s3/S3Constants.java | 8 + .../apache/jackrabbit/oak/blob/cloud/s3/Utils.java | 8 +- .../cloud/s3/S3DataRecordAccessProviderIT.java | 8 +- .../cloud/s3/S3DataRecordAccessProviderTest.java | 171 ++++++++++++++++++++- .../oak/blob/cloud/s3/S3DataStoreUtils.java | 106 +++++++++---- .../oak/blob/cloud/s3/S3EmulatorSupport.java | 128 +++++++++++++++ .../oak/blob/cloud/s3/S3EmulatorSupportTest.java | 94 +++++++++++ .../jackrabbit/oak/blob/cloud/s3/S3MockRule.java | 109 +++++++++++++ .../jackrabbit/oak/blob/cloud/s3/TestS3Ds.java | 10 ++ .../oak/blob/cloud/s3/TestS3DsCacheOff.java | 23 +++ .../jackrabbit/oak/blob/cloud/s3/UtilsTest.java | 70 ++++++++- oak-doc/src/site/markdown/osgi_config.md | 4 + 13 files changed, 706 insertions(+), 38 deletions(-) diff --git a/oak-blob-cloud/pom.xml b/oak-blob-cloud/pom.xml index 9bccf0d064..5b70e6453f 100644 --- a/oak-blob-cloud/pom.xml +++ b/oak-blob-cloud/pom.xml @@ -487,6 +487,11 @@ <artifactId>org.osgi.service.cm</artifactId> <scope>test</scope> </dependency> + <dependency> + <groupId>org.testcontainers</groupId> + <artifactId>testcontainers</artifactId> + <scope>test</scope> + </dependency> </dependencies> </project> diff --git a/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3Constants.java b/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3Constants.java index 7e70b6f13d..5f7238e3b2 100644 --- a/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3Constants.java +++ b/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3Constants.java @@ -169,6 +169,14 @@ public final class S3Constants { */ public static final String PRESIGNED_HTTP_DOWNLOAD_URI_VERIFY_EXISTS = "presignedHttpDownloadURIVerifyExists"; + /** + * Boolean property to enable path-style access for S3 clients ({@code pathStyleAccess=true}). + * When enabled, requests use {@code http://endpoint/bucket/key} instead of + * {@code https://bucket.endpoint/key}. Defaults to {@code false}. + * Always enabled automatically when the remote storage mode is GCP. + */ + public static final String PATH_STYLE_ACCESS = "pathStyleAccess"; + /** * private constructor so that class cannot initialized from outside. */ diff --git a/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/Utils.java b/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/Utils.java index 6c479076a1..3d5239adab 100644 --- a/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/Utils.java +++ b/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/Utils.java @@ -170,7 +170,7 @@ public final class Utils { .credentialsProvider(Utils.getAwsCredentials(props)) .region(Region.of(Utils.getRegion(props))) .serviceConfiguration(S3Configuration.builder() - .pathStyleAccessEnabled(isGCP) + .pathStyleAccessEnabled(isPathStyleAccessEnabled(props, isGCP)) .chunkedEncodingEnabled(!isGCP) .build()) .build(); @@ -584,12 +584,16 @@ public final class Utils { builder.serviceConfiguration( S3Configuration.builder() - .pathStyleAccessEnabled(isGCP) // enable for GCP + .pathStyleAccessEnabled(isPathStyleAccessEnabled(prop, isGCP)) // always on for GCP; opt-in for S3 via pathStyleAccess property .chunkedEncodingEnabled(!isGCP) // Disable for GCP .useArnRegionEnabled(!isGCP) // Disable for GCP .build()); } + private static boolean isPathStyleAccessEnabled(Properties prop, boolean isGCP) { + return isGCP || Boolean.parseBoolean(prop.getProperty(S3Constants.PATH_STYLE_ACCESS, "false")); + } + // Helper class to hold common Http config private static class HttpClientConfig { final int connectionTimeout; diff --git a/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3DataRecordAccessProviderIT.java b/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3DataRecordAccessProviderIT.java index fa36a39a29..0958b3d941 100644 --- a/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3DataRecordAccessProviderIT.java +++ b/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3DataRecordAccessProviderIT.java @@ -20,6 +20,7 @@ package org.apache.jackrabbit.oak.blob.cloud.s3; import java.io.File; import java.io.IOException; +import java.io.InputStream; import java.net.URI; import javax.net.ssl.HttpsURLConnection; @@ -80,6 +81,11 @@ public class S3DataRecordAccessProviderIT extends AbstractDataRecordAccessProvid @Override protected HttpsURLConnection getHttpsConnection(long length, URI uri) throws IOException { - return S3DataStoreUtils.getHttpsConnection(length, uri); + throw new UnsupportedOperationException("Upload is handled by doHttpsUpload; see S3DataStoreUtils.doHttpUpload"); + } + + @Override + protected void doHttpsUpload(InputStream in, long contentLength, URI uri) throws IOException { + S3DataStoreUtils.doHttpUpload(in, contentLength, uri); } } diff --git a/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3DataRecordAccessProviderTest.java b/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3DataRecordAccessProviderTest.java index cf59f79645..c8f390540d 100644 --- a/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3DataRecordAccessProviderTest.java +++ b/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3DataRecordAccessProviderTest.java @@ -22,20 +22,29 @@ import static org.apache.jackrabbit.oak.blob.cloud.s3.S3DataStoreUtils.getFixtur import static org.apache.jackrabbit.oak.blob.cloud.s3.S3DataStoreUtils.getS3Config; import static org.apache.jackrabbit.oak.blob.cloud.s3.S3DataStoreUtils.getS3DataStore; import static org.apache.jackrabbit.oak.blob.cloud.s3.S3DataStoreUtils.isS3Configured; +import static org.apache.jackrabbit.oak.plugins.blob.datastore.DataStoreUtils.randomStream; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; +import static org.junit.Assume.assumeFalse; import static org.junit.Assume.assumeTrue; import java.io.File; import java.io.IOException; import java.io.InputStream; +import java.net.HttpURLConnection; import java.net.URI; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Properties; import javax.net.ssl.HttpsURLConnection; +import org.apache.jackrabbit.oak.api.blob.BlobDownloadOptions; +import org.apache.jackrabbit.oak.plugins.blob.datastore.directaccess.DataRecordAccessProvider; +import org.apache.jackrabbit.oak.plugins.blob.datastore.directaccess.DataRecordDownloadOptions; import org.apache.jackrabbit.oak.spi.blob.data.DataIdentifier; import org.apache.jackrabbit.oak.spi.blob.data.DataRecord; import org.apache.jackrabbit.oak.spi.blob.data.DataStore; @@ -131,7 +140,12 @@ public class S3DataRecordAccessProviderTest extends AbstractDataRecordAccessProv @Override protected HttpsURLConnection getHttpsConnection(long length, URI uri) throws IOException { - return S3DataStoreUtils.getHttpsConnection(length, uri); + throw new UnsupportedOperationException("S3Mock uses HTTP; upload is handled by doHttpsUpload"); + } + + @Override + protected void doHttpsUpload(InputStream in, long contentLength, URI uri) throws IOException { + S3DataStoreUtils.doHttpUpload(in, contentLength, uri); } @Test @@ -168,4 +182,159 @@ public class S3DataRecordAccessProviderTest extends AbstractDataRecordAccessProv upload = ds.initiateDataRecordUpload(uploadSize, -1); assertEquals(expectedNumURIs, upload.getUploadURIs().size()); } + + @Override + @Test + public void testGetDownloadURIIT() throws DataStoreException, IOException { + assumeTrue("SSE-C doesn't support presigned GET URLs", !isSSECustomerKeyEncryption()); + assumeTrue("S3Presigner does not inherit endpointOverride from S3Client; presigned URLs point to real AWS, not emulator", + !S3EmulatorSupport.isAvailable()); + DataRecord record = null; + DataRecordAccessProvider dataStore = getDataStore(); + try { + InputStream testStream = randomStream(0, 256); + record = doSynchronousAddRecord((DataStore) dataStore, testStream); + URI uri = dataStore.getDownloadURI(record.getIdentifier(), DataRecordDownloadOptions.DEFAULT); + HttpURLConnection conn = (HttpURLConnection) uri.toURL().openConnection(); + try { + conn.setRequestMethod("GET"); + assertEquals(200, conn.getResponseCode()); + + testStream.reset(); + try (InputStream responseStream = conn.getInputStream()) { + assertTrue(Arrays.equals(testStream.readAllBytes(), responseStream.readAllBytes())); + } + } finally { + conn.disconnect(); + } + } finally { + if (null != record) { + doDeleteRecord((DataStore) dataStore, record.getIdentifier()); + } + } + } + + @Override + @Test + public void testGetDownloadURIWithCustomHeadersIT() throws DataStoreException, IOException { + assumeTrue("SSE-C doesn't support presigned GET URLs", !isSSECustomerKeyEncryption()); + assumeTrue("S3Presigner does not inherit endpointOverride from S3Client; presigned URLs point to real AWS, not emulator", + !S3EmulatorSupport.isAvailable()); + String umlautFilename = "Umläutfile.png"; + String umlautFilename_ISO_8859_1 = new String( + StandardCharsets.ISO_8859_1.encode(umlautFilename).array(), + StandardCharsets.ISO_8859_1 + ); + List<String> fileNames = List.of( + "image.png", + "beautiful landscape.png", + "\"filename-with-double-quotes\".png", + "filename-with-one\"double-quote.jpg", + umlautFilename + ); + List<String> iso_8859_1_fileNames = List.of( + "image.png", + "beautiful landscape.png", + "\\\"filename-with-double-quotes\\\".png", + "filename-with-one\\\"double-quote.jpg", + umlautFilename_ISO_8859_1 + ); + List<String> rfc8187_fileNames = List.of( + "image.png", + "beautiful%20landscape.png", + "%22filename-with-double-quotes%22.png", + "filename-with-one%22double-quote.jpg", + "Uml%C3%A4utfile.png" + ); + + DataRecord record = null; + DataRecordAccessProvider dataStore = getDataStore(); + try { + InputStream testStream = randomStream(0, 256); + record = doSynchronousAddRecord((DataStore) dataStore, testStream); + String mimeType = "image/png"; + String dispositionType = "inline"; + for (int i = 0; i < fileNames.size(); i++) { + String fileName = fileNames.get(i); + String iso_8859_1_fileName = iso_8859_1_fileNames.get(i); + String encodedFileName = rfc8187_fileNames.get(i); + DataRecordDownloadOptions downloadOptions = + DataRecordDownloadOptions.fromBlobDownloadOptions( + new BlobDownloadOptions(mimeType, null, fileName, dispositionType) + ); + URI uri = dataStore.getDownloadURI(record.getIdentifier(), downloadOptions); + + HttpURLConnection conn = (HttpURLConnection) uri.toURL().openConnection(); + try { + conn.setRequestMethod("GET"); + assertEquals(200, conn.getResponseCode()); + + assertEquals(mimeType, conn.getHeaderField("Content-Type")); + assertEquals( + String.format("%s; filename=\"%s\"; filename*=UTF-8''%s", + dispositionType, iso_8859_1_fileName, encodedFileName), + conn.getHeaderField("Content-Disposition") + ); + + testStream.reset(); + try (InputStream responseStream = conn.getInputStream()) { + assertTrue(Arrays.equals(testStream.readAllBytes(), responseStream.readAllBytes())); + } + } finally { + conn.disconnect(); + } + } + } finally { + if (null != record) { + doDeleteRecord((DataStore) dataStore, record.getIdentifier()); + } + } + } + + @Override + @Test + public void testSinglePutDirectUploadIT() throws DataRecordUploadException, DataStoreException, IOException { + assumeTrue("S3Presigner does not inherit endpointOverride from S3Client; presigned PUT URLs point to real AWS, not emulator", + !S3EmulatorSupport.isAvailable()); + super.testSinglePutDirectUploadIT(); + } + + @Override + @Test + public void testCompleteAlreadyUploadedBinaryReturnsSameBinaryIT() throws DataStoreException, DataRecordUploadException, IOException { + assumeTrue("S3Presigner does not inherit endpointOverride from S3Client; presigned PUT URLs point to real AWS, not emulator", + !S3EmulatorSupport.isAvailable()); + super.testCompleteAlreadyUploadedBinaryReturnsSameBinaryIT(); + } + + @Override + @Test + public void testGetExpiredReadURIFailsIT() throws DataStoreException, IOException { + assumeTrue("SSE-C doesn't support presigned GET URLs", !isSSECustomerKeyEncryption()); + assumeFalse("S3Presigner does not inherit endpointOverride from S3Client; presigned URLs point to real AWS, not emulator", + S3EmulatorSupport.isAvailable()); + DataRecord record = null; + ConfigurableDataRecordAccessProvider dataStore = getDataStore(); + try { + dataStore.setDirectDownloadURIExpirySeconds(2); + record = doSynchronousAddRecord((DataStore) dataStore, randomStream(0, 256)); + URI uri = dataStore.getDownloadURI(record.getIdentifier(), DataRecordDownloadOptions.DEFAULT); + try { + Thread.sleep(5 * 1000); + } catch (InterruptedException e) { + } + HttpURLConnection conn = (HttpURLConnection) uri.toURL().openConnection(); + try { + conn.setRequestMethod("GET"); + assertEquals(403, conn.getResponseCode()); + } finally { + conn.disconnect(); + } + } finally { + if (null != record) { + doDeleteRecord((DataStore) dataStore, record.getIdentifier()); + } + dataStore.setDirectDownloadURIExpirySeconds(expirySeconds); + } + } } diff --git a/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3DataStoreUtils.java b/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3DataStoreUtils.java index 5b695ebe84..4d30c02cc6 100644 --- a/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3DataStoreUtils.java +++ b/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3DataStoreUtils.java @@ -22,15 +22,16 @@ import java.io.File; import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; +import java.io.OutputStream; +import java.net.HttpURLConnection; import java.net.URI; +import java.net.URLConnection; import java.util.Date; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Properties; -import javax.net.ssl.HttpsURLConnection; - import org.apache.commons.io.IOUtils; import org.apache.jackrabbit.oak.spi.blob.data.DataStore; import org.apache.jackrabbit.oak.commons.PropertiesUtil; @@ -125,9 +126,21 @@ public class S3DataStoreUtils extends DataStoreUtils { props = new Properties(); props.putAll(filtered); } + // Fall back to emulator when no real credentials are present. + if (!hasRealCredentials(props) && S3EmulatorSupport.isAvailable()) { + return S3EmulatorSupport.getEmulatorProperties(); + } return props; } + private static boolean hasRealCredentials(Properties props) { + String accessKey = props.getProperty(S3Constants.ACCESS_KEY, "").trim(); + String secretKey = props.getProperty(S3Constants.SECRET_KEY, "").trim(); + String region = props.getProperty(S3Constants.S3_REGION, "").trim(); + String endpoint = props.getProperty(S3Constants.S3_END_POINT, "").trim(); + return !accessKey.isEmpty() && !secretKey.isEmpty() && (!region.isEmpty() || !endpoint.isEmpty()); + } + public static DataStore getS3DataStore(String className, Properties props, String homeDir) throws Exception { DataStore ds = Class.forName(className).asSubclass(DataStore.class).newInstance(); PropertiesUtil.populate(ds, Utils.asMap(props), false); @@ -150,38 +163,65 @@ public class S3DataStoreUtils extends DataStoreUtils { S3BackendHelper.deleteBucketAndAbortMultipartUploads(bucket, date, props); } - protected static HttpsURLConnection getHttpsConnection(long length, URI uri) throws IOException { - HttpsURLConnection conn = (HttpsURLConnection) uri.toURL().openConnection(); - conn.setDoOutput(true); - conn.setRequestMethod("PUT"); - conn.setRequestProperty("Content-Length", String.valueOf(length)); - - final Properties props = getS3Config(); - DataEncryption encryption = Utils.getDataEncryption(props); - - switch (encryption) { - case SSE_S3: - conn.setRequestProperty("x-amz-server-side-encryption", "AES256"); - System.out.println("Added SSE-S3 header: AES256"); - break; - case SSE_KMS: - conn.setRequestProperty("x-amz-server-side-encryption", "aws:kms"); - System.out.println("Added SSE-KMS header: aws:kms"); - if (Objects.nonNull(props.getProperty(S3Constants.S3_SSE_KMS_KEYID))) { - conn.setRequestProperty("x-amz-server-side-encryption-aws-kms-key-id", props.getProperty(S3Constants.S3_SSE_KMS_KEYID)); - System.out.println("Added KMS Key ID: " + props.getProperty(S3Constants.S3_SSE_KMS_KEYID)); - } - break; - case SSE_C: - String customerKey = props.getProperty(S3Constants.S3_SSE_C_KEY); - String customerKeyMD5 = Utils.calculateMD5(customerKey); - conn.setRequestProperty("x-amz-server-side-encryption-customer-algorithm", "AES256"); - conn.setRequestProperty("x-amz-server-side-encryption-customer-key", customerKey); - conn.setRequestProperty("x-amz-server-side-encryption-customer-key-MD5", customerKeyMD5); - break; - case NONE: - break; + protected static HttpURLConnection getHttpConnection(long length, URI uri) throws IOException { + URLConnection raw = uri.toURL().openConnection(); + if (!(raw instanceof HttpURLConnection)) { + throw new IOException("Expected HTTP(S) connection for URI: " + uri); + } + HttpURLConnection conn = (HttpURLConnection) raw; + try { + conn.setDoOutput(true); + conn.setRequestMethod("PUT"); + conn.setRequestProperty("Content-Length", String.valueOf(length)); + + final Properties props = getS3Config(); + DataEncryption encryption = Utils.getDataEncryption(props); + + switch (encryption) { + case SSE_S3: + conn.setRequestProperty("x-amz-server-side-encryption", "AES256"); + System.out.println("Added SSE-S3 header: AES256"); + break; + case SSE_KMS: + conn.setRequestProperty("x-amz-server-side-encryption", "aws:kms"); + System.out.println("Added SSE-KMS header: aws:kms"); + if (Objects.nonNull(props.getProperty(S3Constants.S3_SSE_KMS_KEYID))) { + conn.setRequestProperty("x-amz-server-side-encryption-aws-kms-key-id", props.getProperty(S3Constants.S3_SSE_KMS_KEYID)); + System.out.println("Added KMS Key ID: " + props.getProperty(S3Constants.S3_SSE_KMS_KEYID)); + } + break; + case SSE_C: + String customerKey = props.getProperty(S3Constants.S3_SSE_C_KEY); + String customerKeyMD5 = Utils.calculateMD5(customerKey); + conn.setRequestProperty("x-amz-server-side-encryption-customer-algorithm", "AES256"); + conn.setRequestProperty("x-amz-server-side-encryption-customer-key", customerKey); + conn.setRequestProperty("x-amz-server-side-encryption-customer-key-MD5", customerKeyMD5); + break; + case NONE: + break; + } + } catch (RuntimeException e) { + conn.disconnect(); + throw e; } return conn; } + + protected static void doHttpUpload(InputStream in, long contentLength, URI uri) throws IOException { + HttpURLConnection conn = getHttpConnection(contentLength, uri); + try { + try (OutputStream out = conn.getOutputStream()) { + IOUtils.copy(in, out); + } + int responseCode = conn.getResponseCode(); + org.junit.Assert.assertTrue(conn.getResponseMessage(), responseCode < 400); + } finally { + InputStream err = conn.getErrorStream(); + if (err != null) { + try { IOUtils.toByteArray(err); } catch (IOException ignored) {} + IOUtils.closeQuietly(err); + } + conn.disconnect(); + } + } } diff --git a/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3EmulatorSupport.java b/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3EmulatorSupport.java new file mode 100644 index 0000000000..d8fc9cdc84 --- /dev/null +++ b/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3EmulatorSupport.java @@ -0,0 +1,128 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.jackrabbit.oak.blob.cloud.s3; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Properties; + +/** + * Lazy singleton that starts an {@link S3MockRule} Docker container on first use and exposes + * S3 emulator properties for use in {@link S3DataStoreUtils#getS3Config()}. + * + * <p>GCP emulator coverage is intentionally out of scope because S3Mock is not compatible + * with Oak's GCP client configuration.</p> + * + * <p>If Docker is unavailable, {@link #isAvailable()} returns {@code false} and tests + * skip rather than fail.</p> + */ +public final class S3EmulatorSupport { + + private static final Logger log = LoggerFactory.getLogger(S3EmulatorSupport.class); + + /** System property reserved for selecting emulator mode; only S3 is currently supported. */ + static final String S3_TEST_MODE_PROP = "s3.test.mode"; + + /** Dummy access key accepted by S3Mock. */ + static final String ACCESS_KEY = "foo"; + + /** Dummy secret key accepted by S3Mock. */ + static final String SECRET_KEY = "bar"; + + /** Default bucket name used when no test-specific bucket is configured. */ + static final String DEFAULT_BUCKET = "s3mock-default-test-bucket"; + + private static volatile S3MockRule instance; + private static volatile boolean startupAttempted = false; + + private S3EmulatorSupport() {} + + /** + * Returns {@code true} if the S3Mock container started successfully and is available for tests. + */ + public static boolean isAvailable() { + if (isUnsupportedMode()) { + return false; + } + ensureStarted(); + return instance != null; + } + + /** + * Returns a {@link Properties} instance configured for the S3 emulator. + * Returns an empty {@link Properties} if the container is not available. + */ + public static Properties getEmulatorProperties() { + if (isUnsupportedMode()) { + return new Properties(); + } + ensureStarted(); + S3MockRule rule = instance; + if (rule == null) { + return new Properties(); + } + String endpoint = rule.getHttpEndpoint(); + Properties props = new Properties(); + props.setProperty(S3Constants.ACCESS_KEY, ACCESS_KEY); + props.setProperty(S3Constants.SECRET_KEY, SECRET_KEY); + props.setProperty(S3Constants.S3_BUCKET, DEFAULT_BUCKET); + props.setProperty(S3Constants.S3_CONN_PROTOCOL, "http"); + props.setProperty(S3Constants.PATH_STYLE_ACCESS, "true"); + props.setProperty(S3Constants.S3_ENCRYPTION, S3Constants.S3_ENCRYPTION_NONE); + props.setProperty(S3Constants.S3_END_POINT, endpoint); + props.setProperty(S3Constants.S3_REGION, "us-east-1"); + props.setProperty(S3Constants.S3_MAX_ERR_RETRY, "3"); + props.setProperty(S3Constants.S3_CONN_TIMEOUT, "10000"); + props.setProperty(S3Constants.S3_SOCK_TIMEOUT, "30000"); + props.setProperty(S3Constants.S3_MAX_CONNS, "10"); + return props; + } + + private static void ensureStarted() { + if (isUnsupportedMode() || instance != null) { + return; + } + synchronized (S3EmulatorSupport.class) { + if (instance != null || startupAttempted) { + return; + } + startupAttempted = true; + S3MockRule rule = new S3MockRule(); + try { + rule.before(); + instance = rule; + Runtime.getRuntime().addShutdownHook(new Thread(() -> { + try { + rule.after(); + } catch (Exception ignored) { + } + })); + } catch (Throwable e) { + try { + rule.after(); + } catch (Exception ignored) { + } + log.warn("S3Mock container failed to start — emulator tests will be skipped", e); + } + } + } + + private static boolean isUnsupportedMode() { + return !"S3".equalsIgnoreCase(System.getProperty(S3_TEST_MODE_PROP, "S3")); + } +} diff --git a/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3EmulatorSupportTest.java b/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3EmulatorSupportTest.java new file mode 100644 index 0000000000..0e58146efc --- /dev/null +++ b/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3EmulatorSupportTest.java @@ -0,0 +1,94 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.jackrabbit.oak.blob.cloud.s3; + +import org.junit.After; +import org.junit.Assert; +import org.junit.Assume; +import org.junit.Before; +import org.junit.Test; + +import java.util.Properties; + +/** + * Tests for {@link S3EmulatorSupport}. + */ +public class S3EmulatorSupportTest { + + private String previousMode; + + @Before + public void setUp() { + previousMode = System.getProperty(S3EmulatorSupport.S3_TEST_MODE_PROP); + } + + @After + public void tearDown() { + if (previousMode == null) { + System.clearProperty(S3EmulatorSupport.S3_TEST_MODE_PROP); + } else { + System.setProperty(S3EmulatorSupport.S3_TEST_MODE_PROP, previousMode); + } + } + + @Test + public void isAvailableReturnsFalseForUnsupportedGcpMode() { + System.setProperty(S3EmulatorSupport.S3_TEST_MODE_PROP, "GCP"); + Assert.assertFalse(S3EmulatorSupport.isAvailable()); + } + + @Test + public void isAvailableReturnsTrueInS3ModeWhenDockerAvailable() { + System.clearProperty(S3EmulatorSupport.S3_TEST_MODE_PROP); + Assume.assumeTrue("Docker is not available", S3EmulatorSupport.isAvailable()); + Assert.assertTrue(S3EmulatorSupport.isAvailable()); + } + + @Test + public void getEmulatorPropertiesReturnsEmptyForUnsupportedGcpMode() { + System.setProperty(S3EmulatorSupport.S3_TEST_MODE_PROP, "GCP"); + Assert.assertTrue(S3EmulatorSupport.getEmulatorProperties().isEmpty()); + } + + @Test + public void getEmulatorPropertiesReturnsEmptyWhenUnavailable() { + // When Docker is not available, isAvailable() is false and getEmulatorProperties() + // must return empty properties rather than throwing. + Assume.assumeFalse("Docker is available", S3EmulatorSupport.isAvailable()); + Properties props = S3EmulatorSupport.getEmulatorProperties(); + Assert.assertTrue("Expected empty properties when emulator is unavailable", props.isEmpty()); + } + + @Test + public void getEmulatorPropertiesInS3ModeContainsRequiredKeys() { + System.clearProperty(S3EmulatorSupport.S3_TEST_MODE_PROP); + Assume.assumeTrue("Docker is not available", S3EmulatorSupport.isAvailable()); + + Properties props = S3EmulatorSupport.getEmulatorProperties(); + Assert.assertNotNull(props.getProperty(S3Constants.ACCESS_KEY)); + Assert.assertNotNull(props.getProperty(S3Constants.SECRET_KEY)); + Assert.assertNotNull(props.getProperty(S3Constants.S3_END_POINT)); + Assert.assertNotNull(props.getProperty(S3Constants.S3_REGION)); + Assert.assertNotNull(props.getProperty(S3Constants.S3_BUCKET)); + Assert.assertEquals("true", props.getProperty(S3Constants.PATH_STYLE_ACCESS)); + Assert.assertNotNull(props.getProperty(S3Constants.S3_MAX_ERR_RETRY)); + Assert.assertNotNull(props.getProperty(S3Constants.S3_CONN_TIMEOUT)); + Assert.assertEquals("http", props.getProperty(S3Constants.S3_CONN_PROTOCOL)); + Assert.assertEquals(S3Constants.S3_ENCRYPTION_NONE, props.getProperty(S3Constants.S3_ENCRYPTION)); + Assert.assertNull("S3 mode must not set MODE", props.get(S3Constants.MODE)); + } +} diff --git a/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3MockRule.java b/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3MockRule.java new file mode 100644 index 0000000000..d1878c7749 --- /dev/null +++ b/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3MockRule.java @@ -0,0 +1,109 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.jackrabbit.oak.blob.cloud.s3; + +import org.junit.Assume; +import org.junit.rules.ExternalResource; +import org.junit.runner.Description; +import org.junit.runners.model.MultipleFailureException; +import org.junit.runners.model.Statement; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.utility.DockerImageName; + +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.atomic.AtomicReference; + +/** + * JUnit rule that starts an Adobe S3Mock Docker container for emulator-based S3 integration tests. + * Mirrors the pattern used by {@code AzuriteDockerRule} in oak-blob-cloud-azure. + * + * <p>On Docker failure, the rule calls {@link Assume#assumeNoException} so tests are skipped + * rather than failed, preserving the same behaviour as when no credentials are configured.</p> + */ +public class S3MockRule extends ExternalResource { + + private static final DockerImageName DOCKER_IMAGE = DockerImageName.parse("adobe/s3mock:5.0.0"); + static final int HTTP_PORT = 9090; + private final AtomicReference<Exception> startupException = new AtomicReference<>(); + + private GenericContainer<?> container; + + @Override + protected void before() throws Throwable { + startupException.set(null); + container = new GenericContainer<>(DOCKER_IMAGE) + .withExposedPorts(HTTP_PORT) + .withStartupTimeout(Duration.ofSeconds(60)); + try { + container.start(); + } catch (Exception e) { + startupException.set(e); + throw e; + } + } + + @Override + protected void after() { + if (container != null) { + container.stop(); + } + } + + @Override + public Statement apply(Statement base, Description description) { + return new Statement() { + @Override + public void evaluate() throws Throwable { + try { + before(); + } catch (Exception e) { + Assume.assumeNoException(e); + throw e; + } + List<Throwable> errors = new ArrayList<>(); + try { + base.evaluate(); + } catch (Throwable t) { + errors.add(t); + } finally { + try { + after(); + } catch (Throwable t) { + errors.add(t); + } + } + MultipleFailureException.assertEmpty(errors); + } + }; + } + + /** + * Returns the mapped HTTP port on the host for the running S3Mock container. + */ + public int getHttpPort() { + return container.getMappedPort(HTTP_PORT); + } + + /** + * Returns the HTTP endpoint URL for the running S3Mock container, e.g. {@code http://127.0.0.1:PORT}. + */ + public String getHttpEndpoint() { + return "http://127.0.0.1:"; + getHttpPort(); + } +} diff --git a/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/TestS3Ds.java b/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/TestS3Ds.java index 00e79a73c2..51d55ec985 100644 --- a/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/TestS3Ds.java +++ b/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/TestS3Ds.java @@ -315,6 +315,16 @@ public class TestS3Ds extends AbstractDataStoreTest { public void testDeleteAllOlderThan() { } + // deleteRecord removes the object from S3 but leaves the local file cache intact. + // getRecordIfStored then returns the cached copy, causing a spurious failure. + // The cache-off subclass (TestS3DsCacheOff) re-enables this test where it is valid. + @Override + public void testDeleteRecord() { + Assume.assumeFalse("S3 local cache masks deletions; not verifiable against the emulator", + S3EmulatorSupport.isAvailable()); + super.testDeleteRecord(); + } + // helper methods private PutObjectResponse httpPut(@Nullable DataRecordUpload uploadContext, InputStream inputstream, long length) { diff --git a/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/TestS3DsCacheOff.java b/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/TestS3DsCacheOff.java index fb62f1ee8e..eea9f30cfd 100644 --- a/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/TestS3DsCacheOff.java +++ b/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/TestS3DsCacheOff.java @@ -17,7 +17,9 @@ package org.apache.jackrabbit.oak.blob.cloud.s3; import org.apache.jackrabbit.oak.spi.blob.data.CachingDataStore; +import org.junit.Assume; import org.junit.Before; +import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -40,4 +42,25 @@ public class TestS3DsCacheOff extends TestS3Ds { props.setProperty("cacheSize", "0"); super.setUp(); } + + // Re-enable: with cache off, deleteRecord is immediately visible via S3 so the + // assertion that getRecordIfStored returns null after deletion is valid here. + @Override + @Test + public void testDeleteRecord() { + try { + doDeleteRecordTest(); + } catch (Exception e) { + org.junit.Assert.fail(e.getMessage()); + } + } + + // S3Backend updates duplicate records via CopyObject (copy-to-self). S3Mock does not + // support that operation, so this test cannot run against the emulator with cache off. + @Override + public void testAddDuplicateRecord() { + Assume.assumeFalse("S3Mock does not support CopyObject used by S3Backend for duplicate record updates", + S3EmulatorSupport.isAvailable()); + super.testAddDuplicateRecord(); + } } diff --git a/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/UtilsTest.java b/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/UtilsTest.java index 580c111e3c..fc4b999464 100644 --- a/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/UtilsTest.java +++ b/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/UtilsTest.java @@ -18,8 +18,13 @@ package org.apache.jackrabbit.oak.blob.cloud.s3; import org.junit.Assert; +import org.junit.Assume; import org.junit.Test; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.OutputStream; import java.net.URI; import java.util.Properties; @@ -201,4 +206,67 @@ public class UtilsTest { Assert.assertEquals(S3Backend.RemoteStorageMode.GCP, props.get(S3Constants.MODE)); } -} \ No newline at end of file + @Test + public void testSetRemoteStorageModeDefaultsPreSetGCPToS3() { + Properties props = new Properties(); + props.setProperty("s3EndPoint", "http://127.0.0.1:9090";); + props.put(S3Constants.MODE, S3Backend.RemoteStorageMode.GCP); + Utils.setRemoteStorageMode(props); + Assert.assertEquals(S3Backend.RemoteStorageMode.S3, props.get(S3Constants.MODE)); + } + + @Test + public void testSetRemoteStorageModeDefaultsBlankModeToS3() { + Properties props = new Properties(); + props.setProperty("s3EndPoint", "https://s3.amazonaws.com";); + props.setProperty(S3Constants.MODE, ""); + Utils.setRemoteStorageMode(props); + Assert.assertEquals(S3Backend.RemoteStorageMode.S3, props.get(S3Constants.MODE)); + } + + @Test + public void testPathStyleAccessConstantHasExpectedStringValue() { + Assert.assertEquals("pathStyleAccess", S3Constants.PATH_STYLE_ACCESS); + } + + @Test + public void testPathStyleAccessDefaultsToFalseWhenPropertyAbsent() { + Properties props = new Properties(); + Assert.assertFalse(Boolean.parseBoolean(props.getProperty(S3Constants.PATH_STYLE_ACCESS, "false"))); + } + + @Test + public void testPathStyleAccessTrueWhenPropertySetToTrue() { + Properties props = new Properties(); + props.setProperty(S3Constants.PATH_STYLE_ACCESS, "true"); + Assert.assertTrue(Boolean.parseBoolean(props.getProperty(S3Constants.PATH_STYLE_ACCESS, "false"))); + } + + @Test + public void isS3ConfiguredReturnsFalseForPartialCredentialsWithNoRegionOrEndpoint() throws IOException { + // accessKey + secretKey without region or endpoint must not count as real credentials + File tmp = File.createTempFile("s3test-partial-creds", ".properties"); + String previousConfig = System.getProperty("s3.config"); + try { + Properties partial = new Properties(); + partial.setProperty(S3Constants.ACCESS_KEY, "somekey"); + partial.setProperty(S3Constants.SECRET_KEY, "somesecret"); + try (OutputStream out = new FileOutputStream(tmp)) { + partial.store(out, null); + } + System.setProperty("s3.config", tmp.getAbsolutePath()); + // Skip when emulator is available — getS3Config() would fall back to it, making isS3Configured() true + Assume.assumeFalse("Emulator available — would override partial creds", S3EmulatorSupport.isAvailable()); + Assert.assertFalse("Partial credentials (no region/endpoint) must not be treated as real", + S3DataStoreUtils.isS3Configured()); + } finally { + if (previousConfig == null) { + System.clearProperty("s3.config"); + } else { + System.setProperty("s3.config", previousConfig); + } + tmp.delete(); + } + } + +} diff --git a/oak-doc/src/site/markdown/osgi_config.md b/oak-doc/src/site/markdown/osgi_config.md index 67c471500a..0b8ae52598 100644 --- a/oak-doc/src/site/markdown/osgi_config.md +++ b/oak-doc/src/site/markdown/osgi_config.md @@ -367,6 +367,10 @@ s3Region s3EndPoint : S3 rest API endpoint. Can help reduce latency of redirection from standard endpoint if a different region configured. +pathStyleAccess +: Default - `false` +: Set to `true` to use path-style S3 requests (`http://endpoint/bucket/key`) instead of virtual-host-style requests. This is useful for S3-compatible endpoints that do not support bucket names in the host. Note: always enabled automatically when the remote storage mode is GCP; the property has no effect in that case. + connectionTimeout : S3 connection timeout. See [AWS S3 documentation](https://docs.aws.amazon.com/sdk-for-java/v1/developer-guide/section-client-configuration.html).
