This is an automated email from the ASF dual-hosted git repository. imaxon pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 0729764841c54adf579c02024f2226c249135c2c Author: Ian Maxon <[email protected]> AuthorDate: Thu Sep 18 08:43:08 2025 -0700 [ASTERIXDB-3640][TEST]Use s3mock in testcontainers - user model changes: no - storage format changes: no - interface changes: no Details: - Refactor all tests that use Adobe S3 Mock to use the containerized version instead of using it as a Java dependency - Make the cloud storage configuration templated to deal with the fact testcontainers listens on a random ephemeral port each time - Add compensating drop statements to some test suites that were previously leaking entities into the Default namespace Ext-ref: MB-68268 Change-Id: Iad1a6f9331fd844f3e3c04b8f56e811e4117f83a Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/20280 Integration-Tests: Jenkins <[email protected]> Reviewed-by: Michael Blow <[email protected]> Tested-by: Ian Maxon <[email protected]> Reviewed-by: Ian Maxon <[email protected]> --- asterixdb/asterix-app/pom.xml | 14 ++-- .../api/common/CloudStorageIntegrationUtil.java | 9 ++- .../apache/asterix/api/common/LocalCloudUtil.java | 10 ++- .../api/common/LocalCloudUtilAdobeMock.java | 90 +++++++++++++++------- .../AtomicMetadataTransactionWithoutWALTest.java | 8 +- .../AtomicStatementsCancellationTest.java | 23 ++++-- .../test/cloud_storage/CloudPythonTest.java | 19 ++++- .../test/cloud_storage/CloudStorageAzTest.java | 5 +- .../test/cloud_storage/CloudStorageGCSTest.java | 5 +- .../test/cloud_storage/CloudStorageSparseTest.java | 21 +++-- .../test/cloud_storage/CloudStorageTest.java | 27 ++++--- .../cloud_storage/CloudStorageUnstableTest.java | 10 ++- .../cloud_storage/GCSCloudStorageUnstableTest.java | 6 +- .../apache/asterix/test/common/TestConstants.java | 1 + .../apache/asterix/test/common/TestExecutor.java | 13 +++- ...orage-sparse.conf => cc-cloud-storage-main.ftl} | 6 +- ...parse.conf => cc-cloud-storage-sparse.conf.ftl} | 2 +- ...orage-sparse.conf => cc-cloud-storage.conf.ftl} | 4 +- .../post-non-query/post-non-query.099.post.http} | 10 ++- ... => array_slice_bracket_notation.099.ddl.sqlpp} | 3 +- .../array_slice_bracket_notation.1.ddl.sqlpp | 0 .../array_slice_bracket_notation.2.update.sqlpp | 0 .../array_slice_bracket_notation.3.query.sqlpp | 0 .../array_slice_bracket_notation.4.ddl.sqlpp | 0 .../external-collection/test.010.ddl.sqlpp | 8 +- .../hash_join_exchange.099.ddl.sqlpp} | 7 +- .../hash_join_exchange_1.099.ddl.sqlpp} | 7 +- .../join-with-empty-dataset.099.ddl.sqlpp} | 5 +- ...uery.sqlpp => limit_after_offset.099.ddl.sqlpp} | 3 +- .../limit_after_offset.1.ddl.sqlpp} | 4 +- .../limit_after_offset.2.query.sqlpp} | 5 +- .../api/post-non-query/post-non-query.099.adm | 1 + .../runtimets/testsuite_cloud_storage.xml | 2 + asterixdb/pom.xml | 10 +-- 34 files changed, 226 insertions(+), 112 deletions(-) diff --git a/asterixdb/asterix-app/pom.xml b/asterixdb/asterix-app/pom.xml index d5d1c792f2..f1bbb375f9 100644 --- a/asterixdb/asterix-app/pom.xml +++ b/asterixdb/asterix-app/pom.xml @@ -1137,14 +1137,8 @@ </dependency> <dependency> <groupId>com.adobe.testing</groupId> - <artifactId>s3mock</artifactId> + <artifactId>s3mock-testcontainers</artifactId> <scope>test</scope> - <exclusions> - <exclusion> - <groupId>org.springframework.boot</groupId> - <artifactId>spring-boot-starter-logging</artifactId> - </exclusion> - </exclusions> </dependency> <!-- Mock for AWS S3 --> <dependency> @@ -1201,5 +1195,11 @@ <artifactId>hadoop-client-runtime</artifactId> <scope>test</scope> </dependency> + <dependency> + <groupId>org.freemarker</groupId> + <artifactId>freemarker</artifactId> + <version>2.3.31</version> + <scope>test</scope> + </dependency> </dependencies> </project> diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/common/CloudStorageIntegrationUtil.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/common/CloudStorageIntegrationUtil.java index d49fe266a8..b86730b07d 100644 --- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/common/CloudStorageIntegrationUtil.java +++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/common/CloudStorageIntegrationUtil.java @@ -19,18 +19,25 @@ package org.apache.asterix.api.common; import static org.apache.asterix.api.common.AsterixHyracksIntegrationUtil.LoggerHolder.LOGGER; +import static org.apache.asterix.api.common.LocalCloudUtilAdobeMock.fillConfigTemplate; +import static org.apache.asterix.test.cloud_storage.CloudStorageTest.MOCK_SERVER_HOSTNAME_FRAGMENT; import static org.apache.hyracks.util.file.FileUtil.joinPath; +import com.adobe.testing.s3mock.testcontainers.S3MockContainer; + public class CloudStorageIntegrationUtil extends AsterixHyracksIntegrationUtil { public static final String RESOURCES_PATH = joinPath(getProjectPath().toString(), "src", "test", "resources"); public static final String CONFIG_FILE = joinPath(RESOURCES_PATH, "cc-cloud-storage-main.conf"); + public static final String CONFIG_FILE_TEMPLATE = joinPath(RESOURCES_PATH, "cc-cloud-storage-main.ftl"); public static void main(String[] args) throws Exception { boolean cleanStart = Boolean.getBoolean("cleanup.start"); LocalCloudUtilAdobeMock.startS3CloudEnvironment(cleanStart); final AsterixHyracksIntegrationUtil integrationUtil = new AsterixHyracksIntegrationUtil(); - try { + try (S3MockContainer s3Mock = LocalCloudUtilAdobeMock.startS3CloudEnvironment(cleanStart)) { + fillConfigTemplate(MOCK_SERVER_HOSTNAME_FRAGMENT + s3Mock.getHttpServerPort(), CONFIG_FILE_TEMPLATE, + CONFIG_FILE); integrationUtil.run(cleanStart, Boolean.getBoolean("cleanup.shutdown"), System.getProperty("external.lib", ""), CONFIG_FILE); } catch (Exception e) { diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/common/LocalCloudUtil.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/common/LocalCloudUtil.java index d33617b736..289658ead1 100644 --- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/common/LocalCloudUtil.java +++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/common/LocalCloudUtil.java @@ -21,14 +21,18 @@ package org.apache.asterix.api.common; import static org.apache.hyracks.util.file.FileUtil.joinPath; import java.io.File; +import java.net.URI; import org.apache.commons.io.FileUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import io.findify.s3mock.S3Mock; +import software.amazon.awssdk.auth.credentials.AnonymousCredentialsProvider; import software.amazon.awssdk.core.sync.RequestBody; +import software.amazon.awssdk.regions.Region; import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.S3ClientBuilder; import software.amazon.awssdk.services.s3.model.CreateBucketRequest; import software.amazon.awssdk.services.s3.model.DeleteBucketRequest; import software.amazon.awssdk.services.s3.model.DeleteObjectRequest; @@ -73,7 +77,11 @@ public class LocalCloudUtil { s3MockServer.start(); LOGGER.info("S3 mock server started successfully"); - S3Client client = LocalCloudUtilAdobeMock.getS3Client(); + S3ClientBuilder builder = S3Client.builder(); + URI endpoint = URI.create(MOCK_SERVER_ENDPOINT); // endpoint pointing to S3 mock server + builder.region(Region.of(MOCK_SERVER_REGION)).credentialsProvider(AnonymousCredentialsProvider.create()) + .endpointOverride(endpoint); + S3Client client = builder.build(); client.createBucket(CreateBucketRequest.builder().bucket(CLOUD_STORAGE_BUCKET).build()); LOGGER.info("Created bucket {} for cloud storage", CLOUD_STORAGE_BUCKET); diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/common/LocalCloudUtilAdobeMock.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/common/LocalCloudUtilAdobeMock.java index 8790632416..8509afff0f 100644 --- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/common/LocalCloudUtilAdobeMock.java +++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/common/LocalCloudUtilAdobeMock.java @@ -18,87 +18,121 @@ */ package org.apache.asterix.api.common; -import static org.apache.asterix.api.common.LocalCloudUtil.MOCK_SERVER_ENDPOINT; import static org.apache.asterix.api.common.LocalCloudUtil.MOCK_SERVER_REGION; +import java.io.FileReader; +import java.io.FileWriter; +import java.io.IOException; +import java.io.Writer; import java.net.URI; -import java.util.HashMap; +import java.nio.file.Files; +import java.nio.file.Path; import java.util.Map; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import com.adobe.testing.s3mock.S3MockApplication; +import com.adobe.testing.s3mock.testcontainers.S3MockContainer; +import freemarker.template.Configuration; +import freemarker.template.Template; +import freemarker.template.TemplateException; import software.amazon.awssdk.auth.credentials.AnonymousCredentialsProvider; import software.amazon.awssdk.regions.Region; import software.amazon.awssdk.services.s3.S3Client; import software.amazon.awssdk.services.s3.S3ClientBuilder; +import software.amazon.awssdk.services.s3.S3Configuration; import software.amazon.awssdk.services.s3.model.CreateBucketRequest; +import software.amazon.awssdk.services.s3.model.HeadBucketRequest; +import software.amazon.awssdk.services.s3.model.NoSuchBucketException; -// With adobe mock, the s3 objects will be found in /tmp or /var folder +// This runs Adobe S3Mock in a docker container, with the bucket storage directory under target/s3mock. // Search for line "Successfully created {} as root folder" in the info log file -// or else just call the aws cli on localhost:8001 -// eg: aws s3 ls --endpoint http://localhost:8001 +// A randomized port is chosen each time for the S3 endpoint, but it is always on localhost. +// You can inspect the real port mapping (e.g. 45799 -> 9090) via 'docker ps' +// Then to connect, normal S3 utilities work, such as aws s3 ls --endpoint http://localhost:45799 public class LocalCloudUtilAdobeMock { private static final Logger LOGGER = LogManager.getLogger(); - private static final int MOCK_SERVER_PORT = 8001; - private static final int MOCK_SERVER_PORT_HTTPS = 8002; public static final String CLOUD_STORAGE_BUCKET = "cloud-storage-container"; + public static final String S3MOCK_VERSION_TAG = "4.7.0"; public static final String PLAYGROUND_BUCKET = "playground"; - private static S3MockApplication s3Mock; + public static final String CLOUD_URL_KEY = "cloudUrl"; + private static S3MockContainer s3Mock; private LocalCloudUtilAdobeMock() { throw new AssertionError("Do not instantiate"); } - public static void main(String[] args) { + public static void main(String[] args) throws IOException { String cleanStartString = System.getProperty("cleanup.start", "true"); boolean cleanStart = Boolean.parseBoolean(cleanStartString); // Change to 'true' if you want to delete "s3mock" folder on start startS3CloudEnvironment(cleanStart); } - public static S3MockApplication startS3CloudEnvironment(boolean cleanStart) { + public static S3MockContainer startS3CloudEnvironment(boolean cleanStart) throws IOException { return startS3CloudEnvironment(cleanStart, false); } - public static S3MockApplication startS3CloudEnvironment(boolean cleanStart, boolean createPlaygroundContainer) { + public static S3MockContainer startS3CloudEnvironment(boolean cleanStart, boolean createPlaygroundContainer) + throws IOException { // Starting S3 mock server to be used instead of real S3 server LOGGER.info("Starting S3 mock server"); - - Map<String, Object> properties = new HashMap<>(); - properties.put(S3MockApplication.PROP_HTTP_PORT, MOCK_SERVER_PORT); - properties.put(S3MockApplication.PROP_HTTPS_PORT, MOCK_SERVER_PORT_HTTPS); - properties.put(S3MockApplication.PROP_SILENT, false); - LocalCloudUtil.stopS3MockServer(); - s3Mock = S3MockApplication.start(properties); - + s3Mock = new S3MockContainer(S3MOCK_VERSION_TAG).withRetainFilesOnExit(!cleanStart); + if (!cleanStart) { + Path s3MockDataDir = Path.of("target", "s3mock"); + boolean existingData = s3MockDataDir.toFile().exists(); + if (!existingData) { + Files.createDirectory(s3MockDataDir); + } + //be VERY careful with this path. the last component will be entirely deleted + //if RetainFilesOnExist is false + s3Mock.withVolumeAsRoot(s3MockDataDir.toAbsolutePath()); + } + s3Mock.start(); LOGGER.info("S3 mock server started successfully"); S3ClientBuilder builder = S3Client.builder(); - URI endpoint = URI.create(MOCK_SERVER_ENDPOINT); // endpoint pointing to S3 mock server + URI endpoint = URI.create(s3Mock.getHttpEndpoint()); // endpoint pointing to S3 mock server builder.region(Region.of(MOCK_SERVER_REGION)).credentialsProvider(AnonymousCredentialsProvider.create()) .endpointOverride(endpoint); - S3Client client = builder.build(); - client.createBucket(CreateBucketRequest.builder().bucket(CLOUD_STORAGE_BUCKET).build()); - LOGGER.info("Created bucket {} for cloud storage", CLOUD_STORAGE_BUCKET); - + S3Client client = + builder.serviceConfiguration(S3Configuration.builder().pathStyleAccessEnabled(true).build()).build(); + createIfNotExists(CLOUD_STORAGE_BUCKET, client); if (createPlaygroundContainer) { - client.createBucket(CreateBucketRequest.builder().bucket(PLAYGROUND_BUCKET).build()); - LOGGER.info("Created bucket {}", PLAYGROUND_BUCKET); + createIfNotExists(PLAYGROUND_BUCKET, client); } client.close(); return s3Mock; } + public static void createIfNotExists(String bucketName, S3Client client) { + try { + client.headBucket(HeadBucketRequest.builder().bucket(bucketName).build()); + } catch (NoSuchBucketException nsbe) { + //i don't want to use exceptions as control flow, yet here we are + client.createBucket(CreateBucketRequest.builder().bucket(bucketName).build()); + LOGGER.info("Created bucket {}", bucketName); + } + } + + public static void fillConfigTemplate(String cloudUrl, String templatePath, String configPath) + throws IOException, TemplateException { + try (FileReader tmplt = new FileReader(templatePath); Writer result = new FileWriter(configPath)) { + Template cfg = new Template("cc.conf", tmplt, new Configuration(Configuration.VERSION_2_3_31)); + cfg.process(Map.of(CLOUD_URL_KEY, cloudUrl), result); + } + + } + public static void shutdownSilently() { if (s3Mock != null) { try { LOGGER.info("test cleanup, stopping S3 mock server"); s3Mock.stop(); + s3Mock.close(); LOGGER.info("test cleanup, stopped S3 mock server"); } catch (Exception ex) { // do nothing @@ -109,7 +143,7 @@ public class LocalCloudUtilAdobeMock { public static S3Client getS3Client() { S3ClientBuilder builder = S3Client.builder(); - URI endpoint = URI.create(MOCK_SERVER_ENDPOINT); + URI endpoint = URI.create(s3Mock.getHttpEndpoint()); builder.region(Region.of(MOCK_SERVER_REGION)).credentialsProvider(AnonymousCredentialsProvider.create()) .endpointOverride(endpoint); S3Client client = builder.build(); diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/atomic_statements/AtomicMetadataTransactionWithoutWALTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/atomic_statements/AtomicMetadataTransactionWithoutWALTest.java index b38d82b0f4..debe200367 100644 --- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/atomic_statements/AtomicMetadataTransactionWithoutWALTest.java +++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/atomic_statements/AtomicMetadataTransactionWithoutWALTest.java @@ -18,6 +18,9 @@ */ package org.apache.asterix.test.atomic_statements; +import static org.apache.asterix.api.common.LocalCloudUtilAdobeMock.fillConfigTemplate; +import static org.apache.asterix.test.cloud_storage.CloudStorageTest.CONFIG_FILE_TEMPLATE; +import static org.apache.asterix.test.cloud_storage.CloudStorageTest.MOCK_SERVER_HOSTNAME_FRAGMENT; import static org.apache.hyracks.util.file.FileUtil.joinPath; import java.io.InputStream; @@ -36,6 +39,7 @@ import org.junit.Assert; import org.junit.Before; import org.junit.Test; +import com.adobe.testing.s3mock.testcontainers.S3MockContainer; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; @@ -55,7 +59,9 @@ public class AtomicMetadataTransactionWithoutWALTest { @Before public void setUp() throws Exception { boolean cleanStart = Boolean.getBoolean("cleanup.start"); - LocalCloudUtilAdobeMock.startS3CloudEnvironment(cleanStart); + S3MockContainer s3Mock = LocalCloudUtilAdobeMock.startS3CloudEnvironment(cleanStart); + fillConfigTemplate(MOCK_SERVER_HOSTNAME_FRAGMENT + s3Mock.getHttpServerPort(), CONFIG_FILE_TEMPLATE, + CONFIG_FILE); integrationUtil.setGracefulShutdown(false); integrationUtil.init(true, CONFIG_FILE); } diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/atomic_statements/AtomicStatementsCancellationTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/atomic_statements/AtomicStatementsCancellationTest.java index 02e9fd0d28..9227534a2d 100644 --- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/atomic_statements/AtomicStatementsCancellationTest.java +++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/atomic_statements/AtomicStatementsCancellationTest.java @@ -19,10 +19,9 @@ package org.apache.asterix.test.atomic_statements; import static java.nio.charset.StandardCharsets.UTF_8; +import static org.apache.asterix.api.common.LocalCloudUtilAdobeMock.fillConfigTemplate; import static org.apache.asterix.api.http.server.QueryServiceRequestParameters.Parameter.CLIENT_ID; -import static org.apache.hyracks.util.file.FileUtil.joinPath; -import java.io.File; import java.io.InputStream; import java.net.URI; import java.util.ArrayList; @@ -38,7 +37,9 @@ import java.util.concurrent.Future; import org.apache.asterix.api.common.AsterixHyracksIntegrationUtil; import org.apache.asterix.api.common.LocalCloudUtilAdobeMock; import org.apache.asterix.common.TestDataUtil; +import org.apache.asterix.common.config.GlobalConfig; import org.apache.asterix.common.utils.Servlets; +import org.apache.asterix.test.common.TestConstants; import org.apache.asterix.test.common.TestExecutor; import org.apache.asterix.testframework.context.TestCaseContext; import org.apache.asterix.testframework.xml.ParameterTypeEnum; @@ -49,27 +50,33 @@ import org.junit.Assert; import org.junit.Before; import org.junit.Test; +import com.adobe.testing.s3mock.testcontainers.S3MockContainer; + public class AtomicStatementsCancellationTest { private final ExecutorService executor = Executors.newSingleThreadExecutor(); - private static final String TEST_CONFIG_FILE_NAME = "cc-cloud-storage.conf"; + public static final String CONFIG_FILE_TEMPLATE = "src/test/resources/cc-cloud-storage.conf.ftl"; + public static final String CONFIG_FILE = "target/cc-cloud-storage.conf"; private static final TestExecutor TEST_EXECUTOR = new TestExecutor(); - private static final String TEST_CONFIG_PATH = - joinPath(System.getProperty("user.dir"), "src", "test", "resources");; - private static final String TEST_CONFIG_FILE_PATH = TEST_CONFIG_PATH + File.separator + TEST_CONFIG_FILE_NAME; private static final AsterixHyracksIntegrationUtil integrationUtil = new AsterixHyracksIntegrationUtil(); private static final String DATASET_NAME = "ds_0"; private static final int BATCH_SIZE = 20000; private static final int NUM_UPSERTS = 100; + public static final String MOCK_SERVER_HOSTNAME_FRAGMENT = "http://127.0.0.1:";; @Before public void setUp() throws Exception { boolean cleanStart = true; - LocalCloudUtilAdobeMock.startS3CloudEnvironment(cleanStart); + S3MockContainer s3Mock = LocalCloudUtilAdobeMock.startS3CloudEnvironment(cleanStart); + fillConfigTemplate(MOCK_SERVER_HOSTNAME_FRAGMENT + s3Mock.getHttpServerPort(), CONFIG_FILE_TEMPLATE, + CONFIG_FILE); + System.setProperty(TestConstants.S3_SERVICE_ENDPOINT_KEY, + MOCK_SERVER_HOSTNAME_FRAGMENT + s3Mock.getHttpServerPort()); integrationUtil.setGracefulShutdown(true); - integrationUtil.init(true, TEST_CONFIG_FILE_PATH); + integrationUtil.init(true, CONFIG_FILE); + System.setProperty(GlobalConfig.CONFIG_FILE_PROPERTY, CONFIG_FILE); createDatasets(); } diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/cloud_storage/CloudPythonTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/cloud_storage/CloudPythonTest.java index e9695197d6..00dab0eeb3 100644 --- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/cloud_storage/CloudPythonTest.java +++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/cloud_storage/CloudPythonTest.java @@ -18,6 +18,8 @@ */ package org.apache.asterix.test.cloud_storage; +import static org.apache.asterix.api.common.LocalCloudUtilAdobeMock.fillConfigTemplate; +import static org.apache.asterix.test.cloud_storage.CloudStorageTest.MOCK_SERVER_HOSTNAME_FRAGMENT; import static org.apache.asterix.test.runtime.ExternalPythonFunctionIT.setNcEndpoints; import java.util.Collection; @@ -25,6 +27,7 @@ import java.util.List; import org.apache.asterix.api.common.LocalCloudUtilAdobeMock; import org.apache.asterix.common.config.GlobalConfig; +import org.apache.asterix.test.common.TestConstants; import org.apache.asterix.test.common.TestExecutor; import org.apache.asterix.test.runtime.LangExecutionUtil; import org.apache.asterix.testframework.context.TestCaseContext; @@ -42,6 +45,8 @@ import org.junit.runners.MethodSorters; import org.junit.runners.Parameterized; import org.junit.runners.Parameterized.Parameters; +import com.adobe.testing.s3mock.testcontainers.S3MockContainer; + /** * Run tests in cloud deployment environment */ @@ -54,8 +59,9 @@ public class CloudPythonTest { private final TestCaseContext tcCtx; private static final String SUITE_TESTS = "testsuite_it_python.xml"; private static final String ONLY_TESTS = "testsuite_cloud_storage_only.xml"; - private static final String CONFIG_FILE_NAME = "src/test/resources/cc-cloud-storage.conf"; private static final String DELTA_RESULT_PATH = "results_cloud"; + public static final String CONFIG_FILE_TEMPLATE = "src/test/resources/cc-cloud-storage.conf.ftl"; + public static final String CONFIG_FILE = "target/cc-cloud-storage.conf"; private static final String EXCLUDED_TESTS = "MP"; public CloudPythonTest(TestCaseContext tcCtx) { @@ -64,18 +70,23 @@ public class CloudPythonTest { @BeforeClass public static void setUp() throws Exception { - LocalCloudUtilAdobeMock.startS3CloudEnvironment(true); + S3MockContainer s3Mock = LocalCloudUtilAdobeMock.startS3CloudEnvironment(true); + fillConfigTemplate(MOCK_SERVER_HOSTNAME_FRAGMENT + s3Mock.getHttpServerPort(), CONFIG_FILE_TEMPLATE, + CONFIG_FILE); + System.setProperty(TestConstants.S3_SERVICE_ENDPOINT_KEY, + MOCK_SERVER_HOSTNAME_FRAGMENT + s3Mock.getHttpServerPort()); TestExecutor testExecutor = new TestExecutor(DELTA_RESULT_PATH); testExecutor.executorId = "cloud"; testExecutor.stripSubstring = "//DB:"; - LangExecutionUtil.setUp(CONFIG_FILE_NAME, testExecutor); + LangExecutionUtil.setUp(CONFIG_FILE, testExecutor); setNcEndpoints(testExecutor); - System.setProperty(GlobalConfig.CONFIG_FILE_PROPERTY, CONFIG_FILE_NAME); + System.setProperty(GlobalConfig.CONFIG_FILE_PROPERTY, CONFIG_FILE); } @AfterClass public static void tearDown() throws Exception { LangExecutionUtil.tearDown(); + LocalCloudUtilAdobeMock.shutdownSilently(); } @Parameters(name = "CloudPythonTest {index}: {0}") diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/cloud_storage/CloudStorageAzTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/cloud_storage/CloudStorageAzTest.java index 2db2ef5348..d3f766e9d1 100644 --- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/cloud_storage/CloudStorageAzTest.java +++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/cloud_storage/CloudStorageAzTest.java @@ -19,6 +19,7 @@ package org.apache.asterix.test.cloud_storage; import static org.apache.asterix.api.common.LocalCloudUtil.CLOUD_STORAGE_BUCKET; +import static org.apache.asterix.test.cloud_storage.CloudStorageGCSTest.S3_ONLY; import java.util.ArrayList; import java.util.Collection; @@ -71,7 +72,6 @@ public class CloudStorageAzTest { @BeforeClass public static void setUp() throws Exception { - LocalCloudUtilAdobeMock.startS3CloudEnvironment(true, true); String endpointString = "http://127.0.0.1:15055/devstoreaccount1/"; + CLOUD_STORAGE_BUCKET; final String accKey = "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw=="; @@ -120,7 +120,8 @@ public class CloudStorageAzTest { @Test public void test() throws Exception { List<TestCase.CompilationUnit> cu = tcCtx.getTestCase().getCompilationUnit(); - Assume.assumeTrue(cu.size() > 1 || !EXCLUDED_TESTS.equals(getText(cu.get(0).getDescription()))); + Assume.assumeTrue(cu.size() > 1 || (!EXCLUDED_TESTS.equals(getText(cu.get(0).getDescription())) + && !S3_ONLY.equals(getText(cu.get(0).getDescription())))); LangExecutionUtil.test(tcCtx); } diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/cloud_storage/CloudStorageGCSTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/cloud_storage/CloudStorageGCSTest.java index 8c071f08a1..23321289a9 100644 --- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/cloud_storage/CloudStorageGCSTest.java +++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/cloud_storage/CloudStorageGCSTest.java @@ -70,6 +70,7 @@ public class CloudStorageGCSTest { private static final String CONFIG_FILE_NAME = "src/test/resources/cc-cloud-storage-gcs.conf"; private static final String DELTA_RESULT_PATH = "results_cloud"; private static final String EXCLUDED_TESTS = "MP"; + public static final String S3_ONLY = "S3Only"; public static final String MOCK_SERVER_HOSTNAME = "http://127.0.0.1:24443";; private static final String MOCK_SERVER_PROJECT_ID = "asterixdb-gcs-test-project-id"; @@ -79,7 +80,6 @@ public class CloudStorageGCSTest { @BeforeClass public static void setUp() throws Exception { - LocalCloudUtilAdobeMock.startS3CloudEnvironment(true, true); Storage storage = StorageOptions.newBuilder().setHost(MOCK_SERVER_HOSTNAME) .setCredentials(NoCredentials.getInstance()).setProjectId(MOCK_SERVER_PROJECT_ID).build().getService(); cleanup(storage); @@ -120,7 +120,8 @@ public class CloudStorageGCSTest { @Test public void test() throws Exception { List<TestCase.CompilationUnit> cu = tcCtx.getTestCase().getCompilationUnit(); - Assume.assumeTrue(cu.size() > 1 || !EXCLUDED_TESTS.equals(getText(cu.get(0).getDescription()))); + Assume.assumeTrue(cu.size() > 1 || (!EXCLUDED_TESTS.equals(getText(cu.get(0).getDescription())) + && !S3_ONLY.equals(getText(cu.get(0).getDescription())))); LangExecutionUtil.test(tcCtx); } diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/cloud_storage/CloudStorageSparseTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/cloud_storage/CloudStorageSparseTest.java index b930b660f5..4973c0bf90 100644 --- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/cloud_storage/CloudStorageSparseTest.java +++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/cloud_storage/CloudStorageSparseTest.java @@ -18,6 +18,9 @@ */ package org.apache.asterix.test.cloud_storage; +import static org.apache.asterix.api.common.LocalCloudUtilAdobeMock.createIfNotExists; +import static org.apache.asterix.api.common.LocalCloudUtilAdobeMock.fillConfigTemplate; + import java.net.URI; import java.util.Collection; import java.util.List; @@ -27,6 +30,7 @@ import org.apache.asterix.common.api.IDatasetLifecycleManager; import org.apache.asterix.common.api.INcApplicationContext; import org.apache.asterix.common.config.GlobalConfig; import org.apache.asterix.common.storage.StorageIOStats; +import org.apache.asterix.test.common.TestConstants; import org.apache.asterix.test.common.TestExecutor; import org.apache.asterix.test.runtime.ExecutionTestUtil; import org.apache.asterix.test.runtime.LangExecutionUtil; @@ -49,11 +53,12 @@ import org.junit.runners.MethodSorters; import org.junit.runners.Parameterized; import org.junit.runners.Parameterized.Parameters; +import com.adobe.testing.s3mock.testcontainers.S3MockContainer; + import software.amazon.awssdk.auth.credentials.AnonymousCredentialsProvider; import software.amazon.awssdk.regions.Region; import software.amazon.awssdk.services.s3.S3Client; import software.amazon.awssdk.services.s3.S3ClientBuilder; -import software.amazon.awssdk.services.s3.model.CreateBucketRequest; /** * Run tests in cloud deployment environment @@ -68,13 +73,13 @@ public class CloudStorageSparseTest { public static final String SUITE_TESTS = "testsuite_cloud_storage.xml"; public static final String ONLY_TESTS = "testsuite_cloud_storage_only.xml"; public static final String CONFIG_FILE_NAME = "src/test/resources/cc-cloud-storage-sparse.conf"; + public static final String CONFIG_FILE_TEMPLATE = "src/test/resources/cc-cloud-storage-sparse.conf.ftl"; public static final String DELTA_RESULT_PATH = "results_cloud"; public static final String EXCLUDED_TESTS = "MP"; public static final String PLAYGROUND_CONTAINER = "playground"; public static final String MOCK_SERVER_REGION = "us-west-2"; - public static final int MOCK_SERVER_PORT = 8001; - public static final String MOCK_SERVER_HOSTNAME = "http://127.0.0.1:"; + MOCK_SERVER_PORT; + public static final String MOCK_SERVER_HOSTNAME_FRAGMENT = "http://127.0.0.1:";; public CloudStorageSparseTest(TestCaseContext tcCtx) { this.tcCtx = tcCtx; @@ -87,7 +92,11 @@ public class CloudStorageSparseTest { } public static void setupEnv(TestExecutor testExecutor) throws Exception { - LocalCloudUtilAdobeMock.startS3CloudEnvironment(true); + S3MockContainer s3Mock = LocalCloudUtilAdobeMock.startS3CloudEnvironment(true); + fillConfigTemplate(MOCK_SERVER_HOSTNAME_FRAGMENT + s3Mock.getHttpServerPort(), CONFIG_FILE_TEMPLATE, + CONFIG_FILE_NAME); + System.setProperty(TestConstants.S3_SERVICE_ENDPOINT_KEY, + MOCK_SERVER_HOSTNAME_FRAGMENT + s3Mock.getHttpServerPort()); testExecutor.executorId = "cloud"; testExecutor.stripSubstring = "//DB:"; LangExecutionUtil.setUp(CONFIG_FILE_NAME, testExecutor); @@ -95,11 +104,11 @@ public class CloudStorageSparseTest { // create the playground bucket and leave it empty, just for external collection-based tests S3ClientBuilder builder = S3Client.builder(); - URI endpoint = URI.create(MOCK_SERVER_HOSTNAME); // endpoint pointing to S3 mock server + URI endpoint = URI.create(MOCK_SERVER_HOSTNAME_FRAGMENT + s3Mock.getHttpServerPort()); // endpoint pointing to S3 mock server builder.region(Region.of(MOCK_SERVER_REGION)).credentialsProvider(AnonymousCredentialsProvider.create()) .endpointOverride(endpoint); S3Client client = builder.build(); - client.createBucket(CreateBucketRequest.builder().bucket(PLAYGROUND_CONTAINER).build()); + createIfNotExists(PLAYGROUND_CONTAINER, client); client.close(); } diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/cloud_storage/CloudStorageTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/cloud_storage/CloudStorageTest.java index 340583810e..dc65daa6b4 100644 --- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/cloud_storage/CloudStorageTest.java +++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/cloud_storage/CloudStorageTest.java @@ -18,12 +18,16 @@ */ package org.apache.asterix.test.cloud_storage; +import static org.apache.asterix.api.common.LocalCloudUtilAdobeMock.createIfNotExists; +import static org.apache.asterix.api.common.LocalCloudUtilAdobeMock.fillConfigTemplate; + import java.net.URI; import java.util.Collection; import java.util.List; import org.apache.asterix.api.common.LocalCloudUtilAdobeMock; import org.apache.asterix.common.config.GlobalConfig; +import org.apache.asterix.test.common.TestConstants; import org.apache.asterix.test.common.TestExecutor; import org.apache.asterix.test.runtime.LangExecutionUtil; import org.apache.asterix.testframework.context.TestCaseContext; @@ -41,11 +45,12 @@ import org.junit.runners.MethodSorters; import org.junit.runners.Parameterized; import org.junit.runners.Parameterized.Parameters; +import com.adobe.testing.s3mock.testcontainers.S3MockContainer; + import software.amazon.awssdk.auth.credentials.AnonymousCredentialsProvider; import software.amazon.awssdk.regions.Region; import software.amazon.awssdk.services.s3.S3Client; import software.amazon.awssdk.services.s3.S3ClientBuilder; -import software.amazon.awssdk.services.s3.model.CreateBucketRequest; /** * Run tests in cloud deployment environment @@ -59,14 +64,14 @@ public class CloudStorageTest { private final TestCaseContext tcCtx; public static final String SUITE_TESTS = "testsuite_cloud_storage.xml"; public static final String ONLY_TESTS = "testsuite_cloud_storage_only.xml"; - public static final String CONFIG_FILE_NAME = "src/test/resources/cc-cloud-storage.conf"; + public static final String CONFIG_FILE_TEMPLATE = "src/test/resources/cc-cloud-storage.conf.ftl"; + public static final String CONFIG_FILE = "target/cc-cloud-storage.conf"; public static final String DELTA_RESULT_PATH = "results_cloud"; public static final String EXCLUDED_TESTS = "MP"; public static final String PLAYGROUND_CONTAINER = "playground"; public static final String MOCK_SERVER_REGION = "us-west-2"; - public static final int MOCK_SERVER_PORT = 8001; - public static final String MOCK_SERVER_HOSTNAME = "http://127.0.0.1:"; + MOCK_SERVER_PORT; + public static final String MOCK_SERVER_HOSTNAME_FRAGMENT = "http://127.0.0.1:";; public CloudStorageTest(TestCaseContext tcCtx) { this.tcCtx = tcCtx; @@ -79,19 +84,23 @@ public class CloudStorageTest { } public static void setupEnv(TestExecutor testExecutor) throws Exception { - LocalCloudUtilAdobeMock.startS3CloudEnvironment(true); + S3MockContainer s3Mock = LocalCloudUtilAdobeMock.startS3CloudEnvironment(true); + fillConfigTemplate(MOCK_SERVER_HOSTNAME_FRAGMENT + s3Mock.getHttpServerPort(), CONFIG_FILE_TEMPLATE, + CONFIG_FILE); + System.setProperty(TestConstants.S3_SERVICE_ENDPOINT_KEY, + MOCK_SERVER_HOSTNAME_FRAGMENT + s3Mock.getHttpServerPort()); testExecutor.executorId = "cloud"; testExecutor.stripSubstring = "//DB:"; - LangExecutionUtil.setUp(CONFIG_FILE_NAME, testExecutor); - System.setProperty(GlobalConfig.CONFIG_FILE_PROPERTY, CONFIG_FILE_NAME); + LangExecutionUtil.setUp(CONFIG_FILE, testExecutor); + System.setProperty(GlobalConfig.CONFIG_FILE_PROPERTY, CONFIG_FILE); // create the playground bucket and leave it empty, just for external collection-based tests S3ClientBuilder builder = S3Client.builder(); - URI endpoint = URI.create(MOCK_SERVER_HOSTNAME); // endpoint pointing to S3 mock server + URI endpoint = URI.create(MOCK_SERVER_HOSTNAME_FRAGMENT + s3Mock.getHttpServerPort()); // endpoint pointing to S3 mock server builder.region(Region.of(MOCK_SERVER_REGION)).credentialsProvider(AnonymousCredentialsProvider.create()) .endpointOverride(endpoint); S3Client client = builder.build(); - client.createBucket(CreateBucketRequest.builder().bucket(PLAYGROUND_CONTAINER).build()); + createIfNotExists(PLAYGROUND_CONTAINER, client); client.close(); } diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/cloud_storage/CloudStorageUnstableTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/cloud_storage/CloudStorageUnstableTest.java index 29087f37a1..ccb6984491 100644 --- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/cloud_storage/CloudStorageUnstableTest.java +++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/cloud_storage/CloudStorageUnstableTest.java @@ -18,6 +18,10 @@ */ package org.apache.asterix.test.cloud_storage; +import static org.apache.asterix.api.common.LocalCloudUtilAdobeMock.fillConfigTemplate; +import static org.apache.asterix.test.cloud_storage.CloudStorageTest.CONFIG_FILE_TEMPLATE; +import static org.apache.asterix.test.cloud_storage.CloudStorageTest.MOCK_SERVER_HOSTNAME_FRAGMENT; + import java.util.ArrayList; import java.util.Collection; import java.util.List; @@ -43,6 +47,8 @@ import org.junit.runners.MethodSorters; import org.junit.runners.Parameterized; import org.junit.runners.Parameterized.Parameters; +import com.adobe.testing.s3mock.testcontainers.S3MockContainer; + /** * Run tests in cloud deployment environment with simulated unstable connection */ @@ -66,7 +72,9 @@ public class CloudStorageUnstableTest { @BeforeClass public static void setUp() throws Exception { System.setProperty(CloudRetryableRequestUtil.CLOUD_UNSTABLE_MODE, "true"); - LocalCloudUtilAdobeMock.startS3CloudEnvironment(true, true); + S3MockContainer s3Mock = LocalCloudUtilAdobeMock.startS3CloudEnvironment(true, true); + fillConfigTemplate(MOCK_SERVER_HOSTNAME_FRAGMENT + s3Mock.getHttpServerPort(), CONFIG_FILE_TEMPLATE, + CONFIG_FILE_NAME); TestExecutor testExecutor = new TestExecutor(DELTA_RESULT_PATH); testExecutor.executorId = "cloud"; testExecutor.stripSubstring = "//DB:"; diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/cloud_storage/GCSCloudStorageUnstableTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/cloud_storage/GCSCloudStorageUnstableTest.java index c97e459fe3..72f5cff885 100644 --- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/cloud_storage/GCSCloudStorageUnstableTest.java +++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/cloud_storage/GCSCloudStorageUnstableTest.java @@ -21,11 +21,11 @@ package org.apache.asterix.test.cloud_storage; import static org.apache.asterix.api.common.LocalCloudUtil.CLOUD_STORAGE_BUCKET; import static org.apache.asterix.api.common.LocalCloudUtil.MOCK_SERVER_REGION; +import static org.apache.asterix.test.cloud_storage.CloudStorageGCSTest.S3_ONLY; import java.util.Collection; import java.util.List; -import org.apache.asterix.api.common.LocalCloudUtilAdobeMock; import org.apache.asterix.common.api.IDatasetLifecycleManager; import org.apache.asterix.common.api.INcApplicationContext; import org.apache.asterix.common.config.GlobalConfig; @@ -83,7 +83,6 @@ public class GCSCloudStorageUnstableTest { @BeforeClass public static void setUp() throws Exception { - LocalCloudUtilAdobeMock.startS3CloudEnvironment(true, true); System.setProperty(CloudRetryableRequestUtil.CLOUD_UNSTABLE_MODE, "true"); Storage storage = StorageOptions.newBuilder().setHost(MOCK_SERVER_HOSTNAME) .setCredentials(NoCredentials.getInstance()).setProjectId(MOCK_SERVER_PROJECT_ID).build().getService(); @@ -111,7 +110,8 @@ public class GCSCloudStorageUnstableTest { @Test public void test() throws Exception { List<TestCase.CompilationUnit> cu = tcCtx.getTestCase().getCompilationUnit(); - Assume.assumeTrue(cu.size() > 1 || !EXCLUDED_TESTS.equals(getText(cu.get(0).getDescription()))); + Assume.assumeTrue(cu.size() > 1 || (!EXCLUDED_TESTS.equals(getText(cu.get(0).getDescription())) + && !S3_ONLY.equals(getText(cu.get(0).getDescription())))); LangExecutionUtil.test(tcCtx); for (NodeControllerService nc : ExecutionTestUtil.integrationUtil.ncs) { IDatasetLifecycleManager lifecycleManager = diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestConstants.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestConstants.java index 8c59c56950..e8d9f34443 100644 --- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestConstants.java +++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestConstants.java @@ -28,6 +28,7 @@ public class TestConstants { public static final String S3_REGION_DEFAULT = "us-west-2"; public static final String S3_SERVICE_ENDPOINT_PLACEHOLDER = "%serviceEndpoint%"; public static final String S3_SERVICE_ENDPOINT_DEFAULT = "http://127.0.0.1:8001";; + public static final String S3_SERVICE_ENDPOINT_KEY = "s3mockEndpoint"; public static final String S3_TEMPLATE = "(\"accessKeyId\"=\"" + S3_ACCESS_KEY_ID_DEFAULT + "\"),\n" + "(\"secretAccessKey\"=\"" + S3_SECRET_ACCESS_KEY_DEFAULT + "\"),\n" + "(\"region\"=\"" + S3_REGION_PLACEHOLDER + "\"),\n" + "(\"serviceEndpoint\"=\"" + S3_SERVICE_ENDPOINT_PLACEHOLDER + "\")"; diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java index 58c92fb3ee..32d6e170ec 100644 --- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java +++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java @@ -19,6 +19,9 @@ package org.apache.asterix.test.common; import static java.nio.charset.StandardCharsets.UTF_8; +import static org.apache.asterix.test.common.TestConstants.S3_ACCESS_KEY_ID_DEFAULT; +import static org.apache.asterix.test.common.TestConstants.S3_REGION_DEFAULT; +import static org.apache.asterix.test.common.TestConstants.S3_SECRET_ACCESS_KEY_DEFAULT; import static org.apache.asterix.test.common.TestConstants.Azure.ACCOUNT_KEY_PLACEHOLDER; import static org.apache.asterix.test.common.TestConstants.Azure.ACCOUNT_NAME_PLACEHOLDER; import static org.apache.asterix.test.common.TestConstants.Azure.AZURITE_ACCOUNT_KEY_DEFAULT; @@ -288,6 +291,7 @@ public class TestExecutor { protected String deltaPath = null; public String stripSubstring = null; public String executorId = null; + private String s3mockTemplate = null; public TestExecutor() { this(Collections.singletonList( @@ -2532,7 +2536,14 @@ public class TestExecutor { protected String setS3TemplateDefault(String str) { if (str.contains("%template%")) { - return str.replace("%template%", TestConstants.S3_TEMPLATE_DEFAULT); + String s3mockEndpoint = System.getProperty(TestConstants.S3_SERVICE_ENDPOINT_KEY); + if (s3mockEndpoint != null && s3mockTemplate == null) { + s3mockTemplate = "(\"accessKeyId\"=\"" + S3_ACCESS_KEY_ID_DEFAULT + "\"),\n" + "(\"secretAccessKey\"=\"" + + S3_SECRET_ACCESS_KEY_DEFAULT + "\"),\n" + "(\"region\"=\"" + S3_REGION_DEFAULT + "\"),\n" + + "(\"serviceEndpoint\"=\"" + s3mockEndpoint + "\")"; + } + return str.replace("%template%", + s3mockTemplate == null ? TestConstants.S3_TEMPLATE_DEFAULT : s3mockTemplate); } else { return str.replace("%template_colons%", TestConstants.S3_TEMPLATE_DEFAULT_NO_PARENTHESES_WITH_COLONS); } diff --git a/asterixdb/asterix-app/src/test/resources/cc-cloud-storage-sparse.conf b/asterixdb/asterix-app/src/test/resources/cc-cloud-storage-main.ftl similarity index 95% copy from asterixdb/asterix-app/src/test/resources/cc-cloud-storage-sparse.conf copy to asterixdb/asterix-app/src/test/resources/cc-cloud-storage-main.ftl index 0842e0179b..8271bfd9a3 100644 --- a/asterixdb/asterix-app/src/test/resources/cc-cloud-storage-sparse.conf +++ b/asterixdb/asterix-app/src/test/resources/cc-cloud-storage-main.ftl @@ -42,7 +42,6 @@ jvm.args=-Xmx4096m --add-opens=jdk.management/com.sun.management.internal=ALL-UN storage.buffercache.size=128MB storage.memorycomponent.globalbudget=512MB storage.max.columns.in.zeroth.segment=800 -storage.page.zero.writer=sparse [cc] address = 127.0.0.1 @@ -60,7 +59,6 @@ compiler.groupmemory=160KB compiler.joinmemory=256KB compiler.textsearchmemory=160KB compiler.windowmemory=192KB -compiler.ordered.fields=false compiler.internal.sanitycheck=true messaging.frame.size=4096 messaging.frame.count=512 @@ -70,8 +68,8 @@ storage.partitioning=static cloud.storage.scheme=s3 cloud.storage.bucket=cloud-storage-container cloud.storage.region=us-west-2 -cloud.storage.endpoint=http://127.0.0.1:8001 +cloud.storage.endpoint=${cloudUrl} cloud.storage.anonymous.auth=true cloud.storage.cache.policy=selective cloud.max.write.requests.per.second=2000 -cloud.max.read.requests.per.second=4000 +cloud.max.read.requests.per.second=4000 \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/cc-cloud-storage-sparse.conf b/asterixdb/asterix-app/src/test/resources/cc-cloud-storage-sparse.conf.ftl similarity index 98% copy from asterixdb/asterix-app/src/test/resources/cc-cloud-storage-sparse.conf copy to asterixdb/asterix-app/src/test/resources/cc-cloud-storage-sparse.conf.ftl index 0842e0179b..fa5c146fa1 100644 --- a/asterixdb/asterix-app/src/test/resources/cc-cloud-storage-sparse.conf +++ b/asterixdb/asterix-app/src/test/resources/cc-cloud-storage-sparse.conf.ftl @@ -70,7 +70,7 @@ storage.partitioning=static cloud.storage.scheme=s3 cloud.storage.bucket=cloud-storage-container cloud.storage.region=us-west-2 -cloud.storage.endpoint=http://127.0.0.1:8001 +cloud.storage.endpoint=${cloudUrl} cloud.storage.anonymous.auth=true cloud.storage.cache.policy=selective cloud.max.write.requests.per.second=2000 diff --git a/asterixdb/asterix-app/src/test/resources/cc-cloud-storage-sparse.conf b/asterixdb/asterix-app/src/test/resources/cc-cloud-storage.conf.ftl similarity index 97% rename from asterixdb/asterix-app/src/test/resources/cc-cloud-storage-sparse.conf rename to asterixdb/asterix-app/src/test/resources/cc-cloud-storage.conf.ftl index 0842e0179b..74c557a8a9 100644 --- a/asterixdb/asterix-app/src/test/resources/cc-cloud-storage-sparse.conf +++ b/asterixdb/asterix-app/src/test/resources/cc-cloud-storage.conf.ftl @@ -42,7 +42,7 @@ jvm.args=-Xmx4096m --add-opens=jdk.management/com.sun.management.internal=ALL-UN storage.buffercache.size=128MB storage.memorycomponent.globalbudget=512MB storage.max.columns.in.zeroth.segment=800 -storage.page.zero.writer=sparse +storage.page.zero.writer=default [cc] address = 127.0.0.1 @@ -70,7 +70,7 @@ storage.partitioning=static cloud.storage.scheme=s3 cloud.storage.bucket=cloud-storage-container cloud.storage.region=us-west-2 -cloud.storage.endpoint=http://127.0.0.1:8001 +cloud.storage.endpoint=${cloudUrl} cloud.storage.anonymous.auth=true cloud.storage.cache.policy=selective cloud.max.write.requests.per.second=2000 diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array_fun/array_slice/array_slice_bracket_notation/array_slice_bracket_notation.1.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/api/post-non-query/post-non-query.099.post.http old mode 100755 new mode 100644 similarity index 79% copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array_fun/array_slice/array_slice_bracket_notation/array_slice_bracket_notation.1.ddl.sqlpp copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/api/post-non-query/post-non-query.099.post.http index 8103562527..6dd6f5e6c5 --- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array_fun/array_slice/array_slice_bracket_notation/array_slice_bracket_notation.1.ddl.sqlpp +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/api/post-non-query/post-non-query.099.post.http @@ -16,6 +16,10 @@ * specific language governing permissions and limitations * under the License. */ - -CREATE TYPE openType AS {id: int}; -CREATE DATASET ds(openType) primary key id; \ No newline at end of file +/* + * Description: testing passing a non-query statement using POST with readonly set to false + * Result: success + */ +-- extractstatus +/query/service +--body={"statement": "DROP TYPE t1;", "readonly": false} \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array_fun/array_slice/array_slice_bracket_notation/array_slice_bracket_notation.2.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array_fun/array_slice/array_slice_bracket_notation/array_slice_bracket_notation.099.ddl.sqlpp old mode 100755 new mode 100644 similarity index 93% copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array_fun/array_slice/array_slice_bracket_notation/array_slice_bracket_notation.2.update.sqlpp copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array_fun/array_slice/array_slice_bracket_notation/array_slice_bracket_notation.099.ddl.sqlpp index a9171ce01a..056fbcdcc1 --- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array_fun/array_slice/array_slice_bracket_notation/array_slice_bracket_notation.2.update.sqlpp +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array_fun/array_slice/array_slice_bracket_notation/array_slice_bracket_notation.099.ddl.sqlpp @@ -17,4 +17,5 @@ * under the License. */ -UPSERT INTO ds {"id": 1, "arr": [0, 1, 2, 3, 4, 5]}; \ No newline at end of file +DROP TYPE openType IF EXISTS; +DROP DATASET ds IF EXISTS; \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array_fun/array_slice/array_slice_bracket_notation/array_slice_bracket_notation.1.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array_fun/array_slice/array_slice_bracket_notation/array_slice_bracket_notation.1.ddl.sqlpp old mode 100755 new mode 100644 diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array_fun/array_slice/array_slice_bracket_notation/array_slice_bracket_notation.2.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array_fun/array_slice/array_slice_bracket_notation/array_slice_bracket_notation.2.update.sqlpp old mode 100755 new mode 100644 diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array_fun/array_slice/array_slice_bracket_notation/array_slice_bracket_notation.3.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array_fun/array_slice/array_slice_bracket_notation/array_slice_bracket_notation.3.query.sqlpp old mode 100755 new mode 100644 diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array_fun/array_slice/array_slice_bracket_notation/array_slice_bracket_notation.4.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array_fun/array_slice/array_slice_bracket_notation/array_slice_bracket_notation.4.ddl.sqlpp old mode 100755 new mode 100644 diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/cloud_storage/storage-size/datasource-function/external-collection/test.010.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/cloud_storage/storage-size/datasource-function/external-collection/test.010.ddl.sqlpp index 5ae3310c8d..ada7b9fec6 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/cloud_storage/storage-size/datasource-function/external-collection/test.010.ddl.sqlpp +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/cloud_storage/storage-size/datasource-function/external-collection/test.010.ddl.sqlpp @@ -26,12 +26,8 @@ CREATE COLLECTION testCollection PRIMARY KEY (id: int); CREATE TYPE test AS open {}; -CREATE EXTERNAL DATASET testExternalCollection(test) USING S3 ( -("accessKeyId"="dummyAccessKey"), -("secretAccessKey"="dummySecretKey"), -("sessionToken"="dummySessionToken"), -("region"="us-west-2"), -("serviceEndpoint"="http://127.0.0.1:8001";), +CREATE EXTERNAL DATASET testExternalCollection(test) USING %adapter% ( +%template%, ("container"="playground"), ("definition"="json-data/reviews/single-line/json"), ("format"="json") diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/limit_after_offset/limit_after_offset.1.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/join/hash_join_exchange/hash_join_exchange.099.ddl.sqlpp similarity index 86% copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/limit_after_offset/limit_after_offset.1.query.sqlpp copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/join/hash_join_exchange/hash_join_exchange.099.ddl.sqlpp index 349b762cba..86b1e4366d 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/limit_after_offset/limit_after_offset.1.query.sqlpp +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/join/hash_join_exchange/hash_join_exchange.099.ddl.sqlpp @@ -17,7 +17,8 @@ * under the License. */ -//success +DROP DATASET customer IF EXISTS; +DROP DATASET orders IF EXISTS; +DROP TYPE cust_type IF EXISTS; +DROP TYPE orders_type IF EXISTS; -create dataset ds primary key (id:string); -SELECT id FROM ds OFFSET 1 LIMIT 5; \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/limit_after_offset/limit_after_offset.1.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/join/hash_join_exchange_1/hash_join_exchange_1.099.ddl.sqlpp similarity index 88% copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/limit_after_offset/limit_after_offset.1.query.sqlpp copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/join/hash_join_exchange_1/hash_join_exchange_1.099.ddl.sqlpp index 349b762cba..31870c9032 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/limit_after_offset/limit_after_offset.1.query.sqlpp +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/join/hash_join_exchange_1/hash_join_exchange_1.099.ddl.sqlpp @@ -17,7 +17,8 @@ * under the License. */ -//success +DROP DATASET A IF EXISTS; +DROP DATASET B IF EXISTS; +DROP TYPE A_TYPE IF EXISTS; +DROP TYPE B_TYPE IF EXISTS; -create dataset ds primary key (id:string); -SELECT id FROM ds OFFSET 1 LIMIT 5; \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array_fun/array_slice/array_slice_bracket_notation/array_slice_bracket_notation.1.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/join/join-with-empty-dataset/join-with-empty-dataset.099.ddl.sqlpp old mode 100755 new mode 100644 similarity index 90% copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array_fun/array_slice/array_slice_bracket_notation/array_slice_bracket_notation.1.ddl.sqlpp copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/join/join-with-empty-dataset/join-with-empty-dataset.099.ddl.sqlpp index 8103562527..af498e5478 --- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array_fun/array_slice/array_slice_bracket_notation/array_slice_bracket_notation.1.ddl.sqlpp +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/join/join-with-empty-dataset/join-with-empty-dataset.099.ddl.sqlpp @@ -17,5 +17,6 @@ * under the License. */ -CREATE TYPE openType AS {id: int}; -CREATE DATASET ds(openType) primary key id; \ No newline at end of file +drop dataset ds1 if exists; +drop dataset ds2 if exists; +drop type openType if exists; \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/limit_after_offset/limit_after_offset.1.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/limit_after_offset/limit_after_offset.099.ddl.sqlpp similarity index 91% rename from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/limit_after_offset/limit_after_offset.1.query.sqlpp rename to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/limit_after_offset/limit_after_offset.099.ddl.sqlpp index 349b762cba..69c1a2f489 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/limit_after_offset/limit_after_offset.1.query.sqlpp +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/limit_after_offset/limit_after_offset.099.ddl.sqlpp @@ -19,5 +19,4 @@ //success -create dataset ds primary key (id:string); -SELECT id FROM ds OFFSET 1 LIMIT 5; \ No newline at end of file +drop dataset ds; \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array_fun/array_slice/array_slice_bracket_notation/array_slice_bracket_notation.2.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/limit_after_offset/limit_after_offset.1.ddl.sqlpp old mode 100755 new mode 100644 similarity index 93% copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array_fun/array_slice/array_slice_bracket_notation/array_slice_bracket_notation.2.update.sqlpp copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/limit_after_offset/limit_after_offset.1.ddl.sqlpp index a9171ce01a..da0963d61a --- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array_fun/array_slice/array_slice_bracket_notation/array_slice_bracket_notation.2.update.sqlpp +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/limit_after_offset/limit_after_offset.1.ddl.sqlpp @@ -17,4 +17,6 @@ * under the License. */ -UPSERT INTO ds {"id": 1, "arr": [0, 1, 2, 3, 4, 5]}; \ No newline at end of file +//success + +create dataset ds primary key (id:string); \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array_fun/array_slice/array_slice_bracket_notation/array_slice_bracket_notation.4.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/limit_after_offset/limit_after_offset.2.query.sqlpp old mode 100755 new mode 100644 similarity index 94% copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array_fun/array_slice/array_slice_bracket_notation/array_slice_bracket_notation.4.ddl.sqlpp copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/limit_after_offset/limit_after_offset.2.query.sqlpp index 43604c69f6..6b2c90cbce --- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array_fun/array_slice/array_slice_bracket_notation/array_slice_bracket_notation.4.ddl.sqlpp +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/limit_after_offset/limit_after_offset.2.query.sqlpp @@ -17,5 +17,6 @@ * under the License. */ -DROP DATASET ds; -DROP TYPE openType; \ No newline at end of file +//success + +SELECT id FROM ds OFFSET 1 LIMIT 5; \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/post-non-query/post-non-query.099.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/post-non-query/post-non-query.099.adm new file mode 100644 index 0000000000..4063cf103a --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/post-non-query/post-non-query.099.adm @@ -0,0 +1 @@ +"success" \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_cloud_storage.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_cloud_storage.xml index 4a612f77cf..683c348f4a 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_cloud_storage.xml +++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_cloud_storage.xml @@ -76,6 +76,8 @@ </test-case> <test-case FilePath="cloud_storage/storage-size/datasource-function"> <compilation-unit name="external-collection"> + <description>S3Only</description> + <placeholder name="adapter" value="S3" /> <output-dir compare="Clean-JSON">external-collection</output-dir> <expected-error>Retrieving storage size is not applicable to type: EXTERNAL.</expected-error> </compilation-unit> diff --git a/asterixdb/pom.xml b/asterixdb/pom.xml index bb4e476e92..74f4271b70 100644 --- a/asterixdb/pom.xml +++ b/asterixdb/pom.xml @@ -1577,15 +1577,9 @@ <!-- Mock for Adobe AWS S3 --> <dependency> <groupId>com.adobe.testing</groupId> - <artifactId>s3mock</artifactId> - <version>2.17.0</version> + <artifactId>s3mock-testcontainers</artifactId> + <version>4.7.0</version> <scope>test</scope> - <exclusions> - <exclusion> - <groupId>org.springframework.boot</groupId> - <artifactId>spring-boot-starter-logging</artifactId> - </exclusion> - </exclusions> </dependency> <!-- Mock for AWS S3 --> <dependency>
