This is an automated email from the ASF dual-hosted git repository.
ostinru pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/cloudberry-pxf.git
The following commit(s) were added to refs/heads/main by this push:
new 174807ba Update S3-related tests to run in Testcontainers (#117)
174807ba is described below
commit 174807bae855b20c9397ce7188b82772006e26ae
Author: Nikolay Antonov <[email protected]>
AuthorDate: Mon Jun 15 17:50:59 2026 +0300
Update S3-related tests to run in Testcontainers (#117)
Fix S3-related tests from #86:
- Add `MinIOContainer` and `S3Application` to manage MinIO
- Generate fixtures via `SmallDataFactory` (prefer composition over
inheritance)
- Migrate `CloudAccessTest` and `S3SelectTest` to TestContainers suite.
- Pre-bake test `s3`/`s3-invalid` PXF server configs into image.
- Move S3 tests from the singlecluster to the testcontainers pipeline
(`pxf-s3`) and delete the unused legacy MinIO scripting.
- Replace s3a client with aws S3 client in tests data management.
---
.github/workflows/pxf-ci.yml | 3 +-
automation/Makefile | 21 +-
automation/pom.xml | 15 +-
.../pxf/automation/applications/S3Application.java | 98 +++++++++
.../automation/testcontainers/MinIOContainer.java | 78 +++++++
.../testcontainers/PXFCloudberryContainer.java | 19 +-
.../pxf/automation/utils/AutomationUtils.java | 60 ++++++
.../testcontainers/pxf-cbdb/script/entrypoint.sh | 51 +----
.../testcontainers/pxf-cbdb/script/pxf-env.sh | 9 -
.../pxf-cbdb/servers/s3-invalid/s3-site.xml | 54 +++++
.../testcontainers/pxf-cbdb/servers/s3/s3-site.xml | 50 +++++
.../pxf/automation/AbstractTestcontainersTest.java | 80 ++++++-
.../pxf/automation/SmallDataFactory.java | 97 +++++++++
.../automation/features/cloud/CloudAccessTest.java | 167 ++++++++-------
.../automation/features/cloud/S3SelectTest.java | 237 +++++++++++----------
ci/docker/pxf-cbdb-dev/common/script/entrypoint.sh | 52 -----
ci/docker/pxf-cbdb-dev/common/script/pxf-env.sh | 9 -
ci/docker/pxf-cbdb-dev/common/script/run_tests.sh | 231 +-------------------
.../pxf-cbdb-dev/common/script/start_minio.bash | 59 -----
.../ubuntu/script/entrypoint_kerberos.sh | 82 +------
ci/singlecluster/Dockerfile | 14 --
21 files changed, 761 insertions(+), 725 deletions(-)
diff --git a/.github/workflows/pxf-ci.yml b/.github/workflows/pxf-ci.yml
index 48cfbeba..6109bd77 100644
--- a/.github/workflows/pxf-ci.yml
+++ b/.github/workflows/pxf-ci.yml
@@ -330,7 +330,6 @@ jobs:
- jdbc
- proxy
- unused
- - s3
- features
- gpdb
- gpdb_fdw
@@ -501,7 +500,6 @@ jobs:
- jdbc
- proxy
- unused
- - s3
- features
- gpdb
- gpdb_fdw
@@ -659,6 +657,7 @@ jobs:
matrix:
tc_group:
- 'pxf-jdbc'
+ - 'pxf-s3'
use_fdw:
- 'false'
- 'true'
diff --git a/automation/Makefile b/automation/Makefile
index 8a1d168c..05313556 100755
--- a/automation/Makefile
+++ b/automation/Makefile
@@ -35,11 +35,6 @@ endif
# lowercase the protocol
PROTOCOL := $(shell echo $(PROTOCOL) | tr A-Z a-z)
-ifeq "$(PROTOCOL)" "minio"
- MINIO=true
- PROTOCOL=s3
-endif
-
PXF_BASE ?= $(PXF_HOME)
TEMPLATES_DIR=$(PXF_HOME)/templates
@@ -193,21 +188,6 @@ ifneq "$(PROTOCOL)" ""
cp $(TEMPLATES_DIR)/pxf-site.xml $(PROTOCOL_HOME)/; \
sed $(SED_OPTS)
's|</configuration>|<property><name>pxf.fs.basePath</name><value>$(BASE_PATH)</value></property></configuration>|g'
$(PROTOCOL_HOME)/pxf-site.xml; \
fi; \
- if [ $(PROTOCOL) = s3 ]; then \
- if [ "$(MINIO)" = "true" ]; then \
- cp $(TEMPLATES_DIR)/minio-site.xml
$(PROTOCOL_HOME)/$(PROTOCOL)-site.xml; \
- sed $(SED_OPTS)
"s|YOUR_MINIO_URL|http://localhost:9000|"
$(PROTOCOL_HOME)/$(PROTOCOL)-site.xml; \
- fi; \
- mkdir -p $(PROTOCOL_HOME)-invalid; \
- cp $(TEMPLATES_DIR)/$(PROTOCOL)-site.xml
$(PROTOCOL_HOME)-invalid/; \
- if [ -z "$(ACCESS_KEY_ID)" ] || [ -z
"$(SECRET_ACCESS_KEY)" ]; then \
- echo "AWS Keys (ACCESS_KEY_ID,
SECRET_ACCESS_KEY) not set"; \
- rm -rf $(PROTOCOL_HOME); \
- exit 1; \
- fi; \
- sed $(SED_OPTS)
"s|YOUR_AWS_ACCESS_KEY_ID|$(ACCESS_KEY_ID)|"
$(PROTOCOL_HOME)/$(PROTOCOL)-site.xml; \
- sed $(SED_OPTS)
"s|YOUR_AWS_SECRET_ACCESS_KEY|$(SECRET_ACCESS_KEY)|"
$(PROTOCOL_HOME)/$(PROTOCOL)-site.xml; \
- fi; \
if [ $(PROTOCOL) = abfss ]; then \
if [ -z "$(ABFSS_ACCOUNT)" ] || [ -z
"$(ABFSS_CLIENT_ENDPOINT)" ] || [ -z "$(ABFSS_CLIENT_ID)" ] || [ -z
"$(ABFSS_CLIENT_SECRET)" ]; then \
echo "ADL Keys (ABFSS_ACCOUNT, ABFSS_CLIENT_ID,
ABFSS_CLIENT_SECRET, ABFSS_CLIENT_ENDPOINT) not set"; \
@@ -255,6 +235,7 @@ endif
# Usage:
# make test-tc => run all testcontainers tests
(Ubuntu)
# make test-tc TC_GROUP=pxf-jdbc => run only pxf-jdbc group
+# make test-tc TC_GROUP=pxf-s3 => run S3 Select + CloudAccess tests
(MinIO sidecar)
# make test-tc DISTRO=rocky9 => run with Rocky Linux 9 base image
.PHONY: test-tc
test-tc: check-env symlink_pxf_jars pxf_regress
diff --git a/automation/pom.xml b/automation/pom.xml
index 825633df..1be23e50 100644
--- a/automation/pom.xml
+++ b/automation/pom.xml
@@ -23,6 +23,7 @@
<orc.version>1.6.13</orc.version>
<snappy.java.version>1.1.10.4</snappy.java.version>
<powermock.version>1.6.4</powermock.version>
+ <argLine.extra></argLine.extra>
</properties>
<repositories>
@@ -59,7 +60,7 @@
<version>2.15</version>
<configuration>
<testFailureIgnore>true</testFailureIgnore>
- <argLine>-Xmx4096m</argLine>
+ <argLine>-Xmx4096m ${argLine.extra}</argLine>
<forkCount>1</forkCount>
<reuseForks>false</reuseForks>
</configuration>
@@ -535,4 +536,16 @@
<version>1.9.5</version>
</dependency>
</dependencies>
+
+ <profiles>
+ <profile>
+ <id>java9-plus</id>
+ <activation>
+ <jdk>[9,)</jdk>
+ </activation>
+ <properties>
+ <argLine.extra>--add-opens java.base/java.lang=ALL-UNNAMED
--add-opens java.base/java.util=ALL-UNNAMED</argLine.extra>
+ </properties>
+ </profile>
+ </profiles>
</project>
\ No newline at end of file
diff --git
a/automation/src/main/java/org/apache/cloudberry/pxf/automation/applications/S3Application.java
b/automation/src/main/java/org/apache/cloudberry/pxf/automation/applications/S3Application.java
new file mode 100644
index 00000000..a92f3f39
--- /dev/null
+++
b/automation/src/main/java/org/apache/cloudberry/pxf/automation/applications/S3Application.java
@@ -0,0 +1,98 @@
+package org.apache.cloudberry.pxf.automation.applications;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import com.amazonaws.auth.AWSStaticCredentialsProvider;
+import com.amazonaws.auth.BasicAWSCredentials;
+import com.amazonaws.client.builder.AwsClientBuilder;
+import com.amazonaws.services.s3.AmazonS3;
+import com.amazonaws.services.s3.AmazonS3ClientBuilder;
+import com.amazonaws.services.s3.model.DeleteObjectsRequest;
+import com.amazonaws.services.s3.model.ListObjectsV2Request;
+import com.amazonaws.services.s3.model.ListObjectsV2Result;
+import com.amazonaws.services.s3.model.PutObjectRequest;
+import com.amazonaws.services.s3.model.S3ObjectSummary;
+import org.apache.cloudberry.pxf.automation.testcontainers.MinIOContainer;
+
+import java.io.IOException;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * S3 API access wrapper used by automation tests to seed and clean fixtures
+ * in a MinIO bucket. Owns the AmazonS3 client; callers should call
+ * `shutdown()` when done (typically in afterClass before stopping the
+ * MinIO container).
+ */
+public class S3Application implements AutoCloseable {
+
+ private final AmazonS3 s3Client;
+
+ public S3Application(MinIOContainer minio) {
+ this.s3Client = buildS3Client(minio.getHostEndpoint(),
minio.getAccessKey(), minio.getSecretKey());
+ }
+
+ public void createBucket(String bucket) {
+ if (!s3Client.doesBucketExistV2(bucket)) {
+ s3Client.createBucket(bucket);
+ }
+ }
+
+ public void putObject(String bucket, String key, Path localFile) throws
IOException {
+ s3Client.putObject(new PutObjectRequest(bucket, key,
localFile.toFile()));
+ }
+
+ public void deletePrefix(String bucket, String prefix) {
+ ListObjectsV2Request request = new ListObjectsV2Request()
+ .withBucketName(bucket)
+ .withPrefix(prefix);
+ ListObjectsV2Result listing;
+ do {
+ listing = s3Client.listObjectsV2(request);
+ List<String> keys = new ArrayList<>();
+ for (S3ObjectSummary summary : listing.getObjectSummaries()) {
+ keys.add(summary.getKey());
+ }
+ if (!keys.isEmpty()) {
+ s3Client.deleteObjects(new
DeleteObjectsRequest(bucket).withKeys(keys.toArray(new String[0])));
+ }
+ request.setContinuationToken(listing.getNextContinuationToken());
+ } while (listing.isTruncated());
+ }
+
+ public void shutdown() {
+ s3Client.shutdown();
+ }
+
+ @Override
+ public void close() {
+ shutdown();
+ }
+
+ private static AmazonS3 buildS3Client(String endpoint, String accessKey,
String secretKey) {
+ return AmazonS3ClientBuilder.standard()
+ .withEndpointConfiguration(new
AwsClientBuilder.EndpointConfiguration(endpoint, "us-east-1"))
+ .withPathStyleAccessEnabled(true)
+ .withCredentials(new AWSStaticCredentialsProvider(
+ new BasicAWSCredentials(accessKey, secretKey)))
+ .build();
+ }
+}
diff --git
a/automation/src/main/java/org/apache/cloudberry/pxf/automation/testcontainers/MinIOContainer.java
b/automation/src/main/java/org/apache/cloudberry/pxf/automation/testcontainers/MinIOContainer.java
new file mode 100644
index 00000000..17c6a2cf
--- /dev/null
+++
b/automation/src/main/java/org/apache/cloudberry/pxf/automation/testcontainers/MinIOContainer.java
@@ -0,0 +1,78 @@
+package org.apache.cloudberry.pxf.automation.testcontainers;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import org.testcontainers.containers.GenericContainer;
+import org.testcontainers.containers.Network;
+import org.testcontainers.containers.wait.strategy.Wait;
+import org.testcontainers.utility.DockerImageName;
+
+/**
+ * TestContainers wrapper around MinIO for S3 / S3 Select automation tests.
+ * The container joins a shared Docker network with alias minio, so PXF inside
the
+ * Cloudberry container can reach it at http://minio:9000.
+ *
+ * This class only manages the container lifecycle and exposes endpoint /
+ * credential accessors. S3 API access (buckets, objects) lives in
+ * {@link org.apache.cloudberry.pxf.automation.applications.S3Application}.
+ */
+public class MinIOContainer extends GenericContainer<MinIOContainer> {
+
+ private static final String DEFAULT_IMAGE =
"minio/minio:RELEASE.2024-11-07T00-52-20Z";
+ private static final String NETWORK_ALIAS = "minio";
+
+ public static final int API_PORT = 9000;
+ public static final int CONSOLE_PORT = 9001;
+
+ public static final String ACCESS_KEY = "admin";
+ public static final String SECRET_KEY = "password";
+ public static final String DEFAULT_BUCKET = "gpdb-ud-scratch";
+
+ public MinIOContainer(Network network) {
+ super(DockerImageName.parse(DEFAULT_IMAGE));
+
+ withNetwork(network)
+ .withNetworkAliases(NETWORK_ALIAS)
+ .withExposedPorts(API_PORT, CONSOLE_PORT)
+ .withEnv("MINIO_ROOT_USER", ACCESS_KEY)
+ .withEnv("MINIO_ROOT_PASSWORD", SECRET_KEY)
+ .withEnv("MINIO_API_SELECT_PARQUET", "on")
+ .withCommand("server", "/data", "--console-address", ":" +
CONSOLE_PORT)
+
.waitingFor(Wait.forHttp("/minio/health/live").forPort(API_PORT));
+ }
+
+ /** S3 API endpoint reachable from the test JVM (mapped port). */
+ public String getHostEndpoint() {
+ return "http://localhost:"; + getMappedPort(API_PORT);
+ }
+
+ /** S3 API endpoint for PXF and other containers on the same Docker
network. */
+ public String getInternalEndpoint() {
+ return "http://"; + NETWORK_ALIAS + ":" + API_PORT;
+ }
+
+ public String getAccessKey() {
+ return ACCESS_KEY;
+ }
+
+ public String getSecretKey() {
+ return SECRET_KEY;
+ }
+}
diff --git
a/automation/src/main/java/org/apache/cloudberry/pxf/automation/testcontainers/PXFCloudberryContainer.java
b/automation/src/main/java/org/apache/cloudberry/pxf/automation/testcontainers/PXFCloudberryContainer.java
index 5382c794..798af68a 100644
---
a/automation/src/main/java/org/apache/cloudberry/pxf/automation/testcontainers/PXFCloudberryContainer.java
+++
b/automation/src/main/java/org/apache/cloudberry/pxf/automation/testcontainers/PXFCloudberryContainer.java
@@ -31,8 +31,8 @@ import
org.testcontainers.containers.wait.strategy.AbstractWaitStrategy;
import org.testcontainers.containers.wait.strategy.Wait;
import org.testcontainers.utility.DockerImageName;
import org.testcontainers.utility.MountableFile;
+import org.apache.cloudberry.pxf.automation.utils.AutomationUtils;
-import java.io.File;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
@@ -137,7 +137,7 @@ public class PXFCloudberryContainer extends
GenericContainer<PXFCloudberryContai
*/
public static synchronized PXFCloudberryContainer getInstance() {
if (instance == null) {
- String repo = resolveProperty("pxf.test.repo.path",
findRepoPath());
+ String repo = resolveProperty("pxf.test.repo.path",
AutomationUtils.findRepoRoot().toString());
String distro = resolveDistro();
String imageName = "pxf/cbdb-testcontainer-" + distro + ":1";
String baseImage = BASE_IMAGES.getOrDefault(distro,
BASE_IMAGES.get("ubuntu"));
@@ -204,21 +204,6 @@ public class PXFCloudberryContainer extends
GenericContainer<PXFCloudberryContai
return (value != null && !value.isEmpty()) ? value : fallback;
}
- private static String findRepoPath() {
- File dir = new File(System.getProperty("user.dir"));
- for (int i = 0; i < 5; i++) {
- if (new File(dir, "automation/pom.xml").exists()) {
- return dir.getAbsolutePath();
- }
- dir = dir.getParentFile();
- if (dir == null)
- break;
- }
- throw new IllegalStateException(
- "Cannot auto-detect cloudberry-pxf repo root. Set
-Dpxf.test.repo.path=...");
- }
-
-
public Network getSharedNetwork() {
return network;
}
diff --git
a/automation/src/main/java/org/apache/cloudberry/pxf/automation/utils/AutomationUtils.java
b/automation/src/main/java/org/apache/cloudberry/pxf/automation/utils/AutomationUtils.java
new file mode 100644
index 00000000..c90aca2e
--- /dev/null
+++
b/automation/src/main/java/org/apache/cloudberry/pxf/automation/utils/AutomationUtils.java
@@ -0,0 +1,60 @@
+package org.apache.cloudberry.pxf.automation.utils;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import java.io.File;
+import java.nio.file.Path;
+
+/**
+ * Shared helpers for the cloudberry-pxf automation suite.
+ */
+public final class AutomationUtils {
+
+ private static final String REPO_MARKER = "automation/pom.xml";
+ private static final int MAX_PARENT_HOPS = 6;
+
+ private AutomationUtils() {
+ }
+
+ /**
+ * Walks upward from {@code user.dir} looking for the cloudberry-pxf repo
+ * root (identified by the {@code automation/pom.xml} marker). Used by code
+ * that needs to read resources from the working tree at runtime.
+ *
+ * @throws IllegalStateException if the marker is not found within a
handful
+ * of parent directories. Set {@code -Dpxf.test.repo.path=...} to
bypass.
+ */
+ public static Path findRepoRoot() {
+ File dir = new File(System.getProperty("user.dir"));
+ for (int i = 0; i < MAX_PARENT_HOPS; i++) {
+ if (new File(dir, REPO_MARKER).exists()) {
+ return dir.toPath().toAbsolutePath().normalize();
+ }
+ dir = dir.getParentFile();
+ if (dir == null) {
+ break;
+ }
+ }
+ throw new IllegalStateException(
+ "Cannot auto-detect cloudberry-pxf repo root from user.dir="
+ + System.getProperty("user.dir")
+ + ". Set -Dpxf.test.repo.path=... to override.");
+ }
+}
diff --git
a/automation/src/main/resources/testcontainers/pxf-cbdb/script/entrypoint.sh
b/automation/src/main/resources/testcontainers/pxf-cbdb/script/entrypoint.sh
index 388880c6..3ff2a513 100755
--- a/automation/src/main/resources/testcontainers/pxf-cbdb/script/entrypoint.sh
+++ b/automation/src/main/resources/testcontainers/pxf-cbdb/script/entrypoint.sh
@@ -234,52 +234,13 @@ EOF
</profiles>
EOF
- # Configure S3 settings
- mkdir -p "$PXF_BASE/servers/s3" "$PXF_HOME/servers/s3"
-
- for s3_site in "$PXF_BASE/servers/s3/s3-site.xml"
"$PXF_BASE/servers/default/s3-site.xml" "$PXF_HOME/servers/s3/s3-site.xml"; do
- mkdir -p "$(dirname "$s3_site")"
- cat > "$s3_site" <<'EOF'
-<?xml version="1.0" encoding="UTF-8"?>
-<configuration>
- <property>
- <name>fs.s3a.endpoint</name>
- <value>http://localhost:9000</value>
- </property>
- <property>
- <name>fs.s3a.access.key</name>
- <value>admin</value>
- </property>
- <property>
- <name>fs.s3a.secret.key</name>
- <value>password</value>
- </property>
- <property>
- <name>fs.s3a.path.style.access</name>
- <value>true</value>
- </property>
- <property>
- <name>fs.s3a.connection.ssl.enabled</name>
- <value>false</value>
- </property>
- <property>
- <name>fs.s3a.impl</name>
- <value>org.apache.hadoop.fs.s3a.S3AFileSystem</value>
- </property>
- <property>
- <name>fs.s3a.aws.credentials.provider</name>
- <value>org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider</value>
- </property>
-</configuration>
-EOF
+ # Configure pxf servers for S3 tests
+ local
s3_servers_src="${REPO_DIR}/automation/src/main/resources/testcontainers/pxf-cbdb/servers"
+ for server_name in s3 s3-invalid; do
+ local server_dir="$PXF_BASE/servers/${server_name}"
+ mkdir -p "$server_dir"
+ cp -v "${s3_servers_src}/${server_name}/s3-site.xml"
"$server_dir/s3-site.xml"
done
- mkdir -p /home/gpadmin/.aws/
- cat > "/home/gpadmin/.aws/credentials" <<'EOF'
-[default]
-aws_access_key_id = admin
-aws_secret_access_key = password
-EOF
-
}
main() {
diff --git
a/automation/src/main/resources/testcontainers/pxf-cbdb/script/pxf-env.sh
b/automation/src/main/resources/testcontainers/pxf-cbdb/script/pxf-env.sh
index 694ccac5..5babfb18 100755
--- a/automation/src/main/resources/testcontainers/pxf-cbdb/script/pxf-env.sh
+++ b/automation/src/main/resources/testcontainers/pxf-cbdb/script/pxf-env.sh
@@ -58,15 +58,6 @@ export
COORDINATOR_DATA_DIRECTORY=${COORDINATOR_DATA_DIRECTORY:-/home/gpadmin/wo
# set cloudberry timezone utc
export PGTZ=UTC
-# --------------------------------------------------------------------
-# Minio defaults
-# --------------------------------------------------------------------
-export AWS_ACCESS_KEY_ID=admin
-export AWS_SECRET_ACCESS_KEY=password
-export PROTOCOL=minio
-export ACCESS_KEY_ID=admin
-export SECRET_ACCESS_KEY=password
-
# --------------------------------------------------------------------
# PXF defaults
# --------------------------------------------------------------------
diff --git
a/automation/src/main/resources/testcontainers/pxf-cbdb/servers/s3-invalid/s3-site.xml
b/automation/src/main/resources/testcontainers/pxf-cbdb/servers/s3-invalid/s3-site.xml
new file mode 100644
index 00000000..b7557be3
--- /dev/null
+++
b/automation/src/main/resources/testcontainers/pxf-cbdb/servers/s3-invalid/s3-site.xml
@@ -0,0 +1,54 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!-- Licensed to the Apache Software Foundation (ASF) under one or more
contributor
+ license agreements. See the NOTICE file distributed with this work for
additional
+ information regarding copyright ownership. The ASF licenses this file to
+ you under the Apache License, Version 2.0 (the "License"); you may not
use
+ this file except in compliance with the License. You may obtain a copy
of
+ the License at http://www.apache.org/licenses/LICENSE-2.0 Unless
required
+ by applicable law or agreed to in writing, software distributed under
the
+ License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
CONDITIONS
+ OF ANY KIND, either express or implied. See the License for the specific
+ language governing permissions and limitations under the License.
+-->
+<!--
+ Pre-baked PXF server 's3-invalid' configuration for testcontainers
automation.
+
+ Endpoint points at the MinIO sidecar, but the credentials are deliberately
+ invalid. Used by CloudAccessTest to verify PXF credential-resolution
+ precedence: a server config with bad keys should surface an authentication
+ failure unless the external table provides overriding credentials (in
+ which case the table-level creds should win).
+
+ Shipped as-is and copied into ${PXF_BASE}/servers/s3-invalid/ by
+ entrypoint.sh.
+-->
+<configuration>
+ <property>
+ <name>fs.s3a.endpoint</name>
+ <value>http://minio:9000</value>
+ </property>
+ <property>
+ <name>fs.s3a.access.key</name>
+ <value>invalid-access-key</value>
+ </property>
+ <property>
+ <name>fs.s3a.secret.key</name>
+ <value>invalid-secret-key</value>
+ </property>
+ <property>
+ <name>fs.s3a.path.style.access</name>
+ <value>true</value>
+ </property>
+ <property>
+ <name>fs.s3a.connection.ssl.enabled</name>
+ <value>false</value>
+ </property>
+ <property>
+ <name>fs.s3a.impl</name>
+ <value>org.apache.hadoop.fs.s3a.S3AFileSystem</value>
+ </property>
+ <property>
+ <name>fs.s3a.aws.credentials.provider</name>
+ <value>org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider</value>
+ </property>
+</configuration>
diff --git
a/automation/src/main/resources/testcontainers/pxf-cbdb/servers/s3/s3-site.xml
b/automation/src/main/resources/testcontainers/pxf-cbdb/servers/s3/s3-site.xml
new file mode 100644
index 00000000..b76f19bc
--- /dev/null
+++
b/automation/src/main/resources/testcontainers/pxf-cbdb/servers/s3/s3-site.xml
@@ -0,0 +1,50 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!-- Licensed to the Apache Software Foundation (ASF) under one or more
contributor
+ license agreements. See the NOTICE file distributed with this work for
additional
+ information regarding copyright ownership. The ASF licenses this file to
+ you under the Apache License, Version 2.0 (the "License"); you may not
use
+ this file except in compliance with the License. You may obtain a copy
of
+ the License at http://www.apache.org/licenses/LICENSE-2.0 Unless
required
+ by applicable law or agreed to in writing, software distributed under
the
+ License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
CONDITIONS
+ OF ANY KIND, either express or implied. See the License for the specific
+ language governing permissions and limitations under the License.
+-->
+<!--
+ Pre-baked PXF server 's3' configuration for testcontainers automation.
+
+ Pointed at the MinIO sidecar that joins the shared Docker network with
+ alias 'minio' (see MinIOContainer).
+
+ Shipped as-is and copied into ${PXF_BASE}/servers/s3/ by entrypoint.sh.
+-->
+<configuration>
+ <property>
+ <name>fs.s3a.endpoint</name>
+ <value>http://minio:9000</value>
+ </property>
+ <property>
+ <name>fs.s3a.access.key</name>
+ <value>admin</value>
+ </property>
+ <property>
+ <name>fs.s3a.secret.key</name>
+ <value>password</value>
+ </property>
+ <property>
+ <name>fs.s3a.path.style.access</name>
+ <value>true</value>
+ </property>
+ <property>
+ <name>fs.s3a.connection.ssl.enabled</name>
+ <value>false</value>
+ </property>
+ <property>
+ <name>fs.s3a.impl</name>
+ <value>org.apache.hadoop.fs.s3a.S3AFileSystem</value>
+ </property>
+ <property>
+ <name>fs.s3a.aws.credentials.provider</name>
+ <value>org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider</value>
+ </property>
+</configuration>
diff --git
a/automation/src/test/java/org/apache/cloudberry/pxf/automation/AbstractTestcontainersTest.java
b/automation/src/test/java/org/apache/cloudberry/pxf/automation/AbstractTestcontainersTest.java
index bda4f5de..8eabd4c2 100644
---
a/automation/src/test/java/org/apache/cloudberry/pxf/automation/AbstractTestcontainersTest.java
+++
b/automation/src/test/java/org/apache/cloudberry/pxf/automation/AbstractTestcontainersTest.java
@@ -28,10 +28,14 @@ import
org.apache.cloudberry.pxf.automation.testcontainers.PXFCloudberryContaine
import org.apache.cloudberry.pxf.automation.utils.system.FDWUtils;
import org.apache.cloudberry.pxf.automation.utils.system.ProtocolUtils;
import org.testng.annotations.AfterClass;
+import org.testng.annotations.AfterMethod;
import org.testng.annotations.BeforeClass;
+import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Listeners;
import reporters.CustomAutomationReport;
+import java.lang.reflect.Method;
+
@Listeners({CustomAutomationLogger.class, CustomAutomationReport.class,
FDWSkipTestAnalyzer.class})
public class AbstractTestcontainersTest {
@@ -74,7 +78,7 @@ public class AbstractTestcontainersTest {
regress = new RegressApplication(container);
- // run users before class
+ // run user's before class
beforeClass();
} finally {
CustomAutomationLogger.revertStdoutStream();
@@ -87,8 +91,11 @@ public class AbstractTestcontainersTest {
if (ProtocolUtils.getPxfTestKeepData().equals("true")) {
return;
}
+ // redirect "clean" logs to log file
CustomAutomationLogger.redirectStdoutStreamToFile(getClass().getSimpleName(),
"clean");
try {
+ // run user's after class
+ afterClass();
if (cloudberry != null) {
cloudberry.close();
}
@@ -97,6 +104,56 @@ public class AbstractTestcontainersTest {
}
}
+
+ /**
+ * will be called before each test method start
+ *
+ * @throws Exception
+ */
+ @BeforeMethod(alwaysRun = true)
+ public void runBeforeMethod() throws Exception {
+ // check if "beforeMethod exists and if so open log file and run it
+ if (checkMethodImplExists("beforeMethod")) {
+ // redirect "runBeforeMethod" logs to log file
+
CustomAutomationLogger.redirectStdoutStreamToFile(getClass().getSimpleName(),
"beforeMethod");
+ try {
+ beforeMethod();
+ } catch (Throwable t) {
+ // in case of failure write stack trace to file stream and
throw the exception
+ t.printStackTrace(System.out);
+ throw t;
+ } finally {
+ // anyways revert System.out to original stream
+ CustomAutomationLogger.revertStdoutStream();
+ }
+ }
+ }
+
+ /**
+ * will be called after each test method ended
+ *
+ * @throws Exception
+ */
+ @AfterMethod(alwaysRun = true)
+ public void runAfterMethod() throws Exception {
+ // check if "afterMethod exists and if so open log file and run it
+ if (checkMethodImplExists("afterMethod")) {
+ // redirect "runAfterMethod" logs to log file
+
CustomAutomationLogger.redirectStdoutStreamToFile(getClass().getSimpleName(),
"afterMethod");
+ try {
+ afterMethod();
+ } catch (Throwable t) {
+ // in case of failure write stack trace to file stream and
throw the exception
+ t.printStackTrace(System.out);
+ throw t;
+ } finally {
+ // anyways revert System.out to original stream
+ CustomAutomationLogger.revertStdoutStream();
+ }
+ }
+ }
+
+
/**
* clean up after the class finished
*
@@ -136,4 +193,25 @@ public class AbstractTestcontainersTest {
bootstrap.runQuery("SELECT 1");
System.out.println("[" + getClass().getSimpleName() + "] Test
databases created");
}
+
+ /**
+ * Check if the test writer used given method and return true if so.
+ *
+ * @param methodName to check
+ * @return true if method exists in declared methods
+ * @throws NoSuchMethodException
+ * @throws SecurityException
+ */
+ private boolean checkMethodImplExists(String methodName) throws
NoSuchMethodException, SecurityException {
+ // get all declared methods
+ Method[] methods = getClass().getDeclaredMethods();
+ // run over methods and look for methodName
+ for (Method method : methods) {
+
+ if (method.getName().equals(methodName)) {
+ return true;
+ }
+ }
+ return false;
+ }
}
diff --git
a/automation/src/test/java/org/apache/cloudberry/pxf/automation/SmallDataFactory.java
b/automation/src/test/java/org/apache/cloudberry/pxf/automation/SmallDataFactory.java
new file mode 100644
index 00000000..7042bccd
--- /dev/null
+++
b/automation/src/test/java/org/apache/cloudberry/pxf/automation/SmallDataFactory.java
@@ -0,0 +1,97 @@
+package org.apache.cloudberry.pxf.automation;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.cloudberry.pxf.automation.structures.tables.basic.Table;
+
+import java.io.BufferedWriter;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Shared test-data fixtures for Testcontainers-based tests.
+ *
+ * Mirrors the data layout of {@link BaseFunctionality}
+ */
+public class SmallDataFactory {
+
+ /**
+ * Create a data table with {@code numRows} rows of small data,
+ * with the following fields: String, int, double, long and boolean.
+ *
+ * @param uniqueName prefix applied to the {@code name} column (empty for
no prefix)
+ * @param numRows number of rows to generate
+ * @return the generated {@link Table}
+ */
+ public Table getSmallData(String uniqueName, int numRows) {
+ List<List<String>> data = new ArrayList<>();
+
+ for (int i = 1; i <= numRows; i++) {
+ List<String> row = new ArrayList<>();
+ row.add(String.format("%s%srow_%d", uniqueName,
StringUtils.isBlank(uniqueName) ? "" : "_", i));
+ row.add(String.valueOf(i));
+ row.add(Double.toString(i));
+ row.add(Long.toString(100000000000L * i));
+ row.add(String.valueOf(i % 2 == 0));
+ data.add(row);
+ }
+
+ Table dataTable = new Table("dataTable", null);
+ dataTable.setData(data);
+
+ return dataTable;
+ }
+
+ public Table getSmallData() {
+ return getSmallData("");
+ }
+
+ public Table getSmallData(String uniqueName) {
+ return getSmallData(uniqueName, 100);
+ }
+
+ /**
+ * Serialize a table to a temporary CSV file, rows separated by newlines
and
+ * fields separated by {@code delimiter}, with no trailing newline. The
caller
+ * owns the returned file and is responsible for deleting it.
+ *
+ * @param table the table to serialize
+ * @param delimiter the field delimiter
+ * @return path to the temporary CSV file
+ */
+ public Path writeTableToCsv(Table table, String delimiter) throws
IOException {
+ Path tempFile = Files.createTempFile("tc-fixture-", ".csv");
+ List<List<String>> data = table.getData();
+ try (BufferedWriter writer = Files.newBufferedWriter(tempFile,
StandardCharsets.UTF_8)) {
+ for (int i = 0; i < data.size(); i++) {
+ writer.append(String.join(delimiter, data.get(i)));
+ if (i != data.size() - 1) {
+ writer.newLine();
+ }
+ }
+ }
+ return tempFile;
+ }
+}
diff --git
a/automation/src/test/java/org/apache/cloudberry/pxf/automation/features/cloud/CloudAccessTest.java
b/automation/src/test/java/org/apache/cloudberry/pxf/automation/features/cloud/CloudAccessTest.java
index 13f0754f..6fcdb3e3 100644
---
a/automation/src/test/java/org/apache/cloudberry/pxf/automation/features/cloud/CloudAccessTest.java
+++
b/automation/src/test/java/org/apache/cloudberry/pxf/automation/features/cloud/CloudAccessTest.java
@@ -1,28 +1,44 @@
package org.apache.cloudberry.pxf.automation.features.cloud;
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
import annotations.WorksWithFDW;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.cloudberry.pxf.automation.components.hdfs.Hdfs;
-import org.apache.cloudberry.pxf.automation.features.BaseFeature;
-import org.apache.cloudberry.pxf.automation.structures.tables.basic.Table;
+import org.apache.cloudberry.pxf.automation.AbstractTestcontainersTest;
+import org.apache.cloudberry.pxf.automation.SmallDataFactory;
+import org.apache.cloudberry.pxf.automation.applications.PXFApplication;
+import org.apache.cloudberry.pxf.automation.applications.S3Application;
+import
org.apache.cloudberry.pxf.automation.structures.tables.pxf.ExternalTable;
import
org.apache.cloudberry.pxf.automation.structures.tables.utils.TableFactory;
-import org.apache.cloudberry.pxf.automation.utils.system.ProtocolUtils;
-import org.apache.cloudberry.pxf.automation.utils.system.ProtocolEnum;
+import org.apache.cloudberry.pxf.automation.testcontainers.MinIOContainer;
import org.testng.annotations.Test;
-import java.net.URI;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
import java.util.UUID;
/**
* Functional CloudAccess Test
*/
@WorksWithFDW
-public class CloudAccessTest extends BaseFeature {
-
- private static final String PROTOCOL_S3 = "s3a://";
- private static final String S3_ENDPOINT =
- System.getProperty("S3_ENDPOINT",
System.getenv().getOrDefault("S3_ENDPOINT", "http://localhost:9000";));
+public class CloudAccessTest extends AbstractTestcontainersTest {
private static final String[] PXF_MULTISERVER_COLS = {
"name text",
@@ -37,55 +53,71 @@ public class CloudAccessTest extends BaseFeature {
"score integer"
};
- private Hdfs s3Server;
- private String s3PathRead, s3PathWrite;
+ private static final String fileName = "data.txt";
+
+ private final SmallDataFactory dataFactory = new SmallDataFactory();
+
+ private MinIOContainer s3Server;
+ private S3Application s3Application;
+ private String s3PathRead;
+ private String s3PathWrite;
+ private String readObjectKeyPrefix;
+ private String writeObjectKeyPrefix;
- /**
- * Prepare all server configurations and components
- */
@Override
public void beforeClass() throws Exception {
- // Initialize server objects
- String random = UUID.randomUUID().toString();
- s3PathRead =
String.format("gpdb-ud-scratch/tmp/pxf_automation_data_read/%s/" , random);
- s3PathWrite =
String.format("gpdb-ud-scratch/tmp/pxf_automation_data_write/%s/", random);
+ s3Server = new MinIOContainer(container.getSharedNetwork());
+ s3Server.start();
+ s3Application = new S3Application(s3Server);
+ s3Application.createBucket(MinIOContainer.DEFAULT_BUCKET);
- Configuration s3Configuration = new Configuration();
- s3Configuration.set("fs.s3a.access.key", ProtocolUtils.getAccess());
- s3Configuration.set("fs.s3a.secret.key", ProtocolUtils.getSecret());
- applyS3Defaults(s3Configuration);
+ String random = UUID.randomUUID().toString();
+ readObjectKeyPrefix =
String.format("tmp/pxf_automation_data_read/%s/", random);
+ writeObjectKeyPrefix =
String.format("tmp/pxf_automation_data_write/%s/", random);
+ s3PathRead = MinIOContainer.DEFAULT_BUCKET + "/" + readObjectKeyPrefix;
+ s3PathWrite = MinIOContainer.DEFAULT_BUCKET + "/" +
writeObjectKeyPrefix;
+ }
- FileSystem fs2 = FileSystem.get(URI.create(PROTOCOL_S3 + s3PathRead +
fileName), s3Configuration);
- s3Server = new Hdfs(fs2, s3Configuration, true);
+ @Override
+ public void afterClass() throws Exception {
+ if (s3Application != null) {
+ if (readObjectKeyPrefix != null) {
+ s3Application.deletePrefix(MinIOContainer.DEFAULT_BUCKET,
readObjectKeyPrefix);
+ }
+ if (writeObjectKeyPrefix != null) {
+ s3Application.deletePrefix(MinIOContainer.DEFAULT_BUCKET,
writeObjectKeyPrefix);
+ }
+ s3Application.shutdown();
+ }
+ if (s3Server != null) {
+ s3Server.stop();
+ }
}
@Override
protected void beforeMethod() throws Exception {
- if (ProtocolUtils.getProtocol() == ProtocolEnum.HDFS) {
- return;
+ uploadSmallCsvFixture(MinIOContainer.DEFAULT_BUCKET,
readObjectKeyPrefix + fileName);
+ }
+
+ // Uploads small CSV test data (see BaseTCFunctionality#getSmallData()) to
the given S3 object.
+ private void uploadSmallCsvFixture(String bucket, String objectKey) throws
IOException {
+ Path tempFile =
dataFactory.writeTableToCsv(dataFactory.getSmallData(), ",");
+ try {
+ System.out.println("[CloudAccessTest] Uploading " + tempFile + "
-> s3://" + bucket + "/" + objectKey);
+ s3Application.putObject(bucket, objectKey, tempFile);
+ } finally {
+ Files.deleteIfExists(tempFile);
}
- super.beforeMethod();
- prepareData();
}
@Override
protected void afterMethod() throws Exception {
- super.afterMethod();
- if (s3Server != null) {
- s3Server.removeDirectory(PROTOCOL_S3 + s3PathRead);
- s3Server.removeDirectory(PROTOCOL_S3 + s3PathWrite);
+ if (s3Application != null) {
+ s3Application.deletePrefix(MinIOContainer.DEFAULT_BUCKET,
readObjectKeyPrefix);
+ s3Application.deletePrefix(MinIOContainer.DEFAULT_BUCKET,
writeObjectKeyPrefix);
}
}
- protected void prepareData() throws Exception {
- // Prepare data in table
- Table dataTable = getSmallData();
-
- // Create Data for s3Server
- s3Server.writeTableToFile(PROTOCOL_S3 + s3PathRead + fileName,
dataTable, ",");
- s3Server.createDirectory(PROTOCOL_S3 + s3PathWrite);
- }
-
/*
* The tests below are for the case where there's NO Hadoop cluster
configured under "default" server
* and assumes the "default" server has not configuration files. They are
part of "s3" group and do not
@@ -133,68 +165,66 @@ public class CloudAccessTest extends BaseFeature {
* both without and with Kerberos security, testing that cloud access
works in presence of "default" server
*/
- @Test(groups = {"gpdb", "security"})
+ @Test(groups = {"testcontainers", "pxf-s3"})
public void testCloudAccessWithHdfsFailsWhenNoServerNoCredsSpecified()
throws Exception {
runTestScenario("no_server_no_credentials_with_hdfs", null, false);
}
- // TODO: pxf_regress shows diff for this test. Should be fixed.
- @Test(enabled = false, groups = {"gpdb", "security"})
+ @Test(groups = {"testcontainers", "pxf-s3"})
public void
testCloudAccessWithHdfsOkWhenServerNoCredsValidConfigFileExists() throws
Exception {
runTestScenario("server_no_credentials_valid_config_with_hdfs", "s3",
false);
}
- @Test(groups = {"gpdb", "security"})
+ @Test(groups = {"testcontainers", "pxf-s3"})
public void
testCloudWriteWithHdfsOkWhenServerNoCredsValidConfigFileExists() throws
Exception {
runTestScenarioForWrite("server_no_credentials_valid_config_with_hdfs_write",
"s3", false);
}
- @Test(groups = {"gpdb", "security"})
+ @Test(groups = {"testcontainers", "pxf-s3"})
public void
testCloudAccessWithHdfsFailsWhenServerNoCredsNoConfigFileExists() throws
Exception {
runTestScenario("server_no_credentials_no_config_with_hdfs",
"s3-non-existent", false);
}
- @Test(groups = {"gpdb", "security"})
+ @Test(groups = {"testcontainers", "pxf-s3"})
public void
testCloudAccessWithHdfsFailsWhenNoServerCredsNoConfigFileExists() throws
Exception {
runTestScenario("no_server_credentials_no_config_with_hdfs", null,
true);
}
- @Test(groups = {"gpdb", "security"})
+ @Test(groups = {"testcontainers", "pxf-s3"})
public void
testCloudAccessWithHdfsFailsWhenServerNoCredsInvalidConfigFileExists() throws
Exception {
runTestScenario("server_no_credentials_invalid_config_with_hdfs",
"s3-invalid", false);
}
- // TODO: pxf_regress shows diff for this test. Should be fixed.
- @Test(enabled = false, groups = {"gpdb", "security"})
+ @Test(groups = {"testcontainers", "pxf-s3"})
public void
testCloudAccessWithHdfsOkWhenServerCredsInvalidConfigFileExists() throws
Exception {
runTestScenario("server_credentials_invalid_config_with_hdfs",
"s3-invalid", true);
}
private void runTestScenario(String name, String server, boolean creds)
throws Exception {
String tableName = "cloudaccess_" + name;
- exTable = TableFactory.getPxfReadableTextTable(tableName,
PXF_MULTISERVER_COLS, s3PathRead + fileName, ",");
+ ExternalTable exTable =
TableFactory.getPxfReadableTextTable(tableName, PXF_MULTISERVER_COLS,
s3PathRead + fileName, ",");
exTable.setProfile("s3:text");
String serverParam = (server == null) ? null : "server=" + server;
exTable.setServer(serverParam);
if (creds) {
- exTable.setUserParameters(new String[]{"accesskey=" +
ProtocolUtils.getAccess(), "secretkey=" + ProtocolUtils.getSecret()});
+ exTable.setUserParameters(new String[]{"accesskey=" +
MinIOContainer.ACCESS_KEY, "secretkey=" + MinIOContainer.SECRET_KEY});
}
- gpdb.createTableAndVerify(exTable);
+ cloudberry.createTableAndVerify(exTable);
- runSqlTest("features/cloud_access/" + name);
+ regress.runSqlTest("features/cloud_access/" + name);
}
private void runTestScenarioForWrite(String name, String server, boolean
creds) throws Exception {
// create writable external table to write to S3
String tableName = "cloudwrite_" + name;
- exTable = TableFactory.getPxfWritableTextTable(tableName,
PXF_WRITE_COLS, s3PathWrite, ",");
+ ExternalTable exTable =
TableFactory.getPxfWritableTextTable(tableName, PXF_WRITE_COLS, s3PathWrite,
",");
exTable.setProfile("s3:text");
String serverParam = (server == null) ? null : "server=" + server;
exTable.setServer(serverParam);
if (creds) {
- exTable.setUserParameters(new String[]{"accesskey=" +
ProtocolUtils.getAccess(), "secretkey=" + ProtocolUtils.getSecret()});
+ exTable.setUserParameters(new String[]{"accesskey=" +
MinIOContainer.ACCESS_KEY, "secretkey=" + MinIOContainer.SECRET_KEY});
}
- gpdb.createTableAndVerify(exTable);
+ cloudberry.createTableAndVerify(exTable);
// create readable external table to read back from S3, making sure
previous insert made it all the way to S3
tableName = "cloudaccess_" + name;
@@ -202,19 +232,10 @@ public class CloudAccessTest extends BaseFeature {
exTable.setProfile("s3:text");
exTable.setServer(serverParam);
if (creds) {
- exTable.setUserParameters(new String[]{"accesskey=" +
ProtocolUtils.getAccess(), "secretkey=" + ProtocolUtils.getSecret()});
+ exTable.setUserParameters(new String[]{"accesskey=" +
MinIOContainer.ACCESS_KEY, "secretkey=" + MinIOContainer.SECRET_KEY});
}
- gpdb.createTableAndVerify(exTable);
-
- runSqlTest("features/cloud_access/" + name);
- }
+ cloudberry.createTableAndVerify(exTable);
- private void applyS3Defaults(Configuration configuration) {
- configuration.set("fs.s3a.endpoint", S3_ENDPOINT);
- configuration.set("fs.s3a.path.style.access", "true");
- configuration.set("fs.s3a.connection.ssl.enabled", "false");
- configuration.set("fs.s3a.impl",
"org.apache.hadoop.fs.s3a.S3AFileSystem");
- configuration.set("fs.s3a.aws.credentials.provider",
- "org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider");
+ regress.runSqlTest("features/cloud_access/" + name);
}
}
diff --git
a/automation/src/test/java/org/apache/cloudberry/pxf/automation/features/cloud/S3SelectTest.java
b/automation/src/test/java/org/apache/cloudberry/pxf/automation/features/cloud/S3SelectTest.java
index 2b51bcd0..9986635c 100644
---
a/automation/src/test/java/org/apache/cloudberry/pxf/automation/features/cloud/S3SelectTest.java
+++
b/automation/src/test/java/org/apache/cloudberry/pxf/automation/features/cloud/S3SelectTest.java
@@ -1,27 +1,41 @@
package org.apache.cloudberry.pxf.automation.features.cloud;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.cloudberry.pxf.automation.components.hdfs.Hdfs;
-import org.apache.cloudberry.pxf.automation.features.BaseFeature;
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import org.apache.cloudberry.pxf.automation.AbstractTestcontainersTest;
+import org.apache.cloudberry.pxf.automation.applications.S3Application;
import
org.apache.cloudberry.pxf.automation.structures.tables.pxf.ReadableExternalTable;
-import org.apache.cloudberry.pxf.automation.utils.system.ProtocolEnum;
-import org.apache.cloudberry.pxf.automation.utils.system.ProtocolUtils;
+import org.apache.cloudberry.pxf.automation.testcontainers.MinIOContainer;
+import org.apache.cloudberry.pxf.automation.utils.AutomationUtils;
import org.testng.annotations.Test;
-import java.net.URI;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
import java.util.UUID;
import static
org.apache.cloudberry.pxf.automation.features.tpch.LineItem.LINEITEM_SCHEMA;
-/**
- * Functional S3 Select Test
- */
-public class S3SelectTest extends BaseFeature {
-
- private static final String PROTOCOL_S3 = "s3a://";
- private static final String S3_ENDPOINT =
- System.getProperty("S3_ENDPOINT",
System.getenv().getOrDefault("S3_ENDPOINT", "http://localhost:9000";));
+/** Functional S3 Select Test */
+public class S3SelectTest extends AbstractTestcontainersTest {
private static final String[] PXF_S3_SELECT_INVALID_COLS = {
"invalid_orderkey BIGINT",
@@ -42,8 +56,10 @@ public class S3SelectTest extends BaseFeature {
"invalid_comment VARCHAR(44)"
};
- private Hdfs s3Server;
+ private MinIOContainer s3Server;
+ private S3Application s3Application;
private String s3Path;
+ private String objectKeyPrefix;
private static final String sampleCsvFile = "sample.csv";
private static final String sampleGzippedCsvFile = "sample.csv.gz";
@@ -53,145 +69,149 @@ public class S3SelectTest extends BaseFeature {
private static final String sampleParquetSnappyFile =
"sample.snappy.parquet";
private static final String sampleParquetGzipFile = "sample.gz.parquet";
+ private static final String[] FIXTURE_FILES = {
+ sampleCsvFile,
+ sampleCsvNoHeaderFile,
+ sampleGzippedCsvFile,
+ sampleBzip2CsvFile,
+ sampleParquetFile,
+ sampleParquetSnappyFile,
+ sampleParquetGzipFile,
+ };
+
+ private static final String FIXTURES_SUBDIR = "s3select";
+
/**
* Prepare all server configurations and components
*/
@Override
public void beforeClass() throws Exception {
- if (ProtocolUtils.getProtocol() == ProtocolEnum.HDFS) {
- return;
- }
- // Initialize server objects
- s3Path =
String.format("gpdb-ud-scratch/tmp/pxf_automation_data/%s/s3select/",
UUID.randomUUID().toString());
- Configuration s3Configuration = new Configuration();
- s3Configuration.set("fs.s3a.access.key", ProtocolUtils.getAccess());
- s3Configuration.set("fs.s3a.secret.key", ProtocolUtils.getSecret());
- applyS3Defaults(s3Configuration);
+ s3Server = new MinIOContainer(container.getSharedNetwork());
+ s3Server.start();
+ s3Application = new S3Application(s3Server);
+ s3Application.createBucket(MinIOContainer.DEFAULT_BUCKET);
+
+ String uuid = UUID.randomUUID().toString();
+ objectKeyPrefix = "tmp/pxf_automation_data/" + uuid + "/s3select/";
+ s3Path = MinIOContainer.DEFAULT_BUCKET + "/" + objectKeyPrefix;
- FileSystem fs2 = FileSystem.get(URI.create(PROTOCOL_S3 + s3Path +
fileName), s3Configuration);
- s3Server = new Hdfs(fs2, s3Configuration, true);
+ uploadFixtures(MinIOContainer.DEFAULT_BUCKET, objectKeyPrefix);
+ // Server 's3' is pre-baked into the container image by entrypoint.sh.
}
@Override
- protected void afterClass() throws Exception {
- super.afterClass();
+ public void afterClass() throws Exception {
+ if (s3Application != null && objectKeyPrefix != null) {
+ s3Application.deletePrefix(MinIOContainer.DEFAULT_BUCKET,
objectKeyPrefix);
+ s3Application.shutdown();
+ }
if (s3Server != null) {
- s3Server.removeDirectory(PROTOCOL_S3 + s3Path);
+ s3Server.stop();
+ }
+ }
+
+ // Uploads committed S3 Select fixture files from
src/test/resources/data/s3select/ into MinIO.
+ private void uploadFixtures(String bucket, String objectKeyPrefix) throws
IOException {
+ Path fixturesDir = resolveFixturesDirectory();
+ for (String filename : FIXTURE_FILES) {
+ Path localFile = fixturesDir.resolve(filename);
+ if (!Files.isRegularFile(localFile)) {
+ throw new IOException("Missing S3 Select fixture: " +
localFile);
+ }
+ String key = objectKeyPrefix + filename;
+ System.out.println("[S3SelectTest] Uploading " + localFile + " ->
s3://" + bucket + "/" + key);
+ s3Application.putObject(bucket, key, localFile);
}
}
- // TODO: pxf_regress shows diff for this test. Should be fixed.
- @Test(enabled = false, groups = {"s3"})
+ private static Path resolveFixturesDirectory() throws IOException {
+ Path relative = Paths.get("src/test/resources/data", FIXTURES_SUBDIR);
+ if (Files.isDirectory(relative)) {
+ return relative.toAbsolutePath().normalize();
+ }
+ Path fromRepo = AutomationUtils.findRepoRoot()
+ .resolve("automation/src/test/resources/data")
+ .resolve(FIXTURES_SUBDIR);
+ if (Files.isDirectory(fromRepo)) {
+ return fromRepo;
+ }
+ throw new IOException("Cannot find s3select fixtures directory (tried "
+ + relative.toAbsolutePath() + " and " + fromRepo + ")");
+ }
+
+ @Test(groups = {"testcontainers", "pxf-s3"})
public void testPlainCsvWithHeaders() throws Exception {
String[] userParameters = {"FILE_HEADER=IGNORE", "S3_SELECT=ON"};
- runTestScenario("csv", "s3", "csv", s3Path,
- localDataResourcesFolder + "/s3select/", sampleCsvFile,
- "|", userParameters);
+ runTestScenario("csv", "s3", "csv", sampleCsvFile, "|",
userParameters);
}
- // TODO: pxf_regress shows diff for this test. Should be fixed.
- @Test(enabled = false, groups = {"s3"})
+ @Test(groups = {"testcontainers", "pxf-s3"})
public void testPlainCsvWithHeadersUsingHeaderInfo() throws Exception {
String[] userParameters = {"FILE_HEADER=USE", "S3_SELECT=ON"};
- runTestScenario("csv_use_headers", "s3", "csv", s3Path,
- localDataResourcesFolder + "/s3select/", sampleCsvFile,
- "|", userParameters);
+ runTestScenario("csv_use_headers", "s3", "csv", sampleCsvFile, "|",
userParameters);
}
- // TODO: pxf_regress shows diff for this test. Should be fixed.
- @Test(enabled = false, groups = {"s3"})
+ @Test(groups = {"testcontainers", "pxf-s3"})
public void testCsvWithHeadersUsingHeaderInfoWithWrongColumnNames() throws
Exception {
String[] userParameters = {"FILE_HEADER=USE", "S3_SELECT=ON"};
- runTestScenario("errors/", "csv_use_headers_with_wrong_col_names",
"s3", "csv", s3Path,
- localDataResourcesFolder + "/s3select/", sampleCsvFile, "/" +
s3Path + sampleCsvFile,
+ runTestScenario("errors/", "csv_use_headers_with_wrong_col_names",
"s3", "csv",
+ sampleCsvFile, "/" + s3Path + sampleCsvFile,
"|", userParameters, PXF_S3_SELECT_INVALID_COLS);
}
- // TODO: pxf_regress shows diff for this test. Should be fixed.
- @Test(enabled = false, groups = {"s3"})
+ @Test(groups = {"testcontainers", "pxf-s3"})
public void testPlainCsvWithNoHeaders() throws Exception {
String[] userParameters = {"FILE_HEADER=NONE", "S3_SELECT=ON"};
- runTestScenario("csv_noheaders", "s3", "csv", s3Path,
- localDataResourcesFolder + "/s3select/", sampleCsvNoHeaderFile,
- "|", userParameters);
+ runTestScenario("csv_noheaders", "s3", "csv", sampleCsvNoHeaderFile,
"|", userParameters);
}
- // TODO: pxf_regress shows diff for this test. Should be fixed.
- @Test(enabled = false, groups = {"s3"})
+ @Test(groups = {"testcontainers", "pxf-s3"})
public void testGzipCsvWithHeadersUsingHeaderInfo() throws Exception {
String[] userParameters = {"FILE_HEADER=USE", "S3_SELECT=ON",
"COMPRESSION_CODEC=gzip"};
- runTestScenario("gzip_csv_use_headers", "s3", "csv", s3Path,
- localDataResourcesFolder + "/s3select/", sampleGzippedCsvFile,
- "|", userParameters);
+ runTestScenario("gzip_csv_use_headers", "s3", "csv",
sampleGzippedCsvFile, "|", userParameters);
}
- // TODO: pxf_regress shows diff for this test. Should be fixed.
- @Test(enabled = false, groups = {"s3"})
+ @Test(groups = {"testcontainers", "pxf-s3"})
public void testBzip2CsvWithHeadersUsingHeaderInfo() throws Exception {
String[] userParameters = {"FILE_HEADER=USE", "S3_SELECT=ON",
"COMPRESSION_CODEC=bzip2"};
- runTestScenario("bzip2_csv_use_headers", "s3", "csv", s3Path,
- localDataResourcesFolder + "/s3select/", sampleBzip2CsvFile,
- "|", userParameters);
+ runTestScenario("bzip2_csv_use_headers", "s3", "csv",
sampleBzip2CsvFile, "|", userParameters);
}
- // TODO: pxf_regress shows diff for this test. Should be fixed.
- @Test(enabled = false, groups = {"s3"})
+ @Test(groups = {"testcontainers", "pxf-s3"})
public void testParquet() throws Exception {
String[] userParameters = {"S3_SELECT=ON"};
- runTestScenario("parquet", "s3", "parquet", s3Path,
- localDataResourcesFolder + "/s3select/", sampleParquetFile,
- null, userParameters);
+ runTestScenario("parquet", "s3", "parquet", sampleParquetFile, null,
userParameters);
}
- // TODO: pxf_regress shows diff for this test. Should be fixed.
- @Test(enabled = false, groups = {"s3"})
+ @Test(groups = {"testcontainers", "pxf-s3"})
public void testParquetWildcardLocation() throws Exception {
String[] userParameters = {"S3_SELECT=ON"};
- runTestScenario("", "parquet", "s3", "parquet", s3Path,
- localDataResourcesFolder + "/s3select/", sampleParquetFile,
"/" + s3Path + "*e.parquet",
+ runTestScenario("", "parquet", "s3", "parquet", sampleParquetFile, "/"
+ s3Path + "*e.parquet",
null, userParameters, LINEITEM_SCHEMA);
}
- // TODO: pxf_regress shows diff for this test. Should be fixed.
- @Test(enabled = false, groups = {"s3"})
+ @Test(groups = {"testcontainers", "pxf-s3"})
public void testSnappyParquet() throws Exception {
String[] userParameters = {"S3_SELECT=ON"};
- runTestScenario("parquet_snappy", "s3", "parquet", s3Path,
- localDataResourcesFolder + "/s3select/",
sampleParquetSnappyFile,
- null, userParameters);
+ runTestScenario("parquet_snappy", "s3", "parquet",
sampleParquetSnappyFile, null, userParameters);
}
- // TODO: pxf_regress shows diff for this test. Should be fixed.
- @Test(enabled = false, groups = {"s3"})
+ @Test(groups = {"testcontainers", "pxf-s3"})
public void testGzipParquet() throws Exception {
String[] userParameters = {"S3_SELECT=ON"};
- runTestScenario("parquet_gzip", "s3", "parquet", s3Path,
- localDataResourcesFolder + "/s3select/", sampleParquetGzipFile,
- null, userParameters);
+ runTestScenario("parquet_gzip", "s3", "parquet",
sampleParquetGzipFile, null, userParameters);
}
private void runTestScenario(
String name,
String server,
String format,
- String s3Path,
- String srcPath,
String filename,
String delimiter,
String[] userParameters)
throws Exception {
-
- runTestScenario("",
- name,
- server,
- format,
- s3Path,
- srcPath,
- filename,
- "/" + s3Path + filename,
- delimiter,
- userParameters,
- LINEITEM_SCHEMA);
+ runTestScenario("", name, server, format, filename, "/" + s3Path +
filename,
+ delimiter, userParameters, LINEITEM_SCHEMA);
}
private void runTestScenario(
@@ -199,40 +219,27 @@ public class S3SelectTest extends BaseFeature {
String name,
String server,
String format,
- String s3Path,
- String srcPath,
String filename,
String locationPath,
String delimiter,
String[] userParameters,
String[] fields)
throws Exception {
-
String tableName = "s3select_" + name;
- String serverParam = (server == null) ? null : "server=" + server;
-
- s3Server.copyFromLocal(srcPath + filename, PROTOCOL_S3 + s3Path +
filename);
-
- exTable = new ReadableExternalTable(tableName, fields, locationPath,
"CSV");
+ ReadableExternalTable exTable = new ReadableExternalTable(tableName,
fields, locationPath, "CSV");
exTable.setProfile("s3:" + format);
- exTable.setServer(serverParam);
+ exTable.setServer("server=" + server);
+ exTable.setHost(pxfHost);
+ exTable.setPort(pxfPort);
- if (delimiter != null)
+ if (delimiter != null) {
exTable.setDelimiter(delimiter);
- if (userParameters != null)
+ }
+ if (userParameters != null) {
exTable.setUserParameters(userParameters);
+ }
- gpdb.createTableAndVerify(exTable);
-
- runSqlTest(String.format("features/s3_select/%s%s", qualifier, name));
- }
-
- private void applyS3Defaults(Configuration configuration) {
- configuration.set("fs.s3a.endpoint", S3_ENDPOINT);
- configuration.set("fs.s3a.path.style.access", "true");
- configuration.set("fs.s3a.connection.ssl.enabled", "false");
- configuration.set("fs.s3a.impl",
"org.apache.hadoop.fs.s3a.S3AFileSystem");
- configuration.set("fs.s3a.aws.credentials.provider",
- "org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider");
+ cloudberry.createTableAndVerify(exTable);
+ regress.runSqlTest(String.format("features/s3_select/%s%s", qualifier,
name));
}
}
diff --git a/ci/docker/pxf-cbdb-dev/common/script/entrypoint.sh
b/ci/docker/pxf-cbdb-dev/common/script/entrypoint.sh
index a08bff16..7a4fe638 100755
--- a/ci/docker/pxf-cbdb-dev/common/script/entrypoint.sh
+++ b/ci/docker/pxf-cbdb-dev/common/script/entrypoint.sh
@@ -332,52 +332,6 @@ EOF
</plugins>
</profile>
</profiles>
-EOF
-
- # Configure S3 settings
- mkdir -p "$PXF_BASE/servers/s3" "$PXF_HOME/servers/s3"
-
- for s3_site in "$PXF_BASE/servers/s3/s3-site.xml"
"$PXF_BASE/servers/default/s3-site.xml" "$PXF_HOME/servers/s3/s3-site.xml"; do
- mkdir -p "$(dirname "$s3_site")"
- cat > "$s3_site" <<'EOF'
-<?xml version="1.0" encoding="UTF-8"?>
-<configuration>
- <property>
- <name>fs.s3a.endpoint</name>
- <value>http://localhost:9000</value>
- </property>
- <property>
- <name>fs.s3a.access.key</name>
- <value>admin</value>
- </property>
- <property>
- <name>fs.s3a.secret.key</name>
- <value>password</value>
- </property>
- <property>
- <name>fs.s3a.path.style.access</name>
- <value>true</value>
- </property>
- <property>
- <name>fs.s3a.connection.ssl.enabled</name>
- <value>false</value>
- </property>
- <property>
- <name>fs.s3a.impl</name>
- <value>org.apache.hadoop.fs.s3a.S3AFileSystem</value>
- </property>
- <property>
- <name>fs.s3a.aws.credentials.provider</name>
- <value>org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider</value>
- </property>
-</configuration>
-EOF
- done
- mkdir -p /home/gpadmin/.aws/
- cat > "/home/gpadmin/.aws/credentials" <<'EOF'
-[default]
-aws_access_key_id = admin
-aws_secret_access_key = password
EOF
}
@@ -556,11 +510,6 @@ start_hive_services() {
done
}
-deploy_minio() {
- log "deploying MinIO"
- bash "${COMMON_SCRIPTS}/start_minio.bash"
-}
-
main() {
detect_java_paths
setup_locale_and_packages
@@ -570,7 +519,6 @@ main() {
build_pxf
configure_pxf
prepare_hadoop_stack
- deploy_minio
health_check
log "entrypoint finished; environment ready for tests"
}
diff --git a/ci/docker/pxf-cbdb-dev/common/script/pxf-env.sh
b/ci/docker/pxf-cbdb-dev/common/script/pxf-env.sh
index e25f7bb2..1551a7b5 100755
--- a/ci/docker/pxf-cbdb-dev/common/script/pxf-env.sh
+++ b/ci/docker/pxf-cbdb-dev/common/script/pxf-env.sh
@@ -61,15 +61,6 @@ export
COORDINATOR_DATA_DIRECTORY=${COORDINATOR_DATA_DIRECTORY:-/home/gpadmin/wo
# set cloudberry timezone utc
export PGTZ=UTC
-# --------------------------------------------------------------------
-# Minio defaults
-# --------------------------------------------------------------------
-export AWS_ACCESS_KEY_ID=admin
-export AWS_SECRET_ACCESS_KEY=password
-export PROTOCOL=minio
-export ACCESS_KEY_ID=admin
-export SECRET_ACCESS_KEY=password
-
# --------------------------------------------------------------------
# PXF defaults
# --------------------------------------------------------------------
diff --git a/ci/docker/pxf-cbdb-dev/common/script/run_tests.sh
b/ci/docker/pxf-cbdb-dev/common/script/run_tests.sh
index 40bd2bd0..5e78d40e 100755
--- a/ci/docker/pxf-cbdb-dev/common/script/run_tests.sh
+++ b/ci/docker/pxf-cbdb-dev/common/script/run_tests.sh
@@ -166,15 +166,6 @@ ensure_hive_ready() {
return 1
}
-ensure_minio_bucket() {
- local mc_bin="/home/gpadmin/workspace/mc"
- if [ -x "${mc_bin}" ]; then
- ${mc_bin} alias set local http://localhost:9000 admin password >/dev/null
2>&1 || true
- ${mc_bin} mb local/gpdb-ud-scratch --ignore-existing >/dev/null 2>&1 ||
true
- ${mc_bin} policy set download local/gpdb-ud-scratch >/dev/null 2>&1 || true
- fi
-}
-
set_xml_property() {
local file="$1" name="$2" value="$3"
if [ ! -f "${file}" ]; then
@@ -201,196 +192,6 @@ ensure_yarn_vmem_settings() {
set_xml_property "${yarn_site}" "yarn.nodemanager.vmem-pmem-ratio" "4.0"
}
-ensure_hadoop_s3a_config() {
- local core_site="${HADOOP_CONF_DIR}/core-site.xml"
- if [ -f "${core_site}" ] && ! grep -q "fs.s3a.endpoint" "${core_site}"; then
- perl -0777 -pe '
-s#</configuration># <property>
- <name>fs.s3a.endpoint</name>
- <value>http://localhost:9000</value>
- </property>
- <property>
- <name>fs.s3a.path.style.access</name>
- <value>true</value>
- </property>
- <property>
- <name>fs.s3a.connection.ssl.enabled</name>
- <value>false</value>
- </property>
- <property>
- <name>fs.s3a.access.key</name>
- <value>'"${AWS_ACCESS_KEY_ID}"'</value>
- </property>
- <property>
- <name>fs.s3a.secret.key</name>
- <value>'"${AWS_SECRET_ACCESS_KEY}"'</value>
- </property>
- <property>
- <name>fs.s3a.aws.credentials.provider</name>
- <value>org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider</value>
- </property>
-</configuration>#' -i "${core_site}"
- fi
-}
-
-# Configure dedicated PXF server "s3" pointing to local MinIO;
-# used by tests that explicitly set server=s3
-configure_pxf_s3_server() {
- local server_dir="${PXF_BASE}/servers/s3"
- mkdir -p "${server_dir}"
- cat > "${server_dir}/s3-site.xml" <<EOF
-<?xml version="1.0" encoding="UTF-8"?>
-<configuration>
- <property>
- <name>fs.s3a.endpoint</name>
- <value>http://localhost:9000</value>
- </property>
- <property>
- <name>fs.s3a.path.style.access</name>
- <value>true</value>
- </property>
- <property>
- <name>fs.s3a.connection.ssl.enabled</name>
- <value>false</value>
- </property>
- <property>
- <name>fs.s3a.impl</name>
- <value>org.apache.hadoop.fs.s3a.S3AFileSystem</value>
- </property>
- <property>
- <name>fs.s3a.aws.credentials.provider</name>
- <value>org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider</value>
- </property>
- <property>
- <name>fs.s3a.access.key</name>
- <value>${AWS_ACCESS_KEY_ID}</value>
- </property>
- <property>
- <name>fs.s3a.secret.key</name>
- <value>${AWS_SECRET_ACCESS_KEY}</value>
- </property>
-</configuration>
-EOF
- cat > "${server_dir}/core-site.xml" <<EOF
-<?xml version="1.0" encoding="UTF-8"?>
-<configuration>
- <property>
- <name>fs.defaultFS</name>
- <value>s3a://</value>
- </property>
- <property>
- <name>fs.s3a.path.style.access</name>
- <value>true</value>
- </property>
- <property>
- <name>fs.s3a.connection.ssl.enabled</name>
- <value>false</value>
- </property>
- <property>
- <name>fs.s3a.endpoint</name>
- <value>http://localhost:9000</value>
- </property>
- <property>
- <name>fs.s3a.impl</name>
- <value>org.apache.hadoop.fs.s3a.S3AFileSystem</value>
- </property>
- <property>
- <name>fs.s3a.aws.credentials.provider</name>
- <value>org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider</value>
- </property>
- <property>
- <name>fs.s3a.access.key</name>
- <value>${AWS_ACCESS_KEY_ID}</value>
- </property>
- <property>
- <name>fs.s3a.secret.key</name>
- <value>${AWS_SECRET_ACCESS_KEY}</value>
- </property>
-</configuration>
-EOF
-}
-
-# Configure default PXF server to point to local MinIO with explicit creds;
-# used by tests that do NOT pass a server=name parameter (default server path)
-configure_pxf_default_s3_server() {
- export AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID:-admin}
- export AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY:-password}
- local default_s3_site="${PXF_BASE}/servers/default/s3-site.xml"
- if [ -f "${default_s3_site}" ]; then
- cat > "${default_s3_site}" <<EOF
-<?xml version="1.0" encoding="UTF-8"?>
-<configuration>
- <property>
- <name>fs.s3a.endpoint</name>
- <value>http://localhost:9000</value>
- </property>
- <property>
- <name>fs.s3a.path.style.access</name>
- <value>true</value>
- </property>
- <property>
- <name>fs.s3a.connection.ssl.enabled</name>
- <value>false</value>
- </property>
- <property>
- <name>fs.s3a.impl</name>
- <value>org.apache.hadoop.fs.s3a.S3AFileSystem</value>
- </property>
- <property>
- <name>fs.s3a.access.key</name>
- <value>${AWS_ACCESS_KEY_ID}</value>
- </property>
- <property>
- <name>fs.s3a.secret.key</name>
- <value>${AWS_SECRET_ACCESS_KEY}</value>
- </property>
-</configuration>
-EOF
- cat > "${PXF_BASE}/servers/default/core-site.xml" <<EOF
-<?xml version="1.0" encoding="UTF-8"?>
-<configuration>
- <property>
- <name>fs.defaultFS</name>
- <value>s3a://</value>
- </property>
- <property>
- <name>fs.s3a.path.style.access</name>
- <value>true</value>
- </property>
- <property>
- <name>fs.s3a.connection.ssl.enabled</name>
- <value>false</value>
- </property>
- <property>
- <name>fs.s3a.endpoint</name>
- <value>http://localhost:9000</value>
- </property>
- <property>
- <name>fs.s3a.impl</name>
- <value>org.apache.hadoop.fs.s3a.S3AFileSystem</value>
- </property>
- <property>
- <name>fs.s3a.aws.credentials.provider</name>
- <value>org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider</value>
- </property>
- <property>
- <name>fs.s3a.access.key</name>
- <value>${AWS_ACCESS_KEY_ID}</value>
- </property>
- <property>
- <name>fs.s3a.secret.key</name>
- <value>${AWS_SECRET_ACCESS_KEY}</value>
- </property>
-</configuration>
-EOF
- # hide HDFS/Hive configs so default server is treated as S3-only
- for f in hdfs-site.xml mapred-site.xml yarn-site.xml hive-site.xml
hbase-site.xml; do
- [ -f "${PXF_BASE}/servers/default/${f}" ] && rm -f
"${PXF_BASE}/servers/default/${f}"
- done
- "${PXF_HOME}/bin/pxf" restart >/dev/null
- fi
-}
-
# Ensure proxy tests can login as testuser from localhost.
ensure_testuser_pg_hba() {
local
pg_hba="/home/gpadmin/workspace/cloudberry/gpAux/gpdemo/datadirs/qddir/demoDataDir-1/pg_hba.conf"
@@ -429,7 +230,6 @@ ensure_testuser_pg_hba() {
}
base_test(){
- # keep PROTOCOL empty so tests use HDFS; we'll set minio only for s3 later
export PROTOCOL=
# ensure gpdb connections target localhost over IPv4 for proxy tests
export PGHOST=127.0.0.1
@@ -484,16 +284,6 @@ base_test(){
make GROUP="unused" || true
save_test_reports "unused"
echo "[run_tests] GROUP=unused finished"
-
- ensure_minio_bucket
- ensure_hadoop_s3a_config
- configure_pxf_s3_server
- configure_pxf_default_s3_server
- export PROTOCOL=s3
- export HADOOP_OPTIONAL_TOOLS=hadoop-aws
- make GROUP="s3" || true
- save_test_reports "s3"
- echo "[run_tests] GROUP=s3 finished"
}
# Restore default PXF server to local HDFS/Hive/HBase configuration
@@ -571,14 +361,7 @@ feature_test(){
cleanup_hive_state
cleanup_hbase_state
- # Prepare MinIO/S3 and restore default server to local HDFS/Hive/HBase
- ensure_minio_bucket
- ensure_hadoop_s3a_config
- configure_pxf_s3_server
configure_pxf_default_hdfs_server
- # Only set default server to MinIO when explicitly running S3 groups; keeping
- # it HDFS-backed avoids hijacking Hive/HDFS tests with fs.defaultFS=s3a://
- #configure_pxf_default_s3_server
export PROTOCOL=
make GROUP="features" || true
@@ -758,7 +541,7 @@ generate_test_summary() {
local group=$(basename "$group_dir")
# Skip if it's not a test group directory
- [[ "$group" =~
^(smoke|hcatalog|hcfs|hdfs|hive|gpdb|sanity|hbase|profile|jdbc|proxy|unused|s3|features|load|performance|pxfExtensionVersion2|pxfExtensionVersion2_1|pxfFdwExtensionVersion1|pxfFdwExtensionVersion2|fdw|gpdb_fdw)$
]] || continue
+ [[ "$group" =~
^(smoke|hcatalog|hcfs|hdfs|hive|gpdb|sanity|hbase|profile|jdbc|proxy|unused|features|load|performance|pxfExtensionVersion2|pxfExtensionVersion2_1|pxfFdwExtensionVersion1|pxfFdwExtensionVersion2|fdw|gpdb_fdw)$
]] || continue
echo "Processing $group test reports from $group_dir"
@@ -916,16 +699,6 @@ run_single_group() {
make GROUP="hbase"
save_test_reports "hbase"
;;
- s3)
- ensure_minio_bucket
- ensure_hadoop_s3a_config
- configure_pxf_s3_server
- configure_pxf_default_s3_server
- export PROTOCOL=s3
- export HADOOP_OPTIONAL_TOOLS=hadoop-aws
- make GROUP="s3"
- save_test_reports "s3"
- ;;
features)
feature_test
;;
@@ -956,7 +729,7 @@ run_single_group() {
;;
*)
echo "Unknown test group: $group"
- echo "Available groups: cli, external-table, fdw, server, sanity, smoke,
hdfs, hcatalog, hcfs, hive, hbase, profile, jdbc, proxy, unused, s3, features,
gpdb, gpdb_fdw, load, performance, bench, pxf_extension"
+ echo "Available groups: cli, external-table, fdw, server, sanity, smoke,
hdfs, hcatalog, hcfs, hive, hbase, profile, jdbc, proxy, unused, features,
gpdb, gpdb_fdw, load, performance, bench, pxf_extension"
exit 1
;;
esac
diff --git a/ci/docker/pxf-cbdb-dev/common/script/start_minio.bash
b/ci/docker/pxf-cbdb-dev/common/script/start_minio.bash
deleted file mode 100755
index 896c6d7f..00000000
--- a/ci/docker/pxf-cbdb-dev/common/script/start_minio.bash
+++ /dev/null
@@ -1,59 +0,0 @@
-#!/bin/bash
-# --------------------------------------------------------------------
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed
-# with this work for additional information regarding copyright
-# ownership. The ASF licenses this file to You under the Apache
-# License, Version 2.0 (the "License"); you may not use this file
-# except in compliance with the License. You may obtain a copy of the
-# License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-# implied. See the License for the specific language governing
-# permissions and limitations under the License.
-#
-# --------------------------------------------------------------------
-
-set -e
-
-WORKSPACE_DIR=${WORKSPACE_DIR:-/home/gpadmin/workspace}
-MINIO_BIN=${WORKSPACE_DIR}/minio
-MC_BIN=${WORKSPACE_DIR}/mc
-MINIO_DATA_DIR=${MINIO_DATA_DIR:-${WORKSPACE_DIR}/minio-data}
-MINIO_PORT=${MINIO_PORT:-9000}
-MINIO_CONSOLE_PORT=${MINIO_CONSOLE_PORT:-9001}
-
-export MINIO_ROOT_USER=${MINIO_ROOT_USER:-admin}
-export MINIO_ROOT_PASSWORD=${MINIO_ROOT_PASSWORD:-password}
-export MINIO_API_SELECT_PARQUET=${MINIO_API_SELECT_PARQUET:-on}
-
-echo "MinIO credentials: rootUser=${MINIO_ROOT_USER}
rootPassword=${MINIO_ROOT_PASSWORD}"
-
-mkdir -p ${MINIO_DATA_DIR}
-
-echo "Starting MinIO server on port ${MINIO_PORT}..."
-${MINIO_BIN} server ${MINIO_DATA_DIR} \
- --address ":${MINIO_PORT}" \
- --console-address ":${MINIO_CONSOLE_PORT}" &
-
-MINIO_PID=$!
-echo "MinIO started with PID: ${MINIO_PID}"
-
-sleep 3
-
-echo "Creating test bucket 'gpdb-ud-scratch'..."
-${MC_BIN} alias set local http://localhost:${MINIO_PORT} ${MINIO_ROOT_USER}
${MINIO_ROOT_PASSWORD}
-${MC_BIN} mb local/gpdb-ud-scratch --ignore-existing
-
-export PROTOCOL=minio
-export ACCESS_KEY_ID=${MINIO_ROOT_USER}
-export SECRET_ACCESS_KEY=${MINIO_ROOT_PASSWORD}
-
-echo "MinIO is ready!"
-echo " Console: http://localhost:${MINIO_CONSOLE_PORT}";
-echo " API: http://localhost:${MINIO_PORT}";
diff --git a/ci/docker/pxf-cbdb-dev/ubuntu/script/entrypoint_kerberos.sh
b/ci/docker/pxf-cbdb-dev/ubuntu/script/entrypoint_kerberos.sh
index f15ee35c..defc3328 100755
--- a/ci/docker/pxf-cbdb-dev/ubuntu/script/entrypoint_kerberos.sh
+++ b/ci/docker/pxf-cbdb-dev/ubuntu/script/entrypoint_kerberos.sh
@@ -310,65 +310,6 @@ setup_ssl_material() {
sudo chown gpadmin:gpadmin "${SSL_KEYSTORE}" "${SSL_TRUSTSTORE}"
}
-deploy_minio() {
- log "deploying MinIO (for S3 tests)"
- bash "${PXF_SCRIPTS}/start_minio.bash"
-}
-
-configure_pxf_s3() {
- log "configuring S3 server definitions for PXF"
- local servers_base=${PXF_BASE:-/home/gpadmin/pxf-base}
- local pxf_conf=/usr/local/pxf/conf
- local s3_sites=(
- "${servers_base}/servers/s3/s3-site.xml"
- "${servers_base}/servers/default/s3-site.xml"
- "${pxf_conf}/servers/s3/s3-site.xml"
- )
- for s3_site in "${s3_sites[@]}"; do
- mkdir -p "$(dirname "${s3_site}")"
- cat > "${s3_site}" <<'EOF'
-<?xml version="1.0" encoding="UTF-8"?>
-<configuration>
- <property>
- <name>fs.s3a.endpoint</name>
- <value>http://localhost:9000</value>
- </property>
- <property>
- <name>fs.s3a.access.key</name>
- <value>admin</value>
- </property>
- <property>
- <name>fs.s3a.secret.key</name>
- <value>password</value>
- </property>
- <property>
- <name>fs.s3a.path.style.access</name>
- <value>true</value>
- </property>
- <property>
- <name>fs.s3a.connection.ssl.enabled</name>
- <value>false</value>
- </property>
- <property>
- <name>fs.s3a.impl</name>
- <value>org.apache.hadoop.fs.s3a.S3AFileSystem</value>
- </property>
- <property>
- <name>fs.s3a.aws.credentials.provider</name>
- <value>org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider</value>
- </property>
-</configuration>
-EOF
- done
-
- mkdir -p /home/gpadmin/.aws/
- cat > "/home/gpadmin/.aws/credentials" <<'EOF'
-[default]
-aws_access_key_id = admin
-aws_secret_access_key = password
-EOF
-}
-
prepare_sut() {
# Generate SUT pointing to container FQDN and overwrite build outputs to
avoid localhost.
local host_fqdn_local=$1
@@ -456,7 +397,7 @@ prepare_hadoop_conf_for_tests() {
local
conf_dir=${HADOOP_CONF_DIR:-/home/gpadmin/workspace/singlecluster/hadoop/etc/hadoop}
local target_base=${REPO_ROOT}/automation/target
mkdir -p "${target_base}/test-classes" "${target_base}/classes"
- for f in core-site.xml hdfs-site.xml mapred-site.xml yarn-site.xml
ssl-client.xml ssl-server.xml s3-site.xml; do
+ for f in core-site.xml hdfs-site.xml mapred-site.xml yarn-site.xml
ssl-client.xml ssl-server.xml; do
if [ -f "${conf_dir}/${f}" ]; then
cp "${conf_dir}/${f}" "${target_base}/test-classes/${f}"
cp "${conf_dir}/${f}" "${target_base}/classes/${f}"
@@ -468,12 +409,6 @@ prepare_hadoop_conf_for_tests() {
cp "${hbase_site}" "${target_base}/test-classes/hbase-site.xml"
cp "${hbase_site}" "${target_base}/classes/hbase-site.xml"
fi
- # Add PXF server S3 configs to the classpath for automation tests.
- local pxf_s3="${PXF_BASE:-/home/gpadmin/pxf-base}/servers/s3/s3-site.xml"
- if [ -f "${pxf_s3}" ]; then
- cp "${pxf_s3}" "${target_base}/test-classes/s3-site.xml"
- cp "${pxf_s3}" "${target_base}/classes/s3-site.xml"
- fi
}
configure_hadoop() {
@@ -507,12 +442,6 @@ configure_hadoop() {
<property><name>hadoop.proxyuser.gpadmin.groups</name><value>*</value></property>
<property><name>hadoop.proxyuser.porter.hosts</name><value>*</value></property>
<property><name>hadoop.proxyuser.porter.groups</name><value>*</value></property>
-
<property><name>fs.s3a.endpoint</name><value>http://localhost:9000</value></property>
- <property><name>fs.s3a.path.style.access</name><value>true</value></property>
-
<property><name>fs.s3a.connection.ssl.enabled</name><value>false</value></property>
-
<property><name>fs.s3a.access.key</name><value>${AWS_ACCESS_KEY_ID:-admin}</value></property>
-
<property><name>fs.s3a.secret.key</name><value>${AWS_SECRET_ACCESS_KEY:-password}</value></property>
-
<property><name>fs.s3a.aws.credentials.provider</name><value>org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider</value></property>
</configuration>
EOF
@@ -1102,14 +1031,11 @@ init_test_env() {
export KRB5CCNAME=${KRB5CCNAME:-/tmp/krb5cc_pxf_automation}
export PXF_TEST_KEEP_DATA=${PXF_TEST_KEEP_DATA:-true}
unset HADOOP_USER_NAME
- local s3_opts="-Dfs.s3a.endpoint=http://localhost:9000
-Dfs.s3a.path.style.access=true -Dfs.s3a.connection.ssl.enabled=false
-Dfs.s3a.access.key=${AWS_ACCESS_KEY_ID:-admin}
-Dfs.s3a.secret.key=${AWS_SECRET_ACCESS_KEY:-password}"
export HDFS_URI="hdfs://${HOST_FQDN_LOCAL}:8020"
- export HADOOP_OPTS="-Dfs.defaultFS=${HDFS_URI}
-Dhadoop.security.authentication=kerberos ${s3_opts}"
+ export HADOOP_OPTS="-Dfs.defaultFS=${HDFS_URI}
-Dhadoop.security.authentication=kerberos"
export HADOOP_CLIENT_OPTS="${HADOOP_OPTS}"
- export MAVEN_OPTS="-Dfs.defaultFS=${HDFS_URI}
-Dhadoop.security.authentication=kerberos ${s3_opts} -Dpxf.host=${PXF_HOST}
-Dpxf.port=${PXF_PORT}"
+ export MAVEN_OPTS="-Dfs.defaultFS=${HDFS_URI}
-Dhadoop.security.authentication=kerberos -Dpxf.host=${PXF_HOST}
-Dpxf.port=${PXF_PORT}"
export PGOPTIONS="${PGOPTIONS:---client-min-messages=error}"
- export AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID:-admin}
- export AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY:-password}
export EXCLUDE_GROUPS_LOCAL=${EXCLUDED_GROUPS:-multiClusterSecurity}
DEFAULT_MAVEN_TEST_OPTS="-Dpxf.host=${PXF_HOST} -Dpxf.port=${PXF_PORT}
-DPXF_SINGLE_NODE=true -DexcludedGroups=${EXCLUDE_GROUPS_LOCAL}"
}
@@ -1278,8 +1204,6 @@ main() {
configure_hbase
configure_pxf
configure_pxf_servers
- configure_pxf_s3
- deploy_minio
configure_pg_hba
start_hdfs_secure
start_hive_secure
diff --git a/ci/singlecluster/Dockerfile b/ci/singlecluster/Dockerfile
index 931d5d3f..aa30c2ae 100644
--- a/ci/singlecluster/Dockerfile
+++ b/ci/singlecluster/Dockerfile
@@ -104,20 +104,6 @@ RUN set -e; \
wget -O go.tgz -q
"https://go.dev/dl/go${GO_VERSION}.linux-${go_arch}.tar.gz"; && \
sudo rm -rf /usr/local/go && sudo tar -C /usr/local -xzf go.tgz && rm
go.tgz
-# Install MinIO and mc
-RUN set -e; \
- arch="${TARGETARCH:-$(uname -m)}"; \
- case "$arch" in \
- amd64|x86_64) minio_arch="amd64" ;; \
- arm64|aarch64) minio_arch="arm64" ;; \
- *) echo "Unsupported architecture: ${arch}"; exit 1 ;; \
- esac; \
- mkdir -p /home/gpadmin/workspace && \
- wget -O /home/gpadmin/workspace/minio
"https://dl.min.io/server/minio/release/linux-${minio_arch}/minio"; && \
- wget -O /home/gpadmin/workspace/mc
"https://dl.min.io/client/mc/release/linux-${minio_arch}/mc"; && \
- chmod +x /home/gpadmin/workspace/minio /home/gpadmin/workspace/mc
-
-
COPY ./templates $GPHD_ROOT
COPY ./conf $GPHD_ROOT/conf
COPY ./bin $GPHD_ROOT/bin
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
