This is an automated email from the ASF dual-hosted git repository. yuqi4733 pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/gravitino.git
commit b1f5724b9cf0091ed8193f56a12be50ed103eb7c Author: qbhan <[email protected]> AuthorDate: Fri Jan 23 12:01:24 2026 +0800 [#9720] feat(trino-connector): Trino integration tests support specifying the version of the testing Trino environment (#9752) ### What changes were proposed in this pull request? `--trino_version` specify the trino version. `--trino_connector_dir` specify the trino connector JAR files path. ### Why are the changes needed? Fix: #9720 ### Does this PR introduce _any_ user-facing change? No ### How was this patch tested? local tests --- docs/how-to-test.md | 4 + .../docker-script/docker-compose.yaml | 8 +- .../init/trino/config/catalog/gravitino.properties | 1 + .../docker-script/init/trino/init.sh | 2 +- .../test/container/TrinoITContainers.java | 16 +++- .../connector/integration/test/TrinoQueryIT.java | 8 +- .../integration/test/TrinoQueryITBase.java | 14 +++- .../integration/test/TrinoQueryTestTool.java | 69 +++++++++++----- .../trino-test-tools/run_test_with_versions.sh | 92 ++++++++++++++++++++++ 9 files changed, 182 insertions(+), 32 deletions(-) diff --git a/docs/how-to-test.md b/docs/how-to-test.md index dfaf63fb62..fc8591889a 100644 --- a/docs/how-to-test.md +++ b/docs/how-to-test.md @@ -66,6 +66,10 @@ To deploy the Gravitino server locally to run the integration tests, follow thes 4. Use the `bash trino-connector/integration-test/trino-test-tools/trino_test.sh` command to run all the Trino test sets in the `trino-connector/integration-test/src/test/resources/trino-ci-testset/testsets` directory. Specify the `--trino_worker_num` parameter to make the Trino test sets run in a distributed environment. + Specify the `--trino_version` parameter to make the Trino test sets run with specific trino version. + Specify the `--trino_connector_dir` parameter to run Trino tests using the JAR files located in the `gravitino-trino-connector` plugin directory. + Alternatively, use `bash trino-connector/integration-test/trino-test-tools/run_test_with_versions.sh` command to execute + tests across different Trino versions. ## Skip tests diff --git a/integration-test-common/docker-script/docker-compose.yaml b/integration-test-common/docker-script/docker-compose.yaml index 1620b53a7e..7b3d46b89e 100644 --- a/integration-test-common/docker-script/docker-compose.yaml +++ b/integration-test-common/docker-script/docker-compose.yaml @@ -79,7 +79,7 @@ services: retries: 5 trino: - image: trinodb/trino:435 + image: trinodb/trino:${TRINO_VERSION:-435} networks: - trino-net container_name: trino-ci-trino @@ -94,7 +94,7 @@ services: entrypoint: /bin/bash /tmp/trino/init.sh volumes: - ./init/trino:/tmp/trino - - ../../trino-connector/trino-connector/build/libs:/usr/lib/trino/plugin/gravitino + - ${GRAVITINO_TRINO_CONNECTOR_DIR:-../../trino-connector/trino-connector/build/libs}:/usr/lib/trino/plugin/gravitino extra_hosts: - "host.docker.internal:host-gateway" healthcheck: @@ -113,7 +113,7 @@ services: condition: service_healthy trino-worker: - image: trinodb/trino:435 + image: trinodb/trino:${TRINO_VERSION:-435} networks: - trino-net deploy: @@ -128,7 +128,7 @@ services: entrypoint: /bin/bash /tmp/trino/init.sh volumes: - ./init/trino:/tmp/trino - - ../../trino-connector/trino-connector/build/libs:/usr/lib/trino/plugin/gravitino + - ${GRAVITINO_TRINO_CONNECTOR_DIR:-../../trino-connector/trino-connector/build/libs}:/usr/lib/trino/plugin/gravitino extra_hosts: - "host.docker.internal:host-gateway" depends_on: diff --git a/integration-test-common/docker-script/init/trino/config/catalog/gravitino.properties b/integration-test-common/docker-script/init/trino/config/catalog/gravitino.properties index 990ee0b147..d84f93364c 100644 --- a/integration-test-common/docker-script/init/trino/config/catalog/gravitino.properties +++ b/integration-test-common/docker-script/init/trino/config/catalog/gravitino.properties @@ -20,3 +20,4 @@ connector.name = gravitino gravitino.uri = http://GRAVITINO_HOST_IP:GRAVITINO_HOST_PORT gravitino.metalake = GRAVITINO_METALAKE_NAME +gravitino.trino.skip-version-validation=true diff --git a/integration-test-common/docker-script/init/trino/init.sh b/integration-test-common/docker-script/init/trino/init.sh index f266ae8c76..385c06e8f5 100644 --- a/integration-test-common/docker-script/init/trino/init.sh +++ b/integration-test-common/docker-script/init/trino/init.sh @@ -28,7 +28,7 @@ cp "$trino_conf_dir/config/node.properties" /etc/trino/node.properties cp "$trino_conf_dir/config/catalog/gravitino.properties" /etc/trino/catalog/gravitino.properties # Copy the MYSQL driver to iceberg connector -cp /usr/lib/trino/plugin/mysql/mysql-connector-j-8.2.0.jar /usr/lib/trino/plugin/iceberg/ +cp /usr/lib/trino/plugin/mysql/*mysql-connector-j-*.jar /usr/lib/trino/plugin/iceberg/ # Update `gravitino.uri = http://GRAVITINO_HOST_IP:GRAVITINO_HOST_PORT` in the `conf/catalog/gravitino.properties` sed -i "s/GRAVITINO_HOST_IP:GRAVITINO_HOST_PORT/${GRAVITINO_HOST_IP}:${GRAVITINO_HOST_PORT}/g" /etc/trino/catalog/gravitino.properties diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/TrinoITContainers.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/TrinoITContainers.java index cb1aed1d68..b26984c6d2 100644 --- a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/TrinoITContainers.java +++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/TrinoITContainers.java @@ -49,19 +49,29 @@ public class TrinoITContainers implements AutoCloseable { } public void launch(int gravitinoServerPort) throws Exception { - launch(gravitinoServerPort, "hive2", false, 0); + launch(gravitinoServerPort, "hive2", false, null, null, null); } public void launch( int gravitinoServerPort, String hiveRuntimeVersion, boolean isTrinoConnectorTest, - int trinoWorkerNum) + Integer trinoWorkerNum, + Integer trinoVersion, + String trinoConnectorDir) throws Exception { shutdown(); Map<String, String> env = new HashMap<>(); - env.put("TRINO_WORKER_NUM", String.valueOf(trinoWorkerNum)); + if (trinoWorkerNum != null) { + env.put("TRINO_WORKER_NUM", String.valueOf(trinoWorkerNum)); + } + if (trinoVersion != null) { + env.put("TRINO_VERSION", String.valueOf(trinoVersion)); + } + if (trinoConnectorDir != null) { + env.put("GRAVITINO_TRINO_CONNECTOR_DIR", trinoConnectorDir); + } env.put("GRAVITINO_SERVER_PORT", String.valueOf(gravitinoServerPort)); env.put("HIVE_RUNTIME_VERSION", hiveRuntimeVersion); env.put("TRINO_CONNECTOR_TEST", String.valueOf(isTrinoConnectorTest)); diff --git a/trino-connector/integration-test/src/test/java/org/apache/gravitino/trino/connector/integration/test/TrinoQueryIT.java b/trino-connector/integration-test/src/test/java/org/apache/gravitino/trino/connector/integration/test/TrinoQueryIT.java index d4e6e2dad0..811e8229f7 100644 --- a/trino-connector/integration-test/src/test/java/org/apache/gravitino/trino/connector/integration/test/TrinoQueryIT.java +++ b/trino-connector/integration-test/src/test/java/org/apache/gravitino/trino/connector/integration/test/TrinoQueryIT.java @@ -71,7 +71,11 @@ public class TrinoQueryIT extends TrinoQueryITBase { static TrinoQueryITBase trinoQueryITBase; - static int trinoWorkerNum = 0; + static Integer trinoWorkerNum; + + static Integer trinoVersion; + + static String trinoConnectorDir; static { testsetsDir = TrinoQueryIT.class.getClassLoader().getResource("trino-ci-testset").getPath(); @@ -80,7 +84,7 @@ public class TrinoQueryIT extends TrinoQueryITBase { @BeforeAll public void setup() throws Exception { - trinoQueryITBase = new TrinoQueryITBase(trinoWorkerNum); + trinoQueryITBase = new TrinoQueryITBase(trinoWorkerNum, trinoVersion, trinoConnectorDir); trinoQueryITBase.setup(); cleanupTestEnv(); diff --git a/trino-connector/integration-test/src/test/java/org/apache/gravitino/trino/connector/integration/test/TrinoQueryITBase.java b/trino-connector/integration-test/src/test/java/org/apache/gravitino/trino/connector/integration/test/TrinoQueryITBase.java index 5580ffbd36..f2197c0e57 100644 --- a/trino-connector/integration-test/src/test/java/org/apache/gravitino/trino/connector/integration/test/TrinoQueryITBase.java +++ b/trino-connector/integration-test/src/test/java/org/apache/gravitino/trino/connector/integration/test/TrinoQueryITBase.java @@ -70,12 +70,18 @@ public class TrinoQueryITBase { private static BaseIT baseIT; - protected int trinoWorkerNum = 0; + protected Integer trinoWorkerNum; + + protected Integer trinoVersion; + + protected String trinoConnectorDir; public TrinoQueryITBase() {} - public TrinoQueryITBase(int trinoWorkerNum) { + public TrinoQueryITBase(Integer trinoWorkerNum, Integer trinoVersion, String trinoConnectorDir) { this.trinoWorkerNum = trinoWorkerNum; + this.trinoVersion = trinoVersion; + this.trinoConnectorDir = trinoConnectorDir; } private void setEnv() throws Exception { @@ -104,7 +110,9 @@ public class TrinoQueryITBase { baseIT.getGravitinoServerPort(), hiveRuntimeVersion, isTrinoConnectorTest, - trinoWorkerNum); + trinoWorkerNum, + trinoVersion, + trinoConnectorDir); trinoUri = trinoITContainers.getTrinoUri(); hiveMetastoreUri = trinoITContainers.getHiveMetastoreUri(); diff --git a/trino-connector/integration-test/src/test/java/org/apache/gravitino/trino/connector/integration/test/TrinoQueryTestTool.java b/trino-connector/integration-test/src/test/java/org/apache/gravitino/trino/connector/integration/test/TrinoQueryTestTool.java index 872b3e1373..13a4bf8ed3 100644 --- a/trino-connector/integration-test/src/test/java/org/apache/gravitino/trino/connector/integration/test/TrinoQueryTestTool.java +++ b/trino-connector/integration-test/src/test/java/org/apache/gravitino/trino/connector/integration/test/TrinoQueryTestTool.java @@ -32,7 +32,7 @@ public class TrinoQueryTestTool { public static void main(String[] args) throws Exception { Options options = new Options(); - + boolean isFailed = false; try { options.addOption( "auto", @@ -92,6 +92,16 @@ public class TrinoQueryTestTool { + "Use a distributed cluster for integration testing when necessary (value > 0), " + "otherwise fall back to a single-node setup with combined coordinator-worker roles."); + options.addOption( + "trino_version", true, "Specify the Trino version to test, the default value is 435."); + + options.addOption( + "trino_connector_dir", + true, + "Specify the Gravitino connector JAR path. " + + "The JAR file under ${trino_connector_dir} will be copied into the test image, " + + "the default value is ${project_root}/trino-connector/trino-connector/build/libs."); + options.addOption("help", false, "Print this help message"); CommandLineParser parser = new PosixParser(); @@ -106,6 +116,10 @@ public class TrinoQueryTestTool { + "TrinoTestTool --auto=all\n\n" + "Run all the testers in the 'testsets' directory with a distributed cluster:\n" + "TrinoTestTool --auto=all --trino_worker_num=3\n\n" + + "Run all the testers in the 'testsets' directory with specific trino version:\n" + + "TrinoTestTool --auto=all --trino_version=435\n\n" + + "Run all the testers in the 'testsets' directory with specific trino connector JAR files:\n" + + "TrinoTestTool --auto=all --trino_connector_dir=./trino-connector/libs\n\n" + "Run all the tpch testset's testers in the 'testsets/tpch' directory:\n" + "TrinoTestTool --testset=tpch --auto=all\n\n" + "Run the tester 'testsets/tpch/00005.sql' in the tpch testset under hive catalog :\n" @@ -221,22 +235,9 @@ public class TrinoQueryTestTool { } } - String trinoWorkerNumConfig = commandLine.getOptionValue("trino_worker_num", "0"); - try { - int trinoWorkerNum = Integer.parseInt(trinoWorkerNumConfig); - if (trinoWorkerNum < 0) { - System.out.println( - "The value of trino_worker_num must be greater than zero, current value is: " - + trinoWorkerNumConfig); - System.exit(1); - } - TrinoQueryIT.trinoWorkerNum = trinoWorkerNum; - } catch (Exception e) { - System.out.println( - "The value of trino_worker_num must be an integer value, current value is: " - + trinoWorkerNumConfig); - System.exit(1); - } + TrinoQueryIT.trinoWorkerNum = extractIntValue(commandLine, "trino_worker_num"); + TrinoQueryIT.trinoVersion = extractIntValue(commandLine, "trino_version"); + TrinoQueryIT.trinoConnectorDir = commandLine.getOptionValue("trino_connector_dir"); checkEnv(); @@ -263,10 +264,40 @@ public class TrinoQueryTestTool { testerRunner.totalCount, testerRunner.passCount, testerRunner.generateTestStatus()); } catch (Exception e) { System.out.println(e.getMessage()); - System.exit(-1); + isFailed = true; } finally { - TrinoQueryIT.cleanup(); + try { + TrinoQueryIT.cleanup(); + } catch (Exception ex) { + System.out.println("Cleanup failed during error handling: " + ex.getMessage()); + } + if (isFailed) { + System.exit(-1); + } + } + } + + private static Integer extractIntValue(CommandLine commandLine, String key) { + String valueConfig = commandLine.getOptionValue(key); + if (valueConfig != null) { + try { + int value = Integer.parseInt(valueConfig); + if (value < 0) { + System.out.println( + "The value of " + + key + + " must be greater than zero, current value is: " + + valueConfig); + System.exit(1); + } + return value; + } catch (Exception e) { + System.out.println( + "The value of " + key + " must be an integer value, current value is: " + valueConfig); + System.exit(1); + } } + return null; } private static void checkEnv() { diff --git a/trino-connector/integration-test/trino-test-tools/run_test_with_versions.sh b/trino-connector/integration-test/trino-test-tools/run_test_with_versions.sh new file mode 100755 index 0000000000..7061d0ad70 --- /dev/null +++ b/trino-connector/integration-test/trino-test-tools/run_test_with_versions.sh @@ -0,0 +1,92 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# This script is used to execute tests for the Trino connector across different Trino versions. +# It wraps the `trino_test.sh` script and iterates through a configured map of Trino versions +# to connector modules. +# +# Arguments: +# --trino_versions_map: Space-separated list of "version:module" pairs. +# Defaults to "435:trino-connector" if not provided. +# --trino_test_args: Arguments passed directly to the underlying test script. +# +# Example: +# ./run_test_with_versions.sh \ +# --trino_versions_map="435:trino-connector 478:trino-connector-470-478" \ +# --trino_test_args="--auto=all" +# +# This configuration will run tests for: +# - Trino 435 using the connector in 'trino-connector' directory +# - Trino 478 using the connector in 'trino-connector-470-478' directory +# And pass "--auto=all" to the underlying test script for each version. + +cd "$(dirname "$0")" + +# Set the Gravitino server directories +GRAVITINO_HOME_DIR=../../../ +GRAVITINO_HOME_DIR=`realpath $GRAVITINO_HOME_DIR` + +# Parse arguments +trino_versions_map="" +trino_test_args="" + +while [[ $# -gt 0 ]]; do + case $1 in + --trino_versions_map=*) + trino_versions_map="${1#*=}" + shift + ;; + --trino_test_args=*) + trino_test_args="${1#*=}" + shift + ;; + *) + echo "Unknown argument: $1" + exit 1 + ;; + esac +done + +if [ -z "$trino_versions_map" ]; then + trino_versions_map="435:trino-connector" +fi + +for entry in $trino_versions_map; do + trino_version=${entry%%:*} + trino_module_name=${entry##*:} + + trino_connector_dir="$GRAVITINO_HOME_DIR/trino-connector/$trino_module_name/build/libs" + + # Combine test arguments with version-specific arguments + args="$trino_test_args --trino_version=${trino_version} --trino_connector_dir=${trino_connector_dir}" + + # execute test + echo "Running test for Trino version: $trino_version with connector: $trino_module_name" + echo "The args: $args" + + sleep 5 + $GRAVITINO_HOME_DIR/trino-connector/integration-test/trino-test-tools/trino_test.sh $args + + if [ $? -ne 0 ]; then + echo "Test failed for Trino version $trino_version" + exit 1 + fi +done + +echo "Test success"
