This is an automated email from the ASF dual-hosted git repository.
yuqi4733 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gravitino.git
The following commit(s) were added to refs/heads/main by this push:
new aefa319c6 [#4238] fix (trino-connector) Make the Trino connector test
more stable and improve the logs (#4213)
aefa319c6 is described below
commit aefa319c64eb0f337e22c8b7c53fc74673b82fba
Author: Yuhui <[email protected]>
AuthorDate: Mon Jul 29 12:00:32 2024 +0800
[#4238] fix (trino-connector) Make the Trino connector test more stable and
improve the logs (#4213)
### What changes were proposed in this pull request?
1. Add more output log for run the Trino Integration test.
2. Enable debug for TrinoQuery runner
3. Increase timeout for wait catalog to create.
### Why are the changes needed?
Fix: #4238
### Does this PR introduce _any_ user-facing change?
NO
### How was this patch tested?
NO
---
.../integration/test/trino/TrinoQueryIT.java | 11 ++++--
.../integration/test/trino/TrinoQueryITBase.java | 2 +-
.../integration/test/trino/TrinoQueryRunner.java | 2 +-
.../integration/test/trino/TrinoQueryTestTool.java | 2 +-
.../init/trino/config/jvm.config} | 32 ++++++++--------
.../trino-it/init/trino/config/log4j2.properties | 44 ++++++++++++++++++++++
integration-test/trino-it/init/trino/init.sh | 3 ++
integration-test/trino-it/launch.sh | 12 +++---
integration-test/trino-test-tools/trino_test.sh | 2 +-
.../connector/catalog/CatalogConnectorManager.java | 2 +-
.../AlterCatalogStoredProcedure.java | 3 +-
.../CreateCatalogStoredProcedure.java | 3 +-
.../storedprocdure/DropCatalogStoredProcedure.java | 3 +-
13 files changed, 88 insertions(+), 33 deletions(-)
diff --git
a/integration-test/src/test/java/org/apache/gravitino/integration/test/trino/TrinoQueryIT.java
b/integration-test/src/test/java/org/apache/gravitino/integration/test/trino/TrinoQueryIT.java
index b7575f94f..87a260488 100644
---
a/integration-test/src/test/java/org/apache/gravitino/integration/test/trino/TrinoQueryIT.java
+++
b/integration-test/src/test/java/org/apache/gravitino/integration/test/trino/TrinoQueryIT.java
@@ -105,9 +105,14 @@ public class TrinoQueryIT extends TrinoQueryITBase {
.pollInterval(1, TimeUnit.SECONDS)
.until(
() -> {
- String[] catalogs = trinoQueryRunner.runQuery("show
catalogs").split("\n");
- LOG.info("Catalogs: {}", Arrays.toString(catalogs));
- return Arrays.stream(catalogs).noneMatch(s ->
s.startsWith("\"gt_"));
+ try {
+ String[] catalogs = trinoQueryRunner.runQuery("show
catalogs").split("\n");
+ LOG.info("Catalogs: {}", Arrays.toString(catalogs));
+ return Arrays.stream(catalogs).noneMatch(s ->
s.startsWith("\"gt_"));
+ } catch (Exception e) {
+ LOG.error("Failed to run query in trino", e);
+ return false;
+ }
});
} catch (Exception e) {
throw new Exception("Failed to clean up test env: " + e.getMessage(), e);
diff --git
a/integration-test/src/test/java/org/apache/gravitino/integration/test/trino/TrinoQueryITBase.java
b/integration-test/src/test/java/org/apache/gravitino/integration/test/trino/TrinoQueryITBase.java
index 8d41020f3..5dce53e35 100644
---
a/integration-test/src/test/java/org/apache/gravitino/integration/test/trino/TrinoQueryITBase.java
+++
b/integration-test/src/test/java/org/apache/gravitino/integration/test/trino/TrinoQueryITBase.java
@@ -245,6 +245,6 @@ public class TrinoQueryITBase {
if (dir.exists()) {
return dir.list();
}
- throw new Exception("Test queries directory does not exist");
+ throw new Exception("Test queries directory " + dirname + " does not
exist");
}
}
diff --git
a/integration-test/src/test/java/org/apache/gravitino/integration/test/trino/TrinoQueryRunner.java
b/integration-test/src/test/java/org/apache/gravitino/integration/test/trino/TrinoQueryRunner.java
index 6129ad6c1..7b0fcaee3 100644
---
a/integration-test/src/test/java/org/apache/gravitino/integration/test/trino/TrinoQueryRunner.java
+++
b/integration-test/src/test/java/org/apache/gravitino/integration/test/trino/TrinoQueryRunner.java
@@ -61,7 +61,7 @@ class TrinoQueryRunner {
.timeZone(ZoneId.systemDefault())
.clientRequestTimeout(new Duration(30, TimeUnit.SECONDS))
.build();
- return new QueryRunner(trinoUri, session, false,
HttpLoggingInterceptor.Level.NONE);
+ return new QueryRunner(trinoUri, session, true,
HttpLoggingInterceptor.Level.NONE);
}
String runQuery(String query) {
diff --git
a/integration-test/src/test/java/org/apache/gravitino/integration/test/trino/TrinoQueryTestTool.java
b/integration-test/src/test/java/org/apache/gravitino/integration/test/trino/TrinoQueryTestTool.java
index 236e994b0..d30643ab8 100644
---
a/integration-test/src/test/java/org/apache/gravitino/integration/test/trino/TrinoQueryTestTool.java
+++
b/integration-test/src/test/java/org/apache/gravitino/integration/test/trino/TrinoQueryTestTool.java
@@ -220,7 +220,7 @@ public class TrinoQueryTestTool {
testerRunner.totalCount, testerRunner.passCount,
testerRunner.generateTestStatus());
} catch (Exception e) {
System.out.println(e.getMessage());
- e.printStackTrace();
+ System.exit(-1);
} finally {
TrinoQueryIT.cleanup();
}
diff --git a/integration-test/trino-test-tools/trino_test.sh
b/integration-test/trino-it/init/trino/config/jvm.config
old mode 100755
new mode 100644
similarity index 55%
copy from integration-test/trino-test-tools/trino_test.sh
copy to integration-test/trino-it/init/trino/config/jvm.config
index 7824cb5b9..57b0e104d
--- a/integration-test/trino-test-tools/trino_test.sh
+++ b/integration-test/trino-it/init/trino/config/jvm.config
@@ -1,4 +1,3 @@
-#!/bin/bash
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
@@ -16,18 +15,21 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-
#
-
-DIR=$(cd "$(dirname "$0")" && pwd)/../..
-export GRAVITINO_ROOT_DIR=$(cd "$DIR" && pwd)
-export GRAVITINO_HOME=$GRAVITINO_ROOT_DIR
-export GRAVITINO_TEST=true
-export HADOOP_USER_NAME=root
-
-echo $GRAVITINO_ROOT_DIR
-cd $GRAVITINO_ROOT_DIR
-
-args="\"$@\""
-
-./gradlew :integration-test:TrinoTest -PappArgs="$args"
+-XX:InitialRAMPercentage=30
+-XX:MaxRAMPercentage=60
+-XX:G1HeapRegionSize=32M
+-XX:+ExplicitGCInvokesConcurrent
+-XX:+HeapDumpOnOutOfMemoryError
+-XX:+ExitOnOutOfMemoryError
+-XX:-OmitStackTraceInFastThrow
+-XX:ReservedCodeCacheSize=256M
+-XX:PerMethodRecompilationCutoff=10000
+-XX:PerBytecodeRecompilationCutoff=10000
+-Djdk.attach.allowAttachSelf=true
+-Djdk.nio.maxCachedBufferSize=2000000
+# Reduce starvation of threads by GClocker, recommend to set about the number
of cpu cores (JDK-8192647)
+-XX:+UnlockDiagnosticVMOptions
+-XX:GCLockerRetryAllocationCount=32
+-DHADOOP_USER_NAME=hive
+-Dlog4j.configurationFile=/etc/trino/log4j2.properties
\ No newline at end of file
diff --git a/integration-test/trino-it/init/trino/config/log4j2.properties
b/integration-test/trino-it/init/trino/config/log4j2.properties
new file mode 100644
index 000000000..c6c18dde3
--- /dev/null
+++ b/integration-test/trino-it/init/trino/config/log4j2.properties
@@ -0,0 +1,44 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+# Set to debug or trace if log4j initialization is failing
+status = info
+
+# Name of the configuration
+name = ConsoleLogConfig
+
+# Console appender configuration
+appender.console.type = Console
+appender.console.name = consoleLogger
+appender.console.layout.type = PatternLayout
+appender.console.layout.pattern = %d{yyyy-MM-dd HH:mm:ss} %-5p [%t] %c{1}:%L -
%m%n
+
+# File appender configuration
+appender.file.type = File
+appender.file.name = fileLogger
+appender.file.fileName = gravitino-trino-connector.log
+appender.file.layout.type = PatternLayout
+appender.file.layout.pattern = %d{yyyy-MM-dd HH:mm:ss} %-5p [%t] %c{1}:%L -
%m%n
+
+# Root logger level
+rootLogger.level = info
+
+# Root logger referring to console and file appenders
+rootLogger.appenderRef.stdout.ref = consoleLogger
+rootLogger.appenderRef.file.ref = fileLogger
diff --git a/integration-test/trino-it/init/trino/init.sh
b/integration-test/trino-it/init/trino/init.sh
index 7e7f6eb1f..b997090ef 100644
--- a/integration-test/trino-it/init/trino/init.sh
+++ b/integration-test/trino-it/init/trino/init.sh
@@ -23,7 +23,10 @@ trino_conf_dir="$(dirname "${BASH_SOURCE-$0}")"
trino_conf_dir="$(cd "${trino_conf_dir}">/dev/null; pwd)"
cp "$trino_conf_dir/config/config.properties" /etc/trino/config.properties
+cp "$trino_conf_dir/config/jvm.config" /etc/trino/jvm.config
+cp "$trino_conf_dir/config/log4j2.properties" /etc/trino/log4j2.properties
cp "$trino_conf_dir/config/catalog/gravitino.properties"
/etc/trino/catalog/gravitino.properties
+
#
# Update `gravitino.uri = http://GRAVITINO_HOST_IP:GRAVITINO_HOST_PORT` in the
`conf/catalog/gravitino.properties`
sed -i
"s/GRAVITINO_HOST_IP:GRAVITINO_HOST_PORT/${GRAVITINO_HOST_IP}:${GRAVITINO_HOST_PORT}/g"
/etc/trino/catalog/gravitino.properties
diff --git a/integration-test/trino-it/launch.sh
b/integration-test/trino-it/launch.sh
index ca51922e2..469c9abf7 100755
--- a/integration-test/trino-it/launch.sh
+++ b/integration-test/trino-it/launch.sh
@@ -33,19 +33,17 @@ fi
cd ${playground_dir}
# create log dir
-mkdir -p ../build/trino-ci-container-log
+LOG_DIR=../build/trino-ci-container-log
+rm -fr $LOG_DIR
+mkdir -p $LOG_DIR
docker compose up -d
-if [ -n "$GRAVITINO_LOG_PATH" ]; then
- LOG_PATH=$GRAVITINO_LOG_PATH
-else
- LOG_PATH=../build/integration-test.log
-fi
+LOG_PATH=$LOG_DIR/trino-ci-docker-compose.log
echo "The docker compose log is: $LOG_PATH"
-nohup docker compose logs -f -t >> $LOG_PATH &
+nohup docker compose logs -f -t > $LOG_PATH &
max_attempts=300
attempts=0
diff --git a/integration-test/trino-test-tools/trino_test.sh
b/integration-test/trino-test-tools/trino_test.sh
index 7824cb5b9..d0d5b43f8 100755
--- a/integration-test/trino-test-tools/trino_test.sh
+++ b/integration-test/trino-test-tools/trino_test.sh
@@ -17,7 +17,7 @@
# specific language governing permissions and limitations
# under the License.
-#
+set -e
DIR=$(cd "$(dirname "$0")" && pwd)/../..
export GRAVITINO_ROOT_DIR=$(cd "$DIR" && pwd)
diff --git
a/trino-connector/src/main/java/org/apache/gravitino/trino/connector/catalog/CatalogConnectorManager.java
b/trino-connector/src/main/java/org/apache/gravitino/trino/connector/catalog/CatalogConnectorManager.java
index f3b9ac126..35dabfebb 100644
---
a/trino-connector/src/main/java/org/apache/gravitino/trino/connector/catalog/CatalogConnectorManager.java
+++
b/trino-connector/src/main/java/org/apache/gravitino/trino/connector/catalog/CatalogConnectorManager.java
@@ -60,7 +60,7 @@ public class CatalogConnectorManager {
private static final int CATALOG_LOAD_FREQUENCY_SECOND = 10;
private static final int NUMBER_EXECUTOR_THREAD = 1;
- private static final int LOAD_METALAKE_TIMEOUT = 30;
+ private static final int LOAD_METALAKE_TIMEOUT = 60;
private final ScheduledExecutorService executorService;
private final CatalogRegister catalogRegister;
diff --git
a/trino-connector/src/main/java/org/apache/gravitino/trino/connector/system/storedprocdure/AlterCatalogStoredProcedure.java
b/trino-connector/src/main/java/org/apache/gravitino/trino/connector/system/storedprocdure/AlterCatalogStoredProcedure.java
index ae9bfcaa5..67c310342 100644
---
a/trino-connector/src/main/java/org/apache/gravitino/trino/connector/system/storedprocdure/AlterCatalogStoredProcedure.java
+++
b/trino-connector/src/main/java/org/apache/gravitino/trino/connector/system/storedprocdure/AlterCatalogStoredProcedure.java
@@ -32,6 +32,7 @@ import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Optional;
+import org.apache.commons.lang3.StringUtils;
import org.apache.gravitino.CatalogChange;
import org.apache.gravitino.NameIdentifier;
import org.apache.gravitino.exceptions.NoSuchCatalogException;
@@ -140,7 +141,7 @@ public class AlterCatalogStoredProcedure extends
GravitinoStoredProcedure {
} catch (Exception e) {
throw new TrinoException(
GravitinoErrorCode.GRAVITINO_UNSUPPORTED_OPERATION,
- "alter catalog failed. " + e.getMessage(),
+ "alter catalog failed. " + (StringUtils.isEmpty(e.getMessage()) ? ""
: e.getMessage()),
e);
}
}
diff --git
a/trino-connector/src/main/java/org/apache/gravitino/trino/connector/system/storedprocdure/CreateCatalogStoredProcedure.java
b/trino-connector/src/main/java/org/apache/gravitino/trino/connector/system/storedprocdure/CreateCatalogStoredProcedure.java
index 17dc1edde..7731be034 100644
---
a/trino-connector/src/main/java/org/apache/gravitino/trino/connector/system/storedprocdure/CreateCatalogStoredProcedure.java
+++
b/trino-connector/src/main/java/org/apache/gravitino/trino/connector/system/storedprocdure/CreateCatalogStoredProcedure.java
@@ -29,6 +29,7 @@ import java.lang.invoke.MethodHandle;
import java.lang.invoke.MethodHandles;
import java.util.List;
import java.util.Map;
+import org.apache.commons.lang3.StringUtils;
import org.apache.gravitino.Catalog;
import org.apache.gravitino.NameIdentifier;
import org.apache.gravitino.exceptions.CatalogAlreadyExistsException;
@@ -114,7 +115,7 @@ public class CreateCatalogStoredProcedure extends
GravitinoStoredProcedure {
} catch (Exception e) {
throw new TrinoException(
GravitinoErrorCode.GRAVITINO_UNSUPPORTED_OPERATION,
- "Create catalog failed. " + e.getMessage(),
+ "Create catalog failed. " + (StringUtils.isEmpty(e.getMessage()) ?
"" : e.getMessage()),
e);
}
}
diff --git
a/trino-connector/src/main/java/org/apache/gravitino/trino/connector/system/storedprocdure/DropCatalogStoredProcedure.java
b/trino-connector/src/main/java/org/apache/gravitino/trino/connector/system/storedprocdure/DropCatalogStoredProcedure.java
index 51c0214c2..7b195b642 100644
---
a/trino-connector/src/main/java/org/apache/gravitino/trino/connector/system/storedprocdure/DropCatalogStoredProcedure.java
+++
b/trino-connector/src/main/java/org/apache/gravitino/trino/connector/system/storedprocdure/DropCatalogStoredProcedure.java
@@ -26,6 +26,7 @@ import io.trino.spi.procedure.Procedure;
import java.lang.invoke.MethodHandle;
import java.lang.invoke.MethodHandles;
import java.util.List;
+import org.apache.commons.lang3.StringUtils;
import org.apache.gravitino.NameIdentifier;
import org.apache.gravitino.exceptions.NoSuchMetalakeException;
import org.apache.gravitino.trino.connector.GravitinoErrorCode;
@@ -103,7 +104,7 @@ public class DropCatalogStoredProcedure extends
GravitinoStoredProcedure {
} catch (Exception e) {
throw new TrinoException(
GravitinoErrorCode.GRAVITINO_UNSUPPORTED_OPERATION,
- "Drop catalog failed. " + e.getMessage(),
+ "Drop catalog failed. " + (StringUtils.isEmpty(e.getMessage()) ? ""
: e.getMessage()),
e);
}
}
