This is an automated email from the ASF dual-hosted git repository.
wanghailin pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/seatunnel.git
The following commit(s) were added to refs/heads/dev by this push:
new 8a41b8eb1f [CI] Fix unstable ci with no left space error (#6356)
8a41b8eb1f is described below
commit 8a41b8eb1f4cf8da2fd5a734914d11766178afee
Author: Jia Fan <[email protected]>
AuthorDate: Thu Feb 22 21:12:51 2024 +0800
[CI] Fix unstable ci with no left space error (#6356)
---
.github/workflows/backend.yml | 8 ++++
.../container/seatunnel/SeaTunnelContainer.java | 47 ++++++++++++----------
.../seatunnel/engine/e2e/JobClientJobProxyIT.java | 16 ++++++++
.../seatunnel/engine/server/master/JobMaster.java | 4 +-
tools/github/free_disk_space.sh | 45 +++++++++++++++++++++
5 files changed, 98 insertions(+), 22 deletions(-)
diff --git a/.github/workflows/backend.yml b/.github/workflows/backend.yml
index d670ec1824..113235daa9 100644
--- a/.github/workflows/backend.yml
+++ b/.github/workflows/backend.yml
@@ -825,6 +825,8 @@ jobs:
java-version: ${{ matrix.java }}
distribution: 'temurin'
cache: 'maven'
+ - name: free disk space
+ run: tools/github/free_disk_space.sh
- name: run jdbc connectors integration test (part-1)
if: needs.changes.outputs.api == 'true'
run: |
@@ -849,6 +851,8 @@ jobs:
java-version: ${{ matrix.java }}
distribution: 'temurin'
cache: 'maven'
+ - name: free disk space
+ run: tools/github/free_disk_space.sh
- name: run jdbc connectors integration test (part-2)
if: needs.changes.outputs.api == 'true'
run: |
@@ -945,6 +949,8 @@ jobs:
java-version: ${{ matrix.java }}
distribution: 'temurin'
cache: 'maven'
+ - name: free disk space
+ run: tools/github/free_disk_space.sh
- name: run jdbc connectors integration test (part-6)
if: needs.changes.outputs.api == 'true'
run: |
@@ -1088,6 +1094,8 @@ jobs:
java-version: ${{ matrix.java }}
distribution: 'temurin'
cache: 'maven'
+ - name: free disk space
+ run: tools/github/free_disk_space.sh
- name: run doris connector integration test
run: |
./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false
-D"license.skipAddThirdParty"=true --no-snapshot-updates -pl
:connector-doris-e2e -am -Pci
diff --git
a/seatunnel-e2e/seatunnel-e2e-common/src/test/java/org/apache/seatunnel/e2e/common/container/seatunnel/SeaTunnelContainer.java
b/seatunnel-e2e/seatunnel-e2e-common/src/test/java/org/apache/seatunnel/e2e/common/container/seatunnel/SeaTunnelContainer.java
index 8a98a50e20..b888a851b9 100644
---
a/seatunnel-e2e/seatunnel-e2e-common/src/test/java/org/apache/seatunnel/e2e/common/container/seatunnel/SeaTunnelContainer.java
+++
b/seatunnel-e2e/seatunnel-e2e-common/src/test/java/org/apache/seatunnel/e2e/common/container/seatunnel/SeaTunnelContainer.java
@@ -209,26 +209,7 @@ public class SeaTunnelContainer extends
AbstractTestContainer {
private List<String> removeSystemThread(List<String> beforeThreads,
List<String> afterThreads)
throws IOException {
- Pattern aqsThread = Pattern.compile("pool-[0-9]-thread-[0-9]");
- afterThreads.removeIf(
- s ->
- s.startsWith("hz.main")
- ||
s.startsWith("seatunnel-coordinator-service")
- || s.startsWith("GC task thread")
- || s.contains("CompilerThread")
- ||
s.contains("NioNetworking-closeListenerExecutor")
- || s.contains("ForkJoinPool.commonPool")
- || s.contains("DestroyJavaVM")
- || s.contains("main-query-state-checker")
- || s.contains("Keep-Alive-SocketCleaner")
- || s.contains("process reaper")
- || s.startsWith("Timer-")
- || s.contains("InterruptTimer")
- || s.contains("Java2D Disposer")
- || s.contains(
-
"org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner")
- || s.startsWith("Log4j2-TF-")
- || aqsThread.matcher(s).matches());
+ afterThreads.removeIf(SeaTunnelContainer::isSystemThread);
afterThreads.removeIf(beforeThreads::contains);
Map<String, String> threadAndClassLoader = getThreadClassLoader();
List<String> notSystemClassLoaderThread =
@@ -247,9 +228,31 @@ public class SeaTunnelContainer extends
AbstractTestContainer {
.collect(Collectors.toList());
notSystemClassLoaderThread.addAll(afterThreads);
notSystemClassLoaderThread.removeIf(this::isIssueWeAlreadyKnow);
+
notSystemClassLoaderThread.removeIf(SeaTunnelContainer::isSystemThread);
return notSystemClassLoaderThread;
}
+ private static boolean isSystemThread(String s) {
+ Pattern aqsThread = Pattern.compile("pool-[0-9]-thread-[0-9]");
+ return s.startsWith("hz.main")
+ || s.startsWith("seatunnel-coordinator-service")
+ || s.startsWith("GC task thread")
+ || s.contains("CompilerThread")
+ || s.contains("NioNetworking-closeListenerExecutor")
+ || s.contains("ForkJoinPool.commonPool")
+ || s.contains("DestroyJavaVM")
+ || s.contains("main-query-state-checker")
+ || s.contains("Keep-Alive-SocketCleaner")
+ || s.contains("process reaper")
+ || s.startsWith("Timer-")
+ || s.contains("InterruptTimer")
+ || s.contains("Java2D Disposer")
+ || s.contains(
+
"org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner")
+ || s.startsWith("Log4j2-TF-")
+ || aqsThread.matcher(s).matches();
+ }
+
private void classLoaderObjectCheck(Integer maxSize) throws IOException,
InterruptedException {
Map<String, Integer> objects = ContainerUtil.getJVMLiveObject(server);
String className =
@@ -307,7 +310,9 @@ public class SeaTunnelContainer extends
AbstractTestContainer {
// MongoDB
|| threadName.startsWith("BufferPoolPruner")
|| threadName.startsWith("MaintenanceTimer")
- || threadName.startsWith("cluster-");
+ || threadName.startsWith("cluster-")
+ // Iceberg
+ || threadName.startsWith("iceberg");
}
@Override
diff --git
a/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/JobClientJobProxyIT.java
b/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/JobClientJobProxyIT.java
index 9f5ddf8c38..7c2e332ba5 100644
---
a/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/JobClientJobProxyIT.java
+++
b/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/JobClientJobProxyIT.java
@@ -24,10 +24,12 @@ import org.apache.commons.lang3.StringUtils;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
+import org.testcontainers.DockerClientFactory;
import org.testcontainers.containers.Container;
import org.testcontainers.containers.GenericContainer;
import org.testcontainers.containers.output.Slf4jLogConsumer;
import org.testcontainers.containers.wait.strategy.Wait;
+import org.testcontainers.images.PullPolicy;
import org.testcontainers.utility.DockerLoggerFactory;
import org.testcontainers.utility.MountableFile;
@@ -35,6 +37,7 @@ import java.io.IOException;
import java.nio.file.Paths;
import java.util.Arrays;
import java.util.Collections;
+import java.util.stream.Collectors;
import static
org.apache.seatunnel.e2e.common.util.ContainerUtil.PROJECT_ROOT_PATH;
@@ -52,6 +55,7 @@ public class JobClientJobProxyIT extends SeaTunnelContainer {
ContainerUtil.adaptPathForWin(
Paths.get(SEATUNNEL_HOME, "bin",
SERVER_SHELL).toString()))
.withNetworkAliases("server")
+ .withImagePullPolicy(PullPolicy.alwaysPull())
.withExposedPorts()
.withLogConsumer(
new Slf4jLogConsumer(
@@ -79,6 +83,18 @@ public class JobClientJobProxyIT extends SeaTunnelContainer {
PROJECT_ROOT_PATH
+
"/seatunnel-shade/seatunnel-hadoop3-3.1.4-uber/target/seatunnel-hadoop3-3.1.4-uber.jar"),
Paths.get(SEATUNNEL_HOME,
"lib/seatunnel-hadoop3-3.1.4-uber.jar").toString());
+ LOG.info(
+ "find images: "
+ +
DockerClientFactory.lazyClient().listImagesCmd().exec().stream()
+ .map(
+ image -> {
+ if (image.getRepoTags() != null) {
+ return image.getRepoTags()[0];
+ } else {
+ return
image.getRepoDigests()[0];
+ }
+ })
+ .collect(Collectors.joining(",")));
server.start();
// execute extra commands
executeExtraCommands(server);
diff --git
a/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/master/JobMaster.java
b/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/master/JobMaster.java
index ca75e185aa..9d9e2b93d6 100644
---
a/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/master/JobMaster.java
+++
b/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/master/JobMaster.java
@@ -200,7 +200,7 @@ public class JobMaster {
jobImmutableInformation.getJobConfig().getName(),
jobImmutableInformation.getJobId(),
jobImmutableInformation.getPluginJarsUrls()));
-
+ ClassLoader appClassLoader =
Thread.currentThread().getContextClassLoader();
ClassLoader classLoader =
new
SeaTunnelChildFirstClassLoader(jobImmutableInformation.getPluginJarsUrls());
logicalDag =
@@ -222,6 +222,8 @@ public class JobMaster {
runningJobStateTimestampsIMap,
engineConfig.getQueueType(),
engineConfig);
+ // revert to app class loader, it may be changed by
PlanUtils.fromLogicalDAG
+ Thread.currentThread().setContextClassLoader(appClassLoader);
this.physicalPlan = planTuple.f0();
this.physicalPlan.setJobMaster(this);
this.checkpointPlanMap = planTuple.f1();
diff --git a/tools/github/free_disk_space.sh b/tools/github/free_disk_space.sh
new file mode 100755
index 0000000000..7e35be93f3
--- /dev/null
+++ b/tools/github/free_disk_space.sh
@@ -0,0 +1,45 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+echo
"=============================================================================="
+echo "Freeing up disk space on CI system"
+echo
"=============================================================================="
+
+echo "Listing 100 largest packages"
+dpkg-query -Wf '${Installed-Size}\t${Package}\n' | sort -n | tail -n 100
+df -h
+echo "Removing large packages"
+sudo apt-get remove -y '^dotnet-.*'
+sudo apt-get remove -y '^llvm-.*'
+sudo apt-get remove -y 'php.*'
+sudo apt-get remove -y '^mongodb-.*'
+sudo apt-get remove -y '^mysql-.*'
+sudo apt-get remove -y azure-cli google-cloud-sdk hhvm google-chrome-stable
firefox powershell mono-devel libgl1-mesa-dri
+sudo apt-get autoremove -y
+sudo apt-get clean
+df -h
+echo "Removing large directories"
+
+sudo rm -rf /usr/share/dotnet/
+sudo rm -rf /usr/local/graalvm/
+sudo rm -rf /usr/local/.ghcup/
+sudo rm -rf /usr/local/share/powershell
+sudo rm -rf /usr/local/share/chromium
+sudo rm -rf /usr/local/lib/android
+sudo rm -rf /usr/local/lib/node_modules
+df -h
\ No newline at end of file
