This is an automated email from the ASF dual-hosted git repository. dzamo pushed a commit to branch 1.20 in repository https://gitbox.apache.org/repos/asf/drill.git
commit 00d51f461d7e1238c26027ad2218ba379583faea Author: luocooong <[email protected]> AuthorDate: Sun Jul 3 20:02:40 2022 +0800 Try and reduce the vm crash on fork Use the drop_caches and split unit tests Ready for the review --- .github/workflows/ci.yml | 22 ++++++++--------- .travis.yml | 2 +- .../apache/drill/categories/EasyOutOfMemory.java | 28 ++++++++++++++++++++++ .../org/apache/drill/categories/package-info.java | 2 ++ .../drill/exec/fn/impl/TestAggregateFunctions.java | 3 ++- .../physical/impl/writer/TestParquetWriter.java | 5 ---- .../drill/exec/server/TestDrillbitResilience.java | 2 +- pom.xml | 2 +- 8 files changed, 45 insertions(+), 21 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 230fdb10c6..7988390440 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -46,22 +46,20 @@ jobs: key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} restore-keys: | ${{ runner.os }}-maven- - # Caches MySQL directory used for JDBC storage plugin tests - - name: Cache MySQL - uses: actions/cache@v2 - with: - path: ~/.embedmysql - key: ${{ runner.os }}-mysql - - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v2 - with: - distribution: 'adopt' - java-version: ${{ matrix.java }} - name: Build and test # The total GitHub Actions memory is 7000Mb. But GitHub CI requires some memory for the container to perform tests run: | MAVEN_OPTS="-XX:+UseG1GC" - mvn install --batch-mode --no-transfer-progress -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=120 # -X -V for debugging + sudo sh -c 'echo 1 > /proc/sys/vm/drop_caches' && \ + mvn install --batch-mode --no-transfer-progress \ + -DexcludedGroups=org.apache.drill.categories.EasyOutOfMemory \ + -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=120 + - name: Test Specific Categories # EasyOutOfMemory + run: | + sudo sh -c 'echo 1 > /proc/sys/vm/drop_caches' && \ + mvn test -pl org.apache.drill.exec:drill-java-exec \ + -Dgroups=org.apache.drill.categories.EasyOutOfMemory \ + -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=120 checkstyle_protobuf: name: Run checkstyle and generate protobufs diff --git a/.travis.yml b/.travis.yml index 6461e8205d..6e576e1640 100644 --- a/.travis.yml +++ b/.travis.yml @@ -73,7 +73,7 @@ install: # For protobuf phase: builds Drill project, performs license checkstyle goal and regenerates Java and C++ Protobuf files - | if [ $PHASE = "tests" ]; then \ - mvn install --batch-mode --no-transfer-progress -DforkCount=1 -DdirectMemoryMb=$DIRECTMEMORYMB \ + mvn install --batch-mode --no-transfer-progress -DdirectMemoryMb=$DIRECTMEMORYMB \ -DexcludedGroups="org.apache.drill.categories.SlowTest,org.apache.drill.categories.UnlikelyTest,org.apache.drill.categories.SecurityTest"; \ elif [ $PHASE = "build_checkstyle_protobuf" ]; then \ MAVEN_OPTS="-Xms1G -Xmx1G" mvn install --no-transfer-progress -Drat.skip=false -Dlicense.skip=false --batch-mode -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn -DskipTests=true -Dmaven.javadoc.skip=true -Dmaven.source.skip=true && \ diff --git a/common/src/test/java/org/apache/drill/categories/EasyOutOfMemory.java b/common/src/test/java/org/apache/drill/categories/EasyOutOfMemory.java new file mode 100644 index 0000000000..2dd29456ab --- /dev/null +++ b/common/src/test/java/org/apache/drill/categories/EasyOutOfMemory.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.categories; + +/** + * Split the unit test for EasyOutOfMemory categories.<br/> + * Note: If you use this category to mark the test class, + * the CI will use a new clean JVM to test them. + * + */ +public interface EasyOutOfMemory extends FlakyTest { + +} diff --git a/common/src/test/java/org/apache/drill/categories/package-info.java b/common/src/test/java/org/apache/drill/categories/package-info.java index bb699c14ba..4f80760fa9 100644 --- a/common/src/test/java/org/apache/drill/categories/package-info.java +++ b/common/src/test/java/org/apache/drill/categories/package-info.java @@ -25,6 +25,8 @@ * <li>{@link org.apache.drill.categories.SlowTest}: These tests run slowly, and are disabled by default.</li> * <li>{@link org.apache.drill.categories.UnlikelyTest}: These tests represent corner cases, specific bug fixes, or tests for pieces of code that are unlikely to be changed. * These are disabled by default</li> + * <li>{@link org.apache.drill.categories.FlakyTest}: A category for tests that intermittently fail.</li> + * <li>{@link org.apache.drill.categories.EasyOutOfMemory}: Inherited class FlakyTest and allow the CI tool uses a new JVM to test the unit.</li> * </ul> * </p> * <p> diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestAggregateFunctions.java b/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestAggregateFunctions.java index 182c983583..11be3bc3c6 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestAggregateFunctions.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestAggregateFunctions.java @@ -31,6 +31,7 @@ import org.apache.drill.exec.planner.physical.PlannerSettings; import org.apache.drill.exec.record.RecordBatchLoader; import org.apache.drill.exec.record.VectorWrapper; import org.apache.drill.exec.util.Text; +import org.apache.drill.categories.EasyOutOfMemory; import org.apache.drill.categories.OperatorTest; import org.apache.drill.categories.PlannerTest; import org.apache.drill.categories.SqlFunctionTest; @@ -68,7 +69,7 @@ import static org.hamcrest.CoreMatchers.containsString; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; -@Category({SqlFunctionTest.class, OperatorTest.class, PlannerTest.class}) +@Category({ SqlFunctionTest.class, OperatorTest.class, PlannerTest.class, EasyOutOfMemory.class }) public class TestAggregateFunctions extends ClusterTest { @Rule diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/writer/TestParquetWriter.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/writer/TestParquetWriter.java index 80ba94418b..28b009e869 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/writer/TestParquetWriter.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/writer/TestParquetWriter.java @@ -52,9 +52,6 @@ import org.junit.Test; import org.junit.experimental.categories.Category; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; -import org.junit.jupiter.api.condition.DisabledOnOs; -import org.junit.jupiter.api.condition.EnabledIfSystemProperty; -import org.junit.jupiter.api.condition.OS; import java.io.File; import java.io.FileWriter; @@ -1003,8 +1000,6 @@ public class TestParquetWriter extends ClusterTest { // Only attempt this test on Linux / amd64 because com.rdblue.brotli-codec // only bundles natives for Mac and Linux on AMD64. See PARQUET-1975. @Test - @EnabledIfSystemProperty(named = "os.arch", matches = "(amd64|x86_64)") - @DisabledOnOs({ OS.WINDOWS }) public void testTPCHReadWriteBrotli() throws Exception { try { client.alterSession(ExecConstants.PARQUET_WRITER_COMPRESSION_TYPE, "brotli"); diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/server/TestDrillbitResilience.java b/exec/java-exec/src/test/java/org/apache/drill/exec/server/TestDrillbitResilience.java index 564112e1fc..3f66fd67a5 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/server/TestDrillbitResilience.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/server/TestDrillbitResilience.java @@ -117,7 +117,7 @@ public class TestDrillbitResilience extends ClusterTest { private static final int NUM_RUNS = 3; private static final int PROBLEMATIC_TEST_NUM_RUNS = 3; private static final int TIMEOUT = 15; - private final static Level CURRENT_LOG_LEVEL = Level.DEBUG; + private final static Level CURRENT_LOG_LEVEL = Level.INFO; /** * Note: Counting sys.memory executes a fragment on every drillbit. This is a better check in comparison to diff --git a/pom.xml b/pom.xml index 27385ae964..bc845a5ab5 100644 --- a/pom.xml +++ b/pom.xml @@ -115,7 +115,7 @@ <codemodel.version>2.6</codemodel.version> <joda.version>2.10.5</joda.version> <javax.el.version>3.0.0</javax.el.version> - <surefire.version>3.0.0-M5</surefire.version> + <surefire.version>3.0.0-M7</surefire.version> <jna.version>5.8.0</jna.version> <commons.compress.version>1.21</commons.compress.version> <hikari.version>4.0.3</hikari.version>
