This is an automated email from the ASF dual-hosted git repository.
dzamo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git
The following commit(s) were added to refs/heads/master by this push:
new 7322e76 DRILL-8145: Fix flaky
TestDrillbitResilience#memoryLeaksWhenCancelled test case (#2471)
7322e76 is described below
commit 7322e7615dc908932740bfd550ccc49471bf7121
Author: Vitalii Diravka <[email protected]>
AuthorDate: Sun Mar 6 18:02:12 2022 +0200
DRILL-8145: Fix flaky TestDrillbitResilience#memoryLeaksWhenCancelled test
case (#2471)
* Set Brotli codec jar and test to occur only on Linux amd64.
* DRILL-8145: Fix flaky TestDrillbitResilience#memoryLeaksWhenCancelled
test case
- changing timeout for TestDrillbitResilience tests
- timing tuning for memoryLeaksWhenCancelled
- update TestContainers version
- -DforkCount=1 for Travis Maven build
- directMemoryMb: 2500 -> 4500 leads to less occasinal test failures
Co-authored-by: James Turton <[email protected]>
---
.travis.yml | 2 +-
exec/java-exec/pom.xml | 28 +++++++++++++++-------
.../physical/impl/writer/TestParquetWriter.java | 10 ++++----
.../drill/exec/server/TestDrillbitResilience.java | 11 ++++-----
pom.xml | 4 ++--
5 files changed, 31 insertions(+), 24 deletions(-)
diff --git a/.travis.yml b/.travis.yml
index 957accd..297645b 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -71,7 +71,7 @@ install:
# For protobuf phase: builds Drill project, performs license checkstyle goal
and regenerates Java and C++ Protobuf files
- |
if [ $PHASE = "tests" ]; then \
- mvn install --batch-mode --no-transfer-progress \
+ mvn install --batch-mode --no-transfer-progress -DforkCount=1 \
-DexcludedGroups="org.apache.drill.categories.SlowTest,org.apache.drill.categories.UnlikelyTest,org.apache.drill.categories.SecurityTest";
\
elif [ $PHASE = "build_checkstyle_protobuf" ]; then \
MAVEN_OPTS="-Xms1G -Xmx1G" mvn install --no-transfer-progress
-Drat.skip=false -Dlicense.skip=false --batch-mode
-Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn
-DskipTests=true -Dmaven.javadoc.skip=true -Dmaven.source.skip=true && \
diff --git a/exec/java-exec/pom.xml b/exec/java-exec/pom.xml
index f80abef..e8bf1fd 100644
--- a/exec/java-exec/pom.xml
+++ b/exec/java-exec/pom.xml
@@ -650,14 +650,6 @@
<version>${testcontainers.version}</version>
<scope>test</scope>
</dependency>
- <dependency>
- <groupId>com.github.rdblue</groupId>
- <artifactId>brotli-codec</artifactId>
- <version>0.1.1</version>
- <!-- brotli-codec bundles natives for linux and darwin, amd64 only so
- we don't ship it so as not to break startup on windows or arm -->
- <scope>provided</scope>
- </dependency>
</dependencies>
<profiles>
@@ -767,8 +759,26 @@
</dependency>
</dependencies>
</profile>
+ <profile>
+ <!-- Only include a Brotli codec in test scope on Linux / amd64, see
PARQUET-1975 -->
+ <id>linux-amd64</id>
+ <activation>
+ <os>
+ <arch>amd64</arch>
+ <name>Linux</name>
+ </os>
+ </activation>
+ <dependencies>
+ <dependency>
+ <groupId>com.github.rdblue</groupId>
+ <artifactId>brotli-codec</artifactId>
+ <version>0.1.1</version>
+ <!-- this codec is not shipped because it breaks startup on
unsupported platforms -->
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+ </profile>
</profiles>
-
<build>
<plugins>
<plugin>
diff --git
a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/writer/TestParquetWriter.java
b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/writer/TestParquetWriter.java
index 7908c10..35335c2 100644
---
a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/writer/TestParquetWriter.java
+++
b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/writer/TestParquetWriter.java
@@ -53,7 +53,6 @@ import org.junit.experimental.categories.Category;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
import org.junit.jupiter.api.condition.EnabledIfSystemProperty;
-import org.junit.jupiter.api.condition.DisabledIfSystemProperty;
import java.io.File;
import java.io.FileWriter;
@@ -999,12 +998,11 @@ public class TestParquetWriter extends ClusterTest {
}
}
- // We currently bundle the JNI-based com.rdblue.brotli-codec and it only
provides
- // natives for Mac and Linux on AMD64. See PARQUET-1975.
+ // Only attempt this test on Linux / amd64 because com.rdblue.brotli-codec
+ // only bundles natives for Mac and Linux on AMD64. See PARQUET-1975.
@Test
- @DisabledIfSystemProperty(named = "os.name", matches = "Windows")
- @EnabledIfSystemProperty(named = "os.arch", matches = "amd64") // reported
for Linux on AMD64
- @EnabledIfSystemProperty(named = "os.arch", matches = "x86_64") // reported
for OS X on AMD64
+ @EnabledIfSystemProperty(named = "os.name", matches = "Linux")
+ @EnabledIfSystemProperty(named = "os.arch", matches = "amd64")
public void testTPCHReadWriteBrotli() throws Exception {
try {
client.alterSession(ExecConstants.PARQUET_WRITER_COMPRESSION_TYPE,
"brotli");
diff --git
a/exec/java-exec/src/test/java/org/apache/drill/exec/server/TestDrillbitResilience.java
b/exec/java-exec/src/test/java/org/apache/drill/exec/server/TestDrillbitResilience.java
index 6abc62b..564112e 100644
---
a/exec/java-exec/src/test/java/org/apache/drill/exec/server/TestDrillbitResilience.java
+++
b/exec/java-exec/src/test/java/org/apache/drill/exec/server/TestDrillbitResilience.java
@@ -18,7 +18,6 @@
package org.apache.drill.exec.server;
import static org.apache.drill.exec.ExecConstants.SLICE_TARGET;
-import static org.apache.drill.exec.ExecConstants.SLICE_TARGET_DEFAULT;
import static
org.apache.drill.exec.planner.physical.PlannerSettings.ENABLE_HASH_AGG_OPTION;
import static
org.apache.drill.exec.planner.physical.PlannerSettings.PARTITION_SENDER_SET_THREADS;
import static org.junit.jupiter.api.Assertions.assertEquals;
@@ -117,7 +116,7 @@ public class TestDrillbitResilience extends ClusterTest {
*/
private static final int NUM_RUNS = 3;
private static final int PROBLEMATIC_TEST_NUM_RUNS = 3;
- private static final int TIMEOUT = 10;
+ private static final int TIMEOUT = 15;
private final static Level CURRENT_LOG_LEVEL = Level.DEBUG;
/**
@@ -619,7 +618,7 @@ public class TestDrillbitResilience extends ClusterTest {
final long after = countAllocatedMemory();
assertEquals(before, after, () -> String.format("We are leaking %d
bytes", after - before));
} finally {
- client.alterSession(SLICE_TARGET, Long.toString(SLICE_TARGET_DEFAULT));
+ client.resetSession(SLICE_TARGET);
}
}
@@ -651,7 +650,7 @@ public class TestDrillbitResilience extends ClusterTest {
assertEquals(before, after, () -> String.format("We are leaking %d
bytes", after - before));
} finally {
- client.alterSession(SLICE_TARGET, Long.toString(SLICE_TARGET_DEFAULT));
+ client.resetSession(SLICE_TARGET);
}
}
@@ -809,7 +808,7 @@ public class TestDrillbitResilience extends ClusterTest {
try {
Thread.sleep(1000);
} catch (InterruptedException e) {
- logger.debug("Sleep thread interrupted. Ignore it");
+ logger.debug("Cancelling thread interrupted. Ignore it");
// just ignore
}
logger.debug("Cancelling {} query started", queryId);
@@ -913,7 +912,7 @@ public class TestDrillbitResilience extends ClusterTest {
// wait to make sure all fragments finished cleaning up
try {
logger.debug("Sleep thread for 2 seconds");
- Thread.sleep(2000);
+ Thread.sleep(1500); // 1500
} catch (InterruptedException e) {
logger.debug("Sleep thread interrupted. Ignore it", e);
// just ignore
diff --git a/pom.xml b/pom.xml
index 7914424..eab12eb 100644
--- a/pom.xml
+++ b/pom.xml
@@ -100,7 +100,7 @@
<asm.version>9.2</asm.version>
<excludedGroups />
<memoryMb>2500</memoryMb>
- <directMemoryMb>2500</directMemoryMb>
+ <directMemoryMb>4500</directMemoryMb>
<rat.skip>true</rat.skip>
<license.skip>true</license.skip>
<docker.repository>apache/drill</docker.repository>
@@ -126,7 +126,7 @@
<commons.cli.version>1.4</commons.cli.version>
<snakeyaml.version>1.26</snakeyaml.version>
<commons.lang3.version>3.10</commons.lang3.version>
- <testcontainers.version>1.16.2</testcontainers.version>
+ <testcontainers.version>1.16.3</testcontainers.version>
<typesafe.config.version>1.0.0</typesafe.config.version>
<commons.codec.version>1.14</commons.codec.version>
<metadata.extractor.version>2.13.0</metadata.extractor.version>
