This is an automated email from the ASF dual-hosted git repository.
gabor pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/parquet-mr.git
The following commit(s) were added to refs/heads/master by this push:
new cc1ae9f PARQUET-2059: Handle resource-intensive tests in CI (#915)
cc1ae9f is described below
commit cc1ae9f857920cf6982a59b3fd389277877fbb9e
Author: Gabor Szadovszky <[email protected]>
AuthorDate: Mon Aug 16 11:20:55 2021 +0200
PARQUET-2059: Handle resource-intensive tests in CI (#915)
---
.../apache/parquet/ResourceIntensiveTestRule.java | 58 ++++++++++++++++++++++
.../parquet/hadoop/TestLargeColumnChunk.java | 9 ++--
pom.xml | 5 ++
3 files changed, 69 insertions(+), 3 deletions(-)
diff --git
a/parquet-column/src/test/java/org/apache/parquet/ResourceIntensiveTestRule.java
b/parquet-column/src/test/java/org/apache/parquet/ResourceIntensiveTestRule.java
new file mode 100644
index 0000000..1a8902b
--- /dev/null
+++
b/parquet-column/src/test/java/org/apache/parquet/ResourceIntensiveTestRule.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.parquet;
+
+import static org.junit.Assume.assumeTrue;
+
+import org.junit.rules.TestRule;
+import org.junit.runner.Description;
+import org.junit.runners.model.Statement;
+
+/**
+ * A test rule for resource intensive tests that are not executed on every
environment (e.g. CI). It is managed by the
+ * system property {@code enableResourceIntensiveTests}. If the value is
{@code true} the related tests are executed,
+ * skipped otherwise.
+ */
+public class ResourceIntensiveTestRule implements TestRule {
+
+ private static final TestRule INSTANCE = new ResourceIntensiveTestRule(
+ Boolean.getBoolean("enableResourceIntensiveTests"));
+
+ public static TestRule get() {
+ return INSTANCE;
+ }
+
+ private final boolean enabled;
+
+ private ResourceIntensiveTestRule(boolean enabled) {
+ this.enabled = enabled;
+ }
+
+ @Override
+ public Statement apply(Statement base, Description description) {
+ return enabled ? base : new Statement() {
+ @Override
+ public void evaluate() {
+ assumeTrue("Resource intensive test is not executed due to the
limitations of the environment", enabled);
+ }
+ };
+ }
+
+}
diff --git
a/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/TestLargeColumnChunk.java
b/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/TestLargeColumnChunk.java
index 90015f5..9872652 100644
---
a/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/TestLargeColumnChunk.java
+++
b/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/TestLargeColumnChunk.java
@@ -35,6 +35,7 @@ import java.util.Random;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
+import org.apache.parquet.ResourceIntensiveTestRule;
import org.apache.parquet.example.data.Group;
import org.apache.parquet.example.data.GroupFactory;
import org.apache.parquet.example.data.simple.SimpleGroupFactory;
@@ -47,13 +48,12 @@ import org.apache.parquet.io.api.Binary;
import org.apache.parquet.schema.MessageType;
import org.junit.*;
import org.junit.rules.TemporaryFolder;
+import org.junit.rules.TestRule;
/**
* This test is to test parquet-mr working with potential int overflows (when
the sizes are greater than
- * Integer.MAX_VALUE). The test requires ~3GB memory so it is likely to fail
in the CI environment, so these
- * tests are flagged to be ignored.
+ * Integer.MAX_VALUE).
*/
-@Ignore
public class TestLargeColumnChunk {
private static final MessageType SCHEMA = buildMessage().addFields(
required(INT64).named("id"),
@@ -72,6 +72,9 @@ public class TestLargeColumnChunk {
private static Path file;
@ClassRule
+ public static TestRule maySkip = ResourceIntensiveTestRule.get();
+
+ @ClassRule
public static TemporaryFolder folder = new TemporaryFolder();
@BeforeClass
diff --git a/pom.xml b/pom.xml
index f143b08..441c503 100644
--- a/pom.xml
+++ b/pom.xml
@@ -112,6 +112,9 @@
<!-- properties for the profiles -->
<surefire.logLevel>INFO</surefire.logLevel>
+
+ <!-- Resource intesive tests are enabled by default but disabled in the CI
envrionment -->
+ <enableResourceIntensiveTests>true</enableResourceIntensiveTests>
</properties>
<modules>
@@ -418,6 +421,7 @@
<!-- Configure log level for Hadoop -->
<hadoop.logLevel>${surefire.logLevel}</hadoop.logLevel>
+
<enableResourceIntensiveTests>${enableResourceIntensiveTests}</enableResourceIntensiveTests>
</systemPropertyVariables>
<excludes>
<exclude>**/benchmark/*.java</exclude>
@@ -617,6 +621,7 @@
<properties>
<surefire.logLevel>WARN</surefire.logLevel>
<surefire.argLine>-XX:MaxJavaStackTraceDepth=10</surefire.argLine>
+ <enableResourceIntensiveTests>false</enableResourceIntensiveTests>
</properties>
</profile>
