Michael Blow has submitted this change and it was merged. Change subject: [NO ISSUE][TEST] Retry on file copy, ignore failures after 3 retries ......................................................................
[NO ISSUE][TEST] Retry on file copy, ignore failures after 3 retries - for NCServiceExecutionIT based tests, print JVM final flags, help ensure max gc wait time is being configured properly by NCService Change-Id: I7127ee0dcba6ead1a19a373b62e4d6aa22467909 Reviewed-on: https://asterix-gerrit.ics.uci.edu/2454 Sonar-Qube: Jenkins <[email protected]> Tested-by: Jenkins <[email protected]> Contrib: Jenkins <[email protected]> Integration-Tests: Jenkins <[email protected]> Reviewed-by: abdullah alamoudi <[email protected]> --- M asterixdb/asterix-common/pom.xml M asterixdb/asterix-common/src/test/java/org/apache/asterix/test/base/AsterixTestHelper.java R asterixdb/asterix-server/src/test/java/org/apache/asterix/test/server/SqlppExecutionNCServiceIT.java M asterixdb/asterix-server/src/test/resources/NCServiceExecutionIT/cc.conf M hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/file/FileUtil.java 5 files changed, 39 insertions(+), 13 deletions(-) Approvals: Anon. E. Moose #1000171: abdullah alamoudi: Looks good to me, approved Jenkins: Verified; No violations found; ; Verified diff --git a/asterixdb/asterix-common/pom.xml b/asterixdb/asterix-common/pom.xml index 79af3b6..ee7fe5c 100644 --- a/asterixdb/asterix-common/pom.xml +++ b/asterixdb/asterix-common/pom.xml @@ -180,10 +180,6 @@ </build> <dependencies> <dependency> - <groupId>commons-io</groupId> - <artifactId>commons-io</artifactId> - </dependency> - <dependency> <groupId>org.apache.commons</groupId> <artifactId>commons-lang3</artifactId> </dependency> diff --git a/asterixdb/asterix-common/src/test/java/org/apache/asterix/test/base/AsterixTestHelper.java b/asterixdb/asterix-common/src/test/java/org/apache/asterix/test/base/AsterixTestHelper.java index 1ef5abd..3840a00 100644 --- a/asterixdb/asterix-common/src/test/java/org/apache/asterix/test/base/AsterixTestHelper.java +++ b/asterixdb/asterix-common/src/test/java/org/apache/asterix/test/base/AsterixTestHelper.java @@ -25,9 +25,15 @@ import java.io.IOException; import java.util.ArrayList; -import org.apache.commons.io.FileUtils; +import org.apache.hyracks.util.file.FileUtil; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; public class AsterixTestHelper { + private static final Logger LOGGER = LogManager.getLogger(); + + private AsterixTestHelper() { + } public static String extToResExt(String fname, String resultExt) { int dot = fname.lastIndexOf('.'); @@ -77,11 +83,12 @@ throw new IllegalArgumentException("Exists and not a directory: " + destDir); } for (File child : srcDir.listFiles()) { + File destChild = new File(destDir, child.getName()); if (child.isDirectory()) { - deepSelectiveCopy(child, new File(destDir, child.getName()), filter); + deepSelectiveCopy(child, destChild, filter); } else if (filter.accept(child)) { - destDir.mkdirs(); - FileUtils.copyFile(child, new File(destDir, child.getName())); + FileUtil.safeCopyFile(child, destChild); + return; } } } diff --git a/asterixdb/asterix-server/src/test/java/org/apache/asterix/test/server/SqlppExecutionIT.java b/asterixdb/asterix-server/src/test/java/org/apache/asterix/test/server/SqlppExecutionNCServiceIT.java similarity index 89% rename from asterixdb/asterix-server/src/test/java/org/apache/asterix/test/server/SqlppExecutionIT.java rename to asterixdb/asterix-server/src/test/java/org/apache/asterix/test/server/SqlppExecutionNCServiceIT.java index f1738fe..88ada80 100644 --- a/asterixdb/asterix-server/src/test/java/org/apache/asterix/test/server/SqlppExecutionIT.java +++ b/asterixdb/asterix-server/src/test/java/org/apache/asterix/test/server/SqlppExecutionNCServiceIT.java @@ -27,9 +27,9 @@ * Runs the runtime test cases under 'asterix-app/src/test/resources/runtimets'. */ @RunWith(Parameterized.class) -public class SqlppExecutionIT extends AbstractExecutionIT { +public class SqlppExecutionNCServiceIT extends AbstractExecutionIT { - @Parameters(name = "SqlppExecutionIT {index}: {0}") + @Parameters(name = "SqlppExecutionNCServiceIT {index}: {0}") public static Collection<Object[]> tests() throws Exception { Collection<Object[]> testArgs = buildTestsInXml("only_sqlpp.xml"); if (testArgs.size() == 0) { @@ -48,7 +48,7 @@ } - public SqlppExecutionIT(TestCaseContext tcCtx) { + public SqlppExecutionNCServiceIT(TestCaseContext tcCtx) { super(tcCtx); } } diff --git a/asterixdb/asterix-server/src/test/resources/NCServiceExecutionIT/cc.conf b/asterixdb/asterix-server/src/test/resources/NCServiceExecutionIT/cc.conf index 4020ca1..4658f1b 100644 --- a/asterixdb/asterix-server/src/test/resources/NCServiceExecutionIT/cc.conf +++ b/asterixdb/asterix-server/src/test/resources/NCServiceExecutionIT/cc.conf @@ -20,7 +20,7 @@ core.dump.dir=../asterix-server/target/tmp/asterix_nc1/coredump iodevices=../asterix-server/target/tmp/asterix_nc1/iodevice1,../asterix-server/target/tmp/asterix_nc1/iodevice2 nc.api.port=19004 -jvm.args=-Xloggc:/tmp/asterix-server-failsafe/asterix_nc1-%p-gc.log -XX:+PrintGC -XX:+PrintGCDateStamps -XX:GCLogFileSize=10M +jvm.args=-Xloggc:/tmp/asterix-server-failsafe/asterix_nc1-%p-gc.log -XX:+PrintGC -XX:+PrintGCDateStamps -XX:GCLogFileSize=10M -XX:+PrintFlagsFinal #jvm.args=-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=5006 [nc/asterix_nc2] @@ -29,7 +29,7 @@ core.dump.dir=../asterix-server/target/tmp/asterix_nc2/coredump iodevices=../asterix-server/target/tmp/asterix_nc2/iodevice1,../asterix-server/target/tmp/asterix_nc2/iodevice2 nc.api.port=19005 -jvm.args=-Xloggc:/tmp/asterix-server-failsafe/asterix_nc2-%p-gc.log -XX:+PrintGC -XX:+PrintGCDateStamps -XX:GCLogFileSize=10M +jvm.args=-Xloggc:/tmp/asterix-server-failsafe/asterix_nc2-%p-gc.log -XX:+PrintGC -XX:+PrintGCDateStamps -XX:GCLogFileSize=10M -XX:+PrintFlagsFinal #jvm.args=-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=5007 [nc] diff --git a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/file/FileUtil.java b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/file/FileUtil.java index 1b0093d..443640a 100644 --- a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/file/FileUtil.java +++ b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/file/FileUtil.java @@ -30,6 +30,7 @@ private static final Logger LOGGER = LogManager.getLogger(); private static final Object LOCK = new Object(); + private static final int MAX_COPY_ATTEMPTS = 3; private FileUtil() { } @@ -63,4 +64,26 @@ return joined.replaceAll("(" + escapedSeparator + ")+", "$1").replaceAll(escapedSeparator + "$", ""); } } + + public static void safeCopyFile(File child, File destChild) throws IOException { + forceMkdirs(destChild.getParentFile()); + IOException ioException = null; + while (true) { + try { + FileUtils.copyFile(child, destChild); + return; + } catch (IOException e) { + if (ioException == null) { + ioException = e; + } else { + ioException.addSuppressed(e); + } + if (ioException.getSuppressed().length >= MAX_COPY_ATTEMPTS) { + LOGGER.warn("Unable to copy {} to {} after " + MAX_COPY_ATTEMPTS + " attempts; skipping file", + child, destChild, e); + return; + } + } + } + } } -- To view, visit https://asterix-gerrit.ics.uci.edu/2454 To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-MessageType: merged Gerrit-Change-Id: I7127ee0dcba6ead1a19a373b62e4d6aa22467909 Gerrit-PatchSet: 3 Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Owner: Michael Blow <[email protected]> Gerrit-Reviewer: Anon. E. Moose #1000171 Gerrit-Reviewer: Jenkins <[email protected]> Gerrit-Reviewer: Michael Blow <[email protected]> Gerrit-Reviewer: Murtadha Hubail <[email protected]> Gerrit-Reviewer: abdullah alamoudi <[email protected]>
