Repository: asterixdb
Updated Branches:
  refs/heads/master 39256dfb6 -> bb29bc7c2


[NO ISSUE][TEST] Retry on file copy, ignore failures after 3 retries

- for NCServiceExecutionIT based tests, print JVM final flags, help ensure
  max gc wait time is being configured properly by NCService

Change-Id: I7127ee0dcba6ead1a19a373b62e4d6aa22467909
Reviewed-on: https://asterix-gerrit.ics.uci.edu/2454
Sonar-Qube: Jenkins <jenk...@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <jenk...@fulliautomatix.ics.uci.edu>
Contrib: Jenkins <jenk...@fulliautomatix.ics.uci.edu>
Integration-Tests: Jenkins <jenk...@fulliautomatix.ics.uci.edu>
Reviewed-by: abdullah alamoudi <bamou...@gmail.com>


Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/bb29bc7c
Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/bb29bc7c
Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/bb29bc7c

Branch: refs/heads/master
Commit: bb29bc7c2d2dab715e945534a8d42071e676013a
Parents: 39256df
Author: Michael Blow <mb...@apache.org>
Authored: Mon Mar 5 18:55:23 2018 -0500
Committer: Michael Blow <mb...@apache.org>
Committed: Tue Mar 6 11:11:28 2018 -0800

----------------------------------------------------------------------
 asterixdb/asterix-common/pom.xml                |  4 --
 .../asterix/test/base/AsterixTestHelper.java    | 15 ++++--
 .../asterix/test/server/SqlppExecutionIT.java   | 54 --------------------
 .../test/server/SqlppExecutionNCServiceIT.java  | 54 ++++++++++++++++++++
 .../test/resources/NCServiceExecutionIT/cc.conf |  4 +-
 .../org/apache/hyracks/util/file/FileUtil.java  | 23 +++++++++
 6 files changed, 90 insertions(+), 64 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/asterixdb/blob/bb29bc7c/asterixdb/asterix-common/pom.xml
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-common/pom.xml b/asterixdb/asterix-common/pom.xml
index 79af3b6..ee7fe5c 100644
--- a/asterixdb/asterix-common/pom.xml
+++ b/asterixdb/asterix-common/pom.xml
@@ -180,10 +180,6 @@
   </build>
   <dependencies>
     <dependency>
-      <groupId>commons-io</groupId>
-      <artifactId>commons-io</artifactId>
-    </dependency>
-    <dependency>
       <groupId>org.apache.commons</groupId>
       <artifactId>commons-lang3</artifactId>
     </dependency>

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/bb29bc7c/asterixdb/asterix-common/src/test/java/org/apache/asterix/test/base/AsterixTestHelper.java
----------------------------------------------------------------------
diff --git 
a/asterixdb/asterix-common/src/test/java/org/apache/asterix/test/base/AsterixTestHelper.java
 
b/asterixdb/asterix-common/src/test/java/org/apache/asterix/test/base/AsterixTestHelper.java
index 1ef5abd..3840a00 100644
--- 
a/asterixdb/asterix-common/src/test/java/org/apache/asterix/test/base/AsterixTestHelper.java
+++ 
b/asterixdb/asterix-common/src/test/java/org/apache/asterix/test/base/AsterixTestHelper.java
@@ -25,9 +25,15 @@ import java.io.FileReader;
 import java.io.IOException;
 import java.util.ArrayList;
 
-import org.apache.commons.io.FileUtils;
+import org.apache.hyracks.util.file.FileUtil;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 
 public class AsterixTestHelper {
+    private static final Logger LOGGER = LogManager.getLogger();
+
+    private AsterixTestHelper() {
+    }
 
     public static String extToResExt(String fname, String resultExt) {
         int dot = fname.lastIndexOf('.');
@@ -77,11 +83,12 @@ public class AsterixTestHelper {
             throw new IllegalArgumentException("Exists and not a directory: " 
+ destDir);
         }
         for (File child : srcDir.listFiles()) {
+            File destChild = new File(destDir, child.getName());
             if (child.isDirectory()) {
-                deepSelectiveCopy(child, new File(destDir, child.getName()), 
filter);
+                deepSelectiveCopy(child, destChild, filter);
             } else if (filter.accept(child)) {
-                destDir.mkdirs();
-                FileUtils.copyFile(child, new File(destDir, child.getName()));
+                FileUtil.safeCopyFile(child, destChild);
+                return;
             }
         }
     }

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/bb29bc7c/asterixdb/asterix-server/src/test/java/org/apache/asterix/test/server/SqlppExecutionIT.java
----------------------------------------------------------------------
diff --git 
a/asterixdb/asterix-server/src/test/java/org/apache/asterix/test/server/SqlppExecutionIT.java
 
b/asterixdb/asterix-server/src/test/java/org/apache/asterix/test/server/SqlppExecutionIT.java
deleted file mode 100644
index f1738fe..0000000
--- 
a/asterixdb/asterix-server/src/test/java/org/apache/asterix/test/server/SqlppExecutionIT.java
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.asterix.test.server;
-
-import java.io.File;
-import java.util.ArrayList;
-import java.util.Collection;
-
-import org.apache.asterix.testframework.context.TestCaseContext;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
-import org.junit.runners.Parameterized.Parameters;
-
-/**
- * Runs the runtime test cases under 
'asterix-app/src/test/resources/runtimets'.
- */
-@RunWith(Parameterized.class)
-public class SqlppExecutionIT extends AbstractExecutionIT {
-
-    @Parameters(name = "SqlppExecutionIT {index}: {0}")
-    public static Collection<Object[]> tests() throws Exception {
-        Collection<Object[]> testArgs = buildTestsInXml("only_sqlpp.xml");
-        if (testArgs.size() == 0) {
-            testArgs = buildTestsInXml("testsuite_sqlpp.xml");
-        }
-        return testArgs;
-    }
-
-    protected static Collection<Object[]> buildTestsInXml(String xmlfile) 
throws Exception {
-        Collection<Object[]> testArgs = new ArrayList<Object[]>();
-        TestCaseContext.Builder b = new TestCaseContext.Builder();
-        for (TestCaseContext ctx : b.build(new File(PATH_BASE), xmlfile)) {
-            testArgs.add(new Object[] { ctx });
-        }
-        return testArgs;
-
-    }
-
-    public SqlppExecutionIT(TestCaseContext tcCtx) {
-        super(tcCtx);
-    }
-}

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/bb29bc7c/asterixdb/asterix-server/src/test/java/org/apache/asterix/test/server/SqlppExecutionNCServiceIT.java
----------------------------------------------------------------------
diff --git 
a/asterixdb/asterix-server/src/test/java/org/apache/asterix/test/server/SqlppExecutionNCServiceIT.java
 
b/asterixdb/asterix-server/src/test/java/org/apache/asterix/test/server/SqlppExecutionNCServiceIT.java
new file mode 100644
index 0000000..88ada80
--- /dev/null
+++ 
b/asterixdb/asterix-server/src/test/java/org/apache/asterix/test/server/SqlppExecutionNCServiceIT.java
@@ -0,0 +1,54 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.asterix.test.server;
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.Collection;
+
+import org.apache.asterix.testframework.context.TestCaseContext;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameters;
+
+/**
+ * Runs the runtime test cases under 
'asterix-app/src/test/resources/runtimets'.
+ */
+@RunWith(Parameterized.class)
+public class SqlppExecutionNCServiceIT extends AbstractExecutionIT {
+
+    @Parameters(name = "SqlppExecutionNCServiceIT {index}: {0}")
+    public static Collection<Object[]> tests() throws Exception {
+        Collection<Object[]> testArgs = buildTestsInXml("only_sqlpp.xml");
+        if (testArgs.size() == 0) {
+            testArgs = buildTestsInXml("testsuite_sqlpp.xml");
+        }
+        return testArgs;
+    }
+
+    protected static Collection<Object[]> buildTestsInXml(String xmlfile) 
throws Exception {
+        Collection<Object[]> testArgs = new ArrayList<Object[]>();
+        TestCaseContext.Builder b = new TestCaseContext.Builder();
+        for (TestCaseContext ctx : b.build(new File(PATH_BASE), xmlfile)) {
+            testArgs.add(new Object[] { ctx });
+        }
+        return testArgs;
+
+    }
+
+    public SqlppExecutionNCServiceIT(TestCaseContext tcCtx) {
+        super(tcCtx);
+    }
+}

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/bb29bc7c/asterixdb/asterix-server/src/test/resources/NCServiceExecutionIT/cc.conf
----------------------------------------------------------------------
diff --git 
a/asterixdb/asterix-server/src/test/resources/NCServiceExecutionIT/cc.conf 
b/asterixdb/asterix-server/src/test/resources/NCServiceExecutionIT/cc.conf
index 4020ca1..4658f1b 100644
--- a/asterixdb/asterix-server/src/test/resources/NCServiceExecutionIT/cc.conf
+++ b/asterixdb/asterix-server/src/test/resources/NCServiceExecutionIT/cc.conf
@@ -20,7 +20,7 @@ txn.log.dir=../asterix-server/target/tmp/asterix_nc1/txnlog
 core.dump.dir=../asterix-server/target/tmp/asterix_nc1/coredump
 
iodevices=../asterix-server/target/tmp/asterix_nc1/iodevice1,../asterix-server/target/tmp/asterix_nc1/iodevice2
 nc.api.port=19004
-jvm.args=-Xloggc:/tmp/asterix-server-failsafe/asterix_nc1-%p-gc.log 
-XX:+PrintGC -XX:+PrintGCDateStamps -XX:GCLogFileSize=10M
+jvm.args=-Xloggc:/tmp/asterix-server-failsafe/asterix_nc1-%p-gc.log 
-XX:+PrintGC -XX:+PrintGCDateStamps -XX:GCLogFileSize=10M -XX:+PrintFlagsFinal
 #jvm.args=-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=5006
 
 [nc/asterix_nc2]
@@ -29,7 +29,7 @@ txn.log.dir=../asterix-server/target/tmp/asterix_nc2/txnlog
 core.dump.dir=../asterix-server/target/tmp/asterix_nc2/coredump
 
iodevices=../asterix-server/target/tmp/asterix_nc2/iodevice1,../asterix-server/target/tmp/asterix_nc2/iodevice2
 nc.api.port=19005
-jvm.args=-Xloggc:/tmp/asterix-server-failsafe/asterix_nc2-%p-gc.log 
-XX:+PrintGC -XX:+PrintGCDateStamps -XX:GCLogFileSize=10M
+jvm.args=-Xloggc:/tmp/asterix-server-failsafe/asterix_nc2-%p-gc.log 
-XX:+PrintGC -XX:+PrintGCDateStamps -XX:GCLogFileSize=10M -XX:+PrintFlagsFinal
 #jvm.args=-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=5007
 
 [nc]

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/bb29bc7c/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/file/FileUtil.java
----------------------------------------------------------------------
diff --git 
a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/file/FileUtil.java
 
b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/file/FileUtil.java
index 1b0093d..443640a 100644
--- 
a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/file/FileUtil.java
+++ 
b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/file/FileUtil.java
@@ -30,6 +30,7 @@ public class FileUtil {
 
     private static final Logger LOGGER = LogManager.getLogger();
     private static final Object LOCK = new Object();
+    private static final int MAX_COPY_ATTEMPTS = 3;
 
     private FileUtil() {
     }
@@ -63,4 +64,26 @@ public class FileUtil {
             return joined.replaceAll("(" + escapedSeparator + ")+", 
"$1").replaceAll(escapedSeparator + "$", "");
         }
     }
+
+    public static void safeCopyFile(File child, File destChild) throws 
IOException {
+        forceMkdirs(destChild.getParentFile());
+        IOException ioException = null;
+        while (true) {
+            try {
+                FileUtils.copyFile(child, destChild);
+                return;
+            } catch (IOException e) {
+                if (ioException == null) {
+                    ioException = e;
+                } else {
+                    ioException.addSuppressed(e);
+                }
+                if (ioException.getSuppressed().length >= MAX_COPY_ATTEMPTS) {
+                    LOGGER.warn("Unable to copy {} to {} after " + 
MAX_COPY_ATTEMPTS + " attempts; skipping file",
+                            child, destChild, e);
+                    return;
+                }
+            }
+        }
+    }
 }

Reply via email to