[ASTERIXDB-2261][TEST] Logging Misc Fixes - fix ASTERIXDB-2261 by preventing *DB and Hyracks test log config files from being discoverable on the test classpath - align test class packages
Change-Id: I9f6145849d87a1ed702b955723bc1dd53e607473 Reviewed-on: https://asterix-gerrit.ics.uci.edu/2317 Integration-Tests: Jenkins <[email protected]> Tested-by: Jenkins <[email protected]> Reviewed-by: Murtadha Hubail <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/2530e39d Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/2530e39d Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/2530e39d Branch: refs/heads/master Commit: 2530e39d7d946a6d57ce33ebc995b9be4e51721d Parents: 1a93695 Author: Michael Blow <[email protected]> Authored: Tue Feb 6 01:19:42 2018 -0500 Committer: Michael Blow <[email protected]> Committed: Tue Feb 6 10:04:06 2018 -0800 ---------------------------------------------------------------------- .../test/resources/log4j2-asterixdb-test.xml | 45 ++++ .../src/test/resources/log4j2-test.xml | 45 ---- asterixdb/asterix-common/pom.xml | 2 +- asterixdb/asterix-server/pom.xml | 1 + .../server/test/AbstractExecutionIT.java | 163 ------------ .../apache/asterix/server/test/LicensingIT.java | 94 ------- .../server/test/NCServiceExecutionIT.java | 266 ------------------- .../asterix/server/test/NcLifecycleIT.java | 123 --------- .../apache/asterix/server/test/RecoveryIT.java | 136 ---------- .../asterix/server/test/ReplicationIT.java | 135 ---------- .../server/test/SampleLocalClusterIT.java | 140 ---------- .../asterix/server/test/SqlppExecutionIT.java | 54 ---- .../test/server/AbstractExecutionIT.java | 163 ++++++++++++ .../apache/asterix/test/server/LicensingIT.java | 94 +++++++ .../test/server/NCServiceExecutionIT.java | 266 +++++++++++++++++++ .../asterix/test/server/NcLifecycleIT.java | 119 +++++++++ .../apache/asterix/test/server/RecoveryIT.java | 136 ++++++++++ .../asterix/test/server/ReplicationIT.java | 135 ++++++++++ .../test/server/SampleLocalClusterIT.java | 140 ++++++++++ .../asterix/test/server/SqlppExecutionIT.java | 54 ++++ asterixdb/pom.xml | 8 +- .../hyracks/hyracks-server/pom.xml | 11 + .../server/process/HyracksCCProcess.java | 47 ---- .../server/process/HyracksNCServiceProcess.java | 45 ---- .../server/process/HyracksServerProcess.java | 126 --------- .../server/process/HyracksVirtualCluster.java | 85 ------ .../apache/hyracks/server/test/NCServiceIT.java | 147 ---------- .../apache/hyracks/test/server/NCServiceIT.java | 148 +++++++++++ .../test/server/process/HyracksCCProcess.java | 47 ++++ .../server/process/HyracksNCServiceProcess.java | 45 ++++ .../server/process/HyracksServerProcess.java | 126 +++++++++ .../server/process/HyracksVirtualCluster.java | 85 ++++++ hyracks-fullstack/pom.xml | 2 +- .../src/test/resources/log4j2-hyracks-test.xml | 41 +++ .../src/test/resources/log4j2-test.xml | 41 --- 35 files changed, 1665 insertions(+), 1650 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/asterixdb/blob/2530e39d/asterixdb/asterix-app/src/test/resources/log4j2-asterixdb-test.xml ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/resources/log4j2-asterixdb-test.xml b/asterixdb/asterix-app/src/test/resources/log4j2-asterixdb-test.xml new file mode 100644 index 0000000..c6ecd7d --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/log4j2-asterixdb-test.xml @@ -0,0 +1,45 @@ +<!-- + ! Licensed to the Apache Software Foundation (ASF) under one + ! or more contributor license agreements. See the NOTICE file + ! distributed with this work for additional information + ! regarding copyright ownership. The ASF licenses this file + ! to you under the Apache License, Version 2.0 (the + ! "License"); you may not use this file except in complianceo + ! with the License. You may obtain a copy of the License at + ! + ! http://www.apache.org/licenses/LICENSE-2.0 + ! + ! Unless required by applicable law or agreed to in writing, + ! software distributed under the License is distributed on an + ! "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + ! KIND, either express or implied. See the License for the + ! specific language governing permissions and limitations + ! under the License. + !--> +<Configuration status="WARN"> + <Appenders> + <Console name="Console" target="SYSTEM_OUT"> + <PatternLayout pattern="%d{HH:mm:ss.SSS} [%t] %-5level %logger{36} - %msg%n"/> + </Console> + <File name="InfoLog" fileName="target/info.log"> + <PatternLayout pattern="%d{HH:mm:ss.SSS} [%t] %-5level %logger{36} - %msg%n"/> + </File> + <Console name="ConsoleTest" target="SYSTEM_OUT"> + <PatternLayout pattern="%d{HH:mm:ss.SSS} [%t] %c{1} - %msg%n"/> + </Console> + </Appenders> + <Loggers> + <Root level="WARN"> + <AppenderRef ref="InfoLog"/> + </Root> + <Logger name="org.apache.hyracks.control.nc.service" level="INFO"/> + <Logger name="org.apache.hyracks" level="INFO"/> + <Logger name="org.apache.asterix" level="INFO"/> + <Logger name="org.apache.hyracks.test" level="INFO"> + <AppenderRef ref="ConsoleTest"/> + </Logger> + <Logger name="org.apache.asterix.test" level="INFO"> + <AppenderRef ref="ConsoleTest"/> + </Logger> + </Loggers> +</Configuration> http://git-wip-us.apache.org/repos/asf/asterixdb/blob/2530e39d/asterixdb/asterix-app/src/test/resources/log4j2-test.xml ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/resources/log4j2-test.xml b/asterixdb/asterix-app/src/test/resources/log4j2-test.xml deleted file mode 100644 index c6ecd7d..0000000 --- a/asterixdb/asterix-app/src/test/resources/log4j2-test.xml +++ /dev/null @@ -1,45 +0,0 @@ -<!-- - ! Licensed to the Apache Software Foundation (ASF) under one - ! or more contributor license agreements. See the NOTICE file - ! distributed with this work for additional information - ! regarding copyright ownership. The ASF licenses this file - ! to you under the Apache License, Version 2.0 (the - ! "License"); you may not use this file except in complianceo - ! with the License. You may obtain a copy of the License at - ! - ! http://www.apache.org/licenses/LICENSE-2.0 - ! - ! Unless required by applicable law or agreed to in writing, - ! software distributed under the License is distributed on an - ! "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - ! KIND, either express or implied. See the License for the - ! specific language governing permissions and limitations - ! under the License. - !--> -<Configuration status="WARN"> - <Appenders> - <Console name="Console" target="SYSTEM_OUT"> - <PatternLayout pattern="%d{HH:mm:ss.SSS} [%t] %-5level %logger{36} - %msg%n"/> - </Console> - <File name="InfoLog" fileName="target/info.log"> - <PatternLayout pattern="%d{HH:mm:ss.SSS} [%t] %-5level %logger{36} - %msg%n"/> - </File> - <Console name="ConsoleTest" target="SYSTEM_OUT"> - <PatternLayout pattern="%d{HH:mm:ss.SSS} [%t] %c{1} - %msg%n"/> - </Console> - </Appenders> - <Loggers> - <Root level="WARN"> - <AppenderRef ref="InfoLog"/> - </Root> - <Logger name="org.apache.hyracks.control.nc.service" level="INFO"/> - <Logger name="org.apache.hyracks" level="INFO"/> - <Logger name="org.apache.asterix" level="INFO"/> - <Logger name="org.apache.hyracks.test" level="INFO"> - <AppenderRef ref="ConsoleTest"/> - </Logger> - <Logger name="org.apache.asterix.test" level="INFO"> - <AppenderRef ref="ConsoleTest"/> - </Logger> - </Loggers> -</Configuration> http://git-wip-us.apache.org/repos/asf/asterixdb/blob/2530e39d/asterixdb/asterix-common/pom.xml ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-common/pom.xml b/asterixdb/asterix-common/pom.xml index 78f75a8..9193a3d 100644 --- a/asterixdb/asterix-common/pom.xml +++ b/asterixdb/asterix-common/pom.xml @@ -169,7 +169,7 @@ <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-dependency-plugin</artifactId> - <configuration> + <configuration combine.children="append"> <ignoredUnusedDeclaredDependencies> <ignoredUnusedDeclaredDependency>org.apache.hyracks:hyracks-storage-am-lsm-btree:*</ignoredUnusedDeclaredDependency> <ignoredUnusedDeclaredDependency>org.apache.hyracks:hyracks-storage-am-lsm-rtree:*</ignoredUnusedDeclaredDependency> http://git-wip-us.apache.org/repos/asf/asterixdb/blob/2530e39d/asterixdb/asterix-server/pom.xml ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-server/pom.xml b/asterixdb/asterix-server/pom.xml index 6080d69..80f4197 100644 --- a/asterixdb/asterix-server/pom.xml +++ b/asterixdb/asterix-server/pom.xml @@ -514,6 +514,7 @@ <dependency> <groupId>org.apache.hyracks</groupId> <artifactId>hyracks-server</artifactId> + <type>test-jar</type> <scope>test</scope> </dependency> <dependency> http://git-wip-us.apache.org/repos/asf/asterixdb/blob/2530e39d/asterixdb/asterix-server/src/test/java/org/apache/asterix/server/test/AbstractExecutionIT.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-server/src/test/java/org/apache/asterix/server/test/AbstractExecutionIT.java b/asterixdb/asterix-server/src/test/java/org/apache/asterix/server/test/AbstractExecutionIT.java deleted file mode 100644 index 8890898..0000000 --- a/asterixdb/asterix-server/src/test/java/org/apache/asterix/server/test/AbstractExecutionIT.java +++ /dev/null @@ -1,163 +0,0 @@ -/* - * Copyright 2009-2013 by The Regents of the University of California - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * you may obtain a copy of the License from - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.asterix.server.test; - -import org.apache.asterix.external.util.ExternalDataConstants; -import org.apache.asterix.external.util.IdentitiyResolverFactory; -import org.apache.asterix.test.base.RetainLogsRule; -import org.apache.asterix.test.common.TestExecutor; -import org.apache.asterix.testframework.context.TestCaseContext; -import org.apache.asterix.testframework.context.TestFileContext; -import org.apache.asterix.testframework.xml.TestCase.CompilationUnit; -import org.apache.asterix.testframework.xml.TestGroup; -import org.apache.commons.lang3.StringUtils; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.codehaus.plexus.util.FileUtils; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TestRule; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; -import org.junit.runners.Parameterized.Parameters; - -import java.io.File; -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; - -import static org.apache.hyracks.util.file.FileUtil.joinPath; - -/** - * Runs the runtime test cases under 'asterix-app/src/test/resources/runtimets'. - */ -@RunWith(Parameterized.class) -public abstract class AbstractExecutionIT { - - protected static final Logger LOGGER = LogManager.getLogger(); - - protected static final String PATH_ACTUAL = joinPath("target", "ittest"); - protected static final String PATH_BASE = joinPath("..", "asterix-app", "src", "test", "resources", "runtimets"); - - protected static final String HDFS_BASE = "../asterix-app/"; - - protected static final TestExecutor testExecutor = new TestExecutor(); - - private static final String EXTERNAL_LIBRARY_TEST_GROUP = "lib"; - - private static final List<String> badTestCases = new ArrayList<>(); - - private static String reportPath = new File(joinPath("target", "failsafe-reports")).getAbsolutePath(); - - @Rule - public TestRule retainLogs = new RetainLogsRule(NCServiceExecutionIT.LOG_DIR, reportPath, this); - - @BeforeClass - public static void setUp() throws Exception { - System.out.println("Starting setup"); - if (LOGGER.isInfoEnabled()) { - LOGGER.info("Starting setup"); - } - File outdir = new File(PATH_ACTUAL); - outdir.mkdirs(); - - File externalTestsJar = - new File(StringUtils.join(new String[] { "..", "asterix-external-data", "target" }, File.separator)) - .listFiles((dir, name) -> name.matches("asterix-external-data-.*-tests.jar"))[0]; - - FileUtils.copyFile(externalTestsJar, - new File(NCServiceExecutionIT.APP_HOME + "/repo", externalTestsJar.getName())); - - NCServiceExecutionIT.setUp(); - - FileUtils.copyDirectoryStructure(new File(joinPath("..", "asterix-app", "data")), - new File(NCServiceExecutionIT.ASTERIX_APP_DIR + "/clusters/local/working_dir/data")); - - FileUtils.copyDirectoryStructure(new File(joinPath("..", "asterix-app", "target", "data")), - new File(NCServiceExecutionIT.ASTERIX_APP_DIR + "/clusters/local/working_dir/target/data")); - - // Set the node resolver to be the identity resolver that expects node names - // to be node controller ids; a valid assumption in test environment. - System.setProperty(ExternalDataConstants.NODE_RESOLVER_FACTORY_PROPERTY, - IdentitiyResolverFactory.class.getName()); - - reportPath = new File(joinPath("target", "failsafe-reports")).getAbsolutePath(); - } - - @AfterClass - public static void tearDown() throws Exception { - File outdir = new File(PATH_ACTUAL); - File[] files = outdir.listFiles(); - if ((files == null) || (files.length == 0)) { - outdir.delete(); - } - //AsterixLifecycleIT.tearDown(); - NCServiceExecutionIT.tearDown(); - if (!badTestCases.isEmpty()) { - System.out.println("The following test cases left some data"); - for (String testCase : badTestCases) { - System.out.println(testCase); - } - } - } - - @Parameters - public static Collection<Object[]> tests() throws Exception { - Collection<Object[]> testArgs = new ArrayList<Object[]>(); - TestCaseContext.Builder b = new TestCaseContext.Builder(); - for (TestCaseContext ctx : b.build(new File(PATH_BASE))) { - testArgs.add(new Object[] { ctx }); - } - return testArgs; - } - - private TestCaseContext tcCtx; - - public AbstractExecutionIT(TestCaseContext tcCtx) { - this.tcCtx = tcCtx; - } - - @Test - public void test() throws Exception { - if (skip()) { - return; - } - testExecutor.executeTest(PATH_ACTUAL, tcCtx, null, false); - testExecutor.cleanup(tcCtx.toString(), badTestCases); - } - - protected boolean skip() { - // If the test case contains library commands, we skip them - List<CompilationUnit> cUnits = tcCtx.getTestCase().getCompilationUnit(); - for (CompilationUnit cUnit : cUnits) { - List<TestFileContext> testFileCtxs = tcCtx.getTestFiles(cUnit); - for (TestFileContext ctx : testFileCtxs) { - if (ctx.getType().equals(EXTERNAL_LIBRARY_TEST_GROUP)) { - return true; - } - } - } - // For now we skip api tests. - for (TestGroup group : tcCtx.getTestGroups()) { - if (group != null && "api".equals(group.getName())) { - LOGGER.info("Skipping test: " + tcCtx.toString()); - return true; - } - } - return false; - } -} http://git-wip-us.apache.org/repos/asf/asterixdb/blob/2530e39d/asterixdb/asterix-server/src/test/java/org/apache/asterix/server/test/LicensingIT.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-server/src/test/java/org/apache/asterix/server/test/LicensingIT.java b/asterixdb/asterix-server/src/test/java/org/apache/asterix/server/test/LicensingIT.java deleted file mode 100644 index 6d4af8c..0000000 --- a/asterixdb/asterix-server/src/test/java/org/apache/asterix/server/test/LicensingIT.java +++ /dev/null @@ -1,94 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.asterix.server.test; - -import java.io.File; -import java.io.IOException; -import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; - -import org.apache.commons.io.FileUtils; -import org.apache.hyracks.util.file.FileUtil; -import org.junit.Assert; -import org.junit.Before; -import org.junit.FixMethodOrder; -import org.junit.Test; -import org.junit.runners.MethodSorters; - -@FixMethodOrder(MethodSorters.JVM) -public class LicensingIT { - - protected String installerDir; - - @Before - public void setup() { - final String pattern = getInstallerDirPattern(); - final String targetDir = getTargetDir(); - final String[] list = new File(targetDir).list((dir, name) -> name.matches(pattern)); - Assert.assertNotNull("installerDir", list); - Assert.assertFalse("Ambiguous install dir (" + pattern + "): " + Arrays.toString(list), list.length > 1); - Assert.assertEquals("Can't find install dir (" + pattern + ")", 1, list.length); - installerDir = FileUtil.joinPath(targetDir, list[0]); - } - - protected String getTargetDir() { - return FileUtil.joinPath("target"); - } - - protected String getInstallerDirPattern() { - return "asterix-server.*-binary-assembly"; - } - - protected String pathToLicensingFiles() { - return ""; - } - - @Test - public void testLicenseNoticeFilesPresent() throws IOException { - for (String name : getRequiredArtifactNames()) { - final String fileName = FileUtil.joinPath(installerDir, pathToLicensingFiles(), name); - Assert.assertTrue(fileName + " missing", new File(fileName).exists()); - } - } - - protected String[] getRequiredArtifactNames() { - return org.apache.commons.lang3.ArrayUtils.add(getLicenseArtifactNames(), "NOTICE"); - } - - @Test - public void ensureNoMissingLicenses() throws IOException { - for (String licenseArtifactName : getLicenseArtifactNames()) { - final File licenseFile = - new File(FileUtil.joinPath(installerDir, pathToLicensingFiles(), licenseArtifactName)); - List<String> badLines = new ArrayList<>(); - for (String line : FileUtils.readLines(licenseFile, StandardCharsets.UTF_8)) { - if (line.matches("^\\s*MISSING:.*")) { - badLines.add(line.trim()); - } - } - Assert.assertEquals("Missing licenses in " + licenseFile + ": " + badLines, 0, badLines.size()); - } - } - - protected String[] getLicenseArtifactNames() { - return new String[] { "LICENSE" }; - } -} http://git-wip-us.apache.org/repos/asf/asterixdb/blob/2530e39d/asterixdb/asterix-server/src/test/java/org/apache/asterix/server/test/NCServiceExecutionIT.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-server/src/test/java/org/apache/asterix/server/test/NCServiceExecutionIT.java b/asterixdb/asterix-server/src/test/java/org/apache/asterix/server/test/NCServiceExecutionIT.java deleted file mode 100644 index d7eea9a..0000000 --- a/asterixdb/asterix-server/src/test/java/org/apache/asterix/server/test/NCServiceExecutionIT.java +++ /dev/null @@ -1,266 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.asterix.server.test; - -import java.io.File; -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; -import java.util.Random; -import java.util.concurrent.TimeUnit; - -import org.apache.asterix.test.common.TestExecutor; -import org.apache.asterix.test.runtime.HDFSCluster; -import org.apache.asterix.testframework.context.TestCaseContext; -import org.apache.asterix.testframework.xml.TestGroup; -import org.apache.commons.io.FileUtils; -import org.apache.commons.lang3.StringUtils; -import org.apache.hyracks.server.process.HyracksCCProcess; -import org.apache.hyracks.server.process.HyracksNCServiceProcess; -import org.apache.hyracks.server.process.HyracksVirtualCluster; -import org.apache.logging.log4j.Level; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.junit.AfterClass; -import org.junit.Assert; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.FixMethodOrder; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.MethodSorters; -import org.junit.runners.Parameterized; -import org.junit.runners.Parameterized.Parameters; - -@FixMethodOrder(MethodSorters.NAME_ASCENDING) -@RunWith(Parameterized.class) -public class NCServiceExecutionIT { - - // Important paths and files for this test. - - // The "target" subdirectory of asterix-server. All outputs go here. - public static final String TARGET_DIR = - StringUtils.join(new String[] { "../asterix-server/target" }, File.separator); - - // Directory where the NCs create and store all data, as configured by - // src/test/resources/NCServiceExecutionIT/cc.conf. - public static final String INSTANCE_DIR = StringUtils.join(new String[] { TARGET_DIR, "tmp" }, File.separator); - - // The log directory, where all CC, NCService, and NC logs are written. CC and - // NCService logs are configured on the HyracksVirtualCluster below. NC logs - // are configured in src/test/resources/NCServiceExecutionIT/ncservice*.conf. - public static final String LOG_DIR = - StringUtils.join(new String[] { TARGET_DIR, "failsafe-reports" }, File.separator); - - // Directory where *.conf files are located. - public static final String CONF_DIR = - StringUtils.join(new String[] { TARGET_DIR, "test-classes", "NCServiceExecutionIT" }, File.separator); - - // The app.home specified for HyracksVirtualCluster. The NCService expects - // to find the NC startup script in ${app.home}/bin. - public static final String APP_HOME = StringUtils.join(new String[] { TARGET_DIR, "appassembler" }, File.separator); - - // Path to the asterix-app directory. This is used as the current working - // directory for the CC and NCService processes, which allows relative file - // paths in "load" statements in test queries to find the right data. It is - // also used for HDFSCluster. - public static final String ASTERIX_APP_DIR = StringUtils.join(new String[] { "..", "asterix-app" }, File.separator); - - // Path to the actual AQL test files, which we borrow from asterix-app. This is - // passed to TestExecutor. - protected static final String TESTS_DIR = - StringUtils.join(new String[] { ASTERIX_APP_DIR, "src", "test", "resources", "runtimets" }, File.separator); - - // Path that actual results are written to. We create and clean this directory - // here, and also pass it to TestExecutor which writes the test output there. - public static final String ACTUAL_RESULTS_DIR = - StringUtils.join(new String[] { TARGET_DIR, "ittest" }, File.separator); - - private static final Logger LOGGER = LogManager.getLogger(); - - enum KillCommand { - CC, - NC1, - NC2; - - @Override - public String toString() { - return "<kill " + name().toLowerCase() + ">"; - } - } - - private static HyracksCCProcess cc; - private static HyracksNCServiceProcess nc1; - private static HyracksNCServiceProcess nc2; - - private final TestCaseContext tcCtx; - private static final TestExecutor testExecutor = new TestExecutor(); - - private static final List<String> badTestCases = new ArrayList<>(); - private static HyracksVirtualCluster cluster; - private final KillCommand killType; - private static boolean clusterActive = false; - - @BeforeClass - public static void setUp() throws Exception { - // Create actual-results output directory. - File outDir = new File(ACTUAL_RESULTS_DIR); - outDir.mkdirs(); - - // Remove any instance data from previous runs. - File instanceDir = new File(INSTANCE_DIR); - if (instanceDir.isDirectory()) { - FileUtils.deleteDirectory(instanceDir); - } - - // HDFSCluster requires the input directory to end with a file separator. - HDFSCluster.getInstance().setup(ASTERIX_APP_DIR + File.separator); - - cluster = new HyracksVirtualCluster(new File(APP_HOME), new File(ASTERIX_APP_DIR)); - nc1 = cluster.addNCService(new File(CONF_DIR, "ncservice1.conf"), new File(LOG_DIR, "ncservice1.log")); - - nc2 = cluster.addNCService(new File(CONF_DIR, "ncservice2.conf"), new File(LOG_DIR, "ncservice2.log")); - - // Start CC - cc = cluster.start(new File(CONF_DIR, "cc.conf"), new File(LOG_DIR, "cc.log")); - - testExecutor.waitForClusterActive(30, TimeUnit.SECONDS); - clusterActive = true; - } - - @AfterClass - public static void tearDown() throws Exception { - File outdir = new File(ACTUAL_RESULTS_DIR); - File[] files = outdir.listFiles(); - if (files == null || files.length == 0) { - outdir.delete(); - } - cluster.stop(); - HDFSCluster.getInstance().cleanup(); - if (!badTestCases.isEmpty()) { - System.out.println("The following test cases left some data"); - for (String testCase : badTestCases) { - System.out.println(testCase); - } - } - } - - @Parameters(name = "NCServiceExecutionTest {index}: {0}") - public static Collection<Object[]> tests() throws Exception { - Collection<Object[]> testArgs = new ArrayList<>(); - Random random = getRandom(); - TestCaseContext.Builder b = new TestCaseContext.Builder(); - for (TestCaseContext ctx : b.build(new File(TESTS_DIR))) { - if (!skip(ctx)) { - testArgs.add(new Object[] { ctx, ctx, null }); - } - // let's kill something every 50 tests - if (testArgs.size() % 50 == 0) { - final KillCommand killCommand = KillCommand.values()[random.nextInt(KillCommand.values().length)]; - testArgs.add(new Object[] { killCommand, null, killCommand }); - } - } - return testArgs; - } - - private static Random getRandom() { - Random random; - if (System.getProperty("random.seed") == null) { - random = new Random() { - @Override - public synchronized void setSeed(long seed) { - super.setSeed(seed); - System.err.println("using generated seed: " + seed + "; use -Drandom.seed to use specific seed"); - } - }; - } else { - final long seed = Long.getLong("random.seed"); - System.err.println("using provided seed (-Drandom.seed): " + seed); - random = new Random(seed); - } - return random; - } - - private static boolean skip(TestCaseContext tcCtx) { - // For now we skip feeds tests, external-library, and api tests. - for (TestGroup group : tcCtx.getTestGroups()) { - if (group.getName().startsWith("external-") || group.getName().equals("feeds") - || group.getName().equals("api")) { - LOGGER.info("Skipping test: " + tcCtx.toString()); - return true; - } - } - return false; - } - - public NCServiceExecutionIT(Object description, TestCaseContext tcCtx, KillCommand killType) { - this.tcCtx = tcCtx; - this.killType = killType; - } - - @Before - public void before() { - if (!clusterActive) { - Assert.fail("Skipping test since cluster is not ACTIVE"); - } - } - - @Test - public void test() throws Exception { - if (tcCtx != null) { - testExecutor.executeTest(ACTUAL_RESULTS_DIR, tcCtx, null, false); - testExecutor.cleanup(tcCtx.toString(), badTestCases); - } else { - switch (killType) { - case CC: - LOGGER.info("Killing CC..."); - cc.stop(true); - cc.start(); - break; - - case NC1: - LOGGER.info("Killing NC1..."); - nc1.stop(); // we can't kill due to ASTERIXDB-1941 - testExecutor.waitForClusterState("UNUSABLE", 60, TimeUnit.SECONDS); // wait for missed heartbeats... - nc1.start(); // this restarts the NC service - testExecutor.startNC("asterix_nc1"); - break; - - case NC2: - LOGGER.info("Killing NC2..."); - nc2.stop(); // we can't kill due to ASTERIXDB-1941 - testExecutor.waitForClusterState("UNUSABLE", 60, TimeUnit.SECONDS); // wait for missed heartbeats... - nc2.start(); // this restarts the NC service - testExecutor.startNC("asterix_nc2"); - break; - - default: - Assert.fail("killType: " + killType); - } - try { - testExecutor.waitForClusterActive(30, TimeUnit.SECONDS); - } catch (Exception e) { - // stop executing the rest of the tests since the cluster is not ACTIVE - LOGGER.log(Level.ERROR, "Cannot continue since cluster is not ACTIVE", e); - clusterActive = false; - Assert.fail("Cluster is not ACTIVE"); - } - } - } -} http://git-wip-us.apache.org/repos/asf/asterixdb/blob/2530e39d/asterixdb/asterix-server/src/test/java/org/apache/asterix/server/test/NcLifecycleIT.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-server/src/test/java/org/apache/asterix/server/test/NcLifecycleIT.java b/asterixdb/asterix-server/src/test/java/org/apache/asterix/server/test/NcLifecycleIT.java deleted file mode 100644 index 9e3a029..0000000 --- a/asterixdb/asterix-server/src/test/java/org/apache/asterix/server/test/NcLifecycleIT.java +++ /dev/null @@ -1,123 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.asterix.server.test; - -import static org.apache.asterix.server.test.NCServiceExecutionIT.APP_HOME; -import static org.apache.asterix.server.test.NCServiceExecutionIT.ASTERIX_APP_DIR; -import static org.apache.asterix.server.test.NCServiceExecutionIT.INSTANCE_DIR; -import static org.apache.asterix.server.test.NCServiceExecutionIT.LOG_DIR; -import static org.apache.asterix.server.test.NCServiceExecutionIT.TARGET_DIR; - -import java.io.File; -import java.net.InetAddress; -import java.net.InetSocketAddress; -import java.util.ArrayList; -import java.util.Collection; -import java.util.HashMap; -import java.util.Map; -import java.util.concurrent.TimeUnit; - -import org.apache.asterix.test.base.RetainLogsRule; -import org.apache.asterix.test.common.TestExecutor; -import org.apache.asterix.testframework.context.TestCaseContext; -import org.apache.commons.io.FileUtils; -import org.apache.commons.lang3.StringUtils; -import org.apache.hyracks.server.process.HyracksVirtualCluster; -import org.apache.hyracks.util.file.FileUtil; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.junit.After; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TestRule; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; - -@RunWith(Parameterized.class) -public class NcLifecycleIT { - - private static final String PATH_BASE = - FileUtil.joinPath("src", "test", "resources", "integrationts", "NcLifecycle"); - private static final String CONF_DIR = - StringUtils.join(new String[] { TARGET_DIR, "test-classes", "NcLifecycleIT" }, File.separator); - private static final String PATH_ACTUAL = FileUtil.joinPath("target", "ittest"); - private static final Logger LOGGER = LogManager.getLogger(); - private static String reportPath = new File(FileUtil.joinPath("target", "failsafe-reports")).getAbsolutePath(); - private static final TestExecutor testExecutor = new TestExecutor(); - private static HyracksVirtualCluster cluster; - - private TestCaseContext tcCtx; - - public NcLifecycleIT(TestCaseContext tcCtx) { - this.tcCtx = tcCtx; - } - - @Rule - public TestRule retainLogs = new RetainLogsRule(NCServiceExecutionIT.ASTERIX_APP_DIR, reportPath, this); - - @Before - public void before() throws Exception { - LOGGER.info("Creating new instance..."); - File instanceDir = new File(INSTANCE_DIR); - if (instanceDir.isDirectory()) { - FileUtils.deleteDirectory(instanceDir); - } - - cluster = new HyracksVirtualCluster(new File(APP_HOME), new File(ASTERIX_APP_DIR)); - cluster.addNCService(new File(CONF_DIR, "ncservice1.conf"), new File(LOG_DIR, "ncservice1.log")); - cluster.addNCService(new File(CONF_DIR, "ncservice2.conf"), new File(LOG_DIR, "ncservice2.log")); - - // Start CC - cluster.start(new File(CONF_DIR, "cc.conf"), new File(LOG_DIR, "cc.log")); - LOGGER.info("Instance created."); - testExecutor.waitForClusterActive(30, TimeUnit.SECONDS); - LOGGER.info("Instance is in ACTIVE state."); - } - - @After - public void after() { - LOGGER.info("Destroying instance..."); - cluster.stop(); - LOGGER.info("Instance destroyed."); - } - - @Test - public void test() throws Exception { - testExecutor.executeTest(PATH_ACTUAL, tcCtx, null, false); - } - - @Parameterized.Parameters(name = "NcLifecycleIT {index}: {0}") - public static Collection<Object[]> tests() throws Exception { - Collection<Object[]> testArgs = buildTestsInXml(TestCaseContext.DEFAULT_TESTSUITE_XML_NAME); - if (testArgs.size() == 0) { - testArgs = buildTestsInXml(TestCaseContext.DEFAULT_TESTSUITE_XML_NAME); - } - return testArgs; - } - - private static Collection<Object[]> buildTestsInXml(String xmlfile) throws Exception { - Collection<Object[]> testArgs = new ArrayList<>(); - TestCaseContext.Builder b = new TestCaseContext.Builder(); - for (TestCaseContext ctx : b.build(new File(PATH_BASE), xmlfile)) { - testArgs.add(new Object[] { ctx }); - } - return testArgs; - } -} http://git-wip-us.apache.org/repos/asf/asterixdb/blob/2530e39d/asterixdb/asterix-server/src/test/java/org/apache/asterix/server/test/RecoveryIT.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-server/src/test/java/org/apache/asterix/server/test/RecoveryIT.java b/asterixdb/asterix-server/src/test/java/org/apache/asterix/server/test/RecoveryIT.java deleted file mode 100644 index 163e279..0000000 --- a/asterixdb/asterix-server/src/test/java/org/apache/asterix/server/test/RecoveryIT.java +++ /dev/null @@ -1,136 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.asterix.server.test; - -import java.io.File; -import java.io.FilenameFilter; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Map; - -import org.apache.asterix.test.base.RetainLogsRule; -import org.apache.asterix.test.common.TestExecutor; -import org.apache.asterix.test.runtime.HDFSCluster; -import org.apache.asterix.testframework.context.TestCaseContext; -import org.apache.commons.io.FileUtils; -import org.apache.commons.lang3.StringUtils; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TestRule; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; -import org.junit.runners.Parameterized.Parameters; - -@RunWith(Parameterized.class) -public class RecoveryIT { - - private static final Logger LOGGER = LogManager.getLogger(); - private static final String PATH_ACTUAL = "target" + File.separator + "rttest" + File.separator; - private static final String PATH_BASE = "src/test/resources/transactionts/"; - private static final String HDFS_BASE = "../asterix-app/"; - private TestCaseContext tcCtx; - private static File asterixInstallerPath; - private static File installerTargetPath; - private static String ncServiceHomeDirName; - private static String ncServiceHomePath; - private static String scriptHomePath; - private static String reportPath; - private static ProcessBuilder pb; - private static Map<String, String> env; - private final TestExecutor testExecutor = new TestExecutor(); - - @Rule - public TestRule retainLogs = new RetainLogsRule(ncServiceHomePath, reportPath, this); - - @BeforeClass - public static void setUp() throws Exception { - File outdir = new File(PATH_ACTUAL); - outdir.mkdirs(); - - File externalTestsJar = - new File(StringUtils.join(new String[] { "..", "asterix-external-data", "target" }, File.separator)) - .listFiles((dir, name) -> name.matches("asterix-external-data-.*-tests.jar"))[0]; - - asterixInstallerPath = new File(System.getProperty("user.dir")); - installerTargetPath = new File(new File(asterixInstallerPath.getParentFile(), "asterix-server"), "target"); - reportPath = new File(installerTargetPath, "failsafe-reports").getAbsolutePath(); - ncServiceHomeDirName = installerTargetPath.list(new FilenameFilter() { - @Override - public boolean accept(File dir, String name) { - return new File(dir, name).isDirectory() && name.startsWith("asterix-server") - && name.endsWith("binary-assembly"); - } - })[0]; - ncServiceHomePath = new File(installerTargetPath, ncServiceHomeDirName).getAbsolutePath(); - - LOGGER.info("NCSERVICE_HOME=" + ncServiceHomePath); - - FileUtils.copyFile(externalTestsJar, new File(ncServiceHomePath + "/repo", externalTestsJar.getName())); - - pb = new ProcessBuilder(); - env = pb.environment(); - env.put("NCSERVICE_HOME", ncServiceHomePath); - env.put("JAVA_HOME", System.getProperty("java.home")); - scriptHomePath = asterixInstallerPath + File.separator + "src" + File.separator + "test" + File.separator - + "resources" + File.separator + "transactionts" + File.separator + "scripts"; - env.put("SCRIPT_HOME", scriptHomePath); - - TestExecutor.executeScript(pb, - scriptHomePath + File.separator + "setup_teardown" + File.separator + "configure_and_validate.sh"); - TestExecutor.executeScript(pb, - scriptHomePath + File.separator + "setup_teardown" + File.separator + "stop_and_delete.sh"); - HDFSCluster.getInstance().setup(HDFS_BASE); - } - - @AfterClass - public static void tearDown() throws Exception { - File outdir = new File(PATH_ACTUAL); - FileUtils.deleteDirectory(outdir); - File dataCopyDir = - new File(ncServiceHomePath + File.separator + ".." + File.separator + ".." + File.separator + "data"); - FileUtils.deleteDirectory(dataCopyDir); - TestExecutor.executeScript(pb, - scriptHomePath + File.separator + "setup_teardown" + File.separator + "stop_and_delete.sh"); - HDFSCluster.getInstance().cleanup(); - } - - @Parameters(name = "RecoveryIT {index}: {0}") - public static Collection<Object[]> tests() throws Exception { - Collection<Object[]> testArgs = new ArrayList<Object[]>(); - TestCaseContext.Builder b = new TestCaseContext.Builder(); - for (TestCaseContext ctx : b.build(new File(PATH_BASE))) { - testArgs.add(new Object[] { ctx }); - } - return testArgs; - } - - public RecoveryIT(TestCaseContext tcCtx) { - this.tcCtx = tcCtx; - } - - @Test - public void test() throws Exception { - testExecutor.executeTest(PATH_ACTUAL, tcCtx, pb, false); - } - -} http://git-wip-us.apache.org/repos/asf/asterixdb/blob/2530e39d/asterixdb/asterix-server/src/test/java/org/apache/asterix/server/test/ReplicationIT.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-server/src/test/java/org/apache/asterix/server/test/ReplicationIT.java b/asterixdb/asterix-server/src/test/java/org/apache/asterix/server/test/ReplicationIT.java deleted file mode 100644 index ad1078d..0000000 --- a/asterixdb/asterix-server/src/test/java/org/apache/asterix/server/test/ReplicationIT.java +++ /dev/null @@ -1,135 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.asterix.server.test; - -import static org.apache.asterix.server.test.NCServiceExecutionIT.APP_HOME; -import static org.apache.asterix.server.test.NCServiceExecutionIT.ASTERIX_APP_DIR; -import static org.apache.asterix.server.test.NCServiceExecutionIT.INSTANCE_DIR; -import static org.apache.asterix.server.test.NCServiceExecutionIT.LOG_DIR; -import static org.apache.asterix.server.test.NCServiceExecutionIT.TARGET_DIR; - -import java.io.File; -import java.net.InetAddress; -import java.net.InetSocketAddress; -import java.util.ArrayList; -import java.util.Collection; -import java.util.HashMap; -import java.util.Map; -import java.util.concurrent.TimeUnit; - -import org.apache.asterix.test.base.RetainLogsRule; -import org.apache.asterix.test.common.TestExecutor; -import org.apache.asterix.testframework.context.TestCaseContext; -import org.apache.commons.io.FileUtils; -import org.apache.commons.lang3.StringUtils; -import org.apache.hyracks.server.process.HyracksVirtualCluster; -import org.apache.hyracks.util.file.FileUtil; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.junit.After; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TestRule; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; - -@RunWith(Parameterized.class) -public class ReplicationIT { - - private static final String PATH_BASE = - FileUtil.joinPath("src", "test", "resources", "integrationts", "replication"); - private static final String CONF_DIR = - StringUtils.join(new String[] { TARGET_DIR, "test-classes", "ReplicationIT" }, File.separator); - private static final String PATH_ACTUAL = FileUtil.joinPath("target", "ittest"); - private static final Logger LOGGER = LogManager.getLogger(); - private static String reportPath = new File(FileUtil.joinPath("target", "failsafe-reports")).getAbsolutePath(); - private static final TestExecutor testExecutor = new TestExecutor(); - private static HyracksVirtualCluster cluster; - - static { - final Map<String, InetSocketAddress> ncEndPoints = new HashMap<>(); - final Map<String, InetSocketAddress> replicationAddress = new HashMap<>(); - final String ip = InetAddress.getLoopbackAddress().getHostAddress(); - ncEndPoints.put("asterix_nc1", InetSocketAddress.createUnresolved(ip, 19004)); - ncEndPoints.put("asterix_nc2", InetSocketAddress.createUnresolved(ip, 19005)); - replicationAddress.put("asterix_nc1", InetSocketAddress.createUnresolved(ip, 2001)); - replicationAddress.put("asterix_nc2", InetSocketAddress.createUnresolved(ip, 2002)); - testExecutor.setNcEndPoints(ncEndPoints); - testExecutor.setNcReplicationAddress(replicationAddress); - } - - private TestCaseContext tcCtx; - - public ReplicationIT(TestCaseContext tcCtx) { - this.tcCtx = tcCtx; - } - - @Rule - public TestRule retainLogs = new RetainLogsRule(NCServiceExecutionIT.ASTERIX_APP_DIR, reportPath, this); - - @Before - public void before() throws Exception { - LOGGER.info("Creating new instance..."); - File instanceDir = new File(INSTANCE_DIR); - if (instanceDir.isDirectory()) { - FileUtils.deleteDirectory(instanceDir); - } - - cluster = new HyracksVirtualCluster(new File(APP_HOME), new File(ASTERIX_APP_DIR)); - cluster.addNCService(new File(CONF_DIR, "ncservice1.conf"), new File(LOG_DIR, "ncservice1.log")); - cluster.addNCService(new File(CONF_DIR, "ncservice2.conf"), new File(LOG_DIR, "ncservice2.log")); - - // Start CC - cluster.start(new File(CONF_DIR, "cc.conf"), new File(LOG_DIR, "cc.log")); - LOGGER.info("Instance created."); - testExecutor.waitForClusterActive(30, TimeUnit.SECONDS); - LOGGER.info("Instance is in ACTIVE state."); - } - - @After - public void after() { - LOGGER.info("Destroying instance..."); - cluster.stop(); - LOGGER.info("Instance destroyed."); - } - - @Test - public void test() throws Exception { - testExecutor.executeTest(PATH_ACTUAL, tcCtx, null, false); - } - - @Parameterized.Parameters(name = "ReplicationIT {index}: {0}") - public static Collection<Object[]> tests() throws Exception { - Collection<Object[]> testArgs = buildTestsInXml(TestCaseContext.DEFAULT_TESTSUITE_XML_NAME); - if (testArgs.size() == 0) { - testArgs = buildTestsInXml(TestCaseContext.DEFAULT_TESTSUITE_XML_NAME); - } - return testArgs; - } - - private static Collection<Object[]> buildTestsInXml(String xmlfile) throws Exception { - Collection<Object[]> testArgs = new ArrayList<>(); - TestCaseContext.Builder b = new TestCaseContext.Builder(); - for (TestCaseContext ctx : b.build(new File(PATH_BASE), xmlfile)) { - testArgs.add(new Object[] { ctx }); - } - return testArgs; - } -} http://git-wip-us.apache.org/repos/asf/asterixdb/blob/2530e39d/asterixdb/asterix-server/src/test/java/org/apache/asterix/server/test/SampleLocalClusterIT.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-server/src/test/java/org/apache/asterix/server/test/SampleLocalClusterIT.java b/asterixdb/asterix-server/src/test/java/org/apache/asterix/server/test/SampleLocalClusterIT.java deleted file mode 100644 index 190cead..0000000 --- a/asterixdb/asterix-server/src/test/java/org/apache/asterix/server/test/SampleLocalClusterIT.java +++ /dev/null @@ -1,140 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.asterix.server.test; - -import static org.apache.hyracks.util.file.FileUtil.joinPath; - -import java.io.File; -import java.io.IOException; -import java.io.InputStream; -import java.io.StringWriter; -import java.net.URI; -import java.net.URL; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; - -import org.apache.asterix.common.utils.Servlets; -import org.apache.asterix.test.base.TestMethodTracer; -import org.apache.asterix.test.common.TestExecutor; -import org.apache.asterix.test.common.TestHelper; -import org.apache.asterix.testframework.context.TestCaseContext.OutputFormat; -import org.apache.commons.io.FileUtils; -import org.apache.commons.io.IOUtils; -import org.junit.AfterClass; -import org.junit.Assert; -import org.junit.BeforeClass; -import org.junit.FixMethodOrder; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TestRule; -import org.junit.runners.MethodSorters; - -@FixMethodOrder(MethodSorters.NAME_ASCENDING) -public class SampleLocalClusterIT { - - // Important paths and files for this test. - - // The "target" subdirectory of asterix-server. All outputs go here. - private static final String TARGET_DIR = joinPath("target"); - - // Directory where the NCs create and store all data, as configured by - // src/test/resources/NCServiceExecutionIT/cc.conf. - private static final String OUTPUT_DIR = joinPath(TARGET_DIR, "sample local cluster"); - - private static final String LOCAL_SAMPLES_DIR = joinPath(OUTPUT_DIR, "opt", "local"); - - @Rule - public TestRule watcher = new TestMethodTracer(); - - @BeforeClass - public static void setUp() throws Exception { - // Create actual-results output directory. - File outDir = new File(OUTPUT_DIR); - - // Remove any instance data from previous runs. - if (outDir.isDirectory()) { - FileUtils.deleteDirectory(outDir); - } - outDir.mkdirs(); - - String[] pathElements = new String[] { TARGET_DIR, - new File(TARGET_DIR).list((dir, name) -> name.matches("asterix-server.*-binary-assembly.zip"))[0] }; - String installerZip = joinPath(pathElements); - - TestHelper.unzip(installerZip, OUTPUT_DIR); - - } - - private static List<File> findLogFiles(File directory, List<File> fileList) { - File[] match = directory.listFiles(pathname -> pathname.isDirectory() || pathname.toString().endsWith(".log")); - if (match != null) { - for (File file : match) { - if (file.isDirectory()) { - findLogFiles(file, fileList); - } else { - fileList.add(file); - } - } - } - return fileList; - } - - @AfterClass - public static void teardown() throws Exception { - - File destDir = new File(TARGET_DIR, joinPath("failsafe-reports", SampleLocalClusterIT.class.getSimpleName())); - - for (File f : findLogFiles(new File(OUTPUT_DIR), new ArrayList<>())) { - FileUtils.copyFileToDirectory(f, destDir); - } - } - - @Test - public void test0_startCluster() throws Exception { - Process process = - new ProcessBuilder(joinPath(LOCAL_SAMPLES_DIR, "bin/stop-sample-cluster.sh"), "-f").inheritIO().start(); - Assert.assertEquals(0, process.waitFor()); - process = new ProcessBuilder(joinPath(LOCAL_SAMPLES_DIR, "bin/start-sample-cluster.sh")).inheritIO().start(); - Assert.assertEquals(0, process.waitFor()); - } - - @Test - public void test1_sanityQuery() throws Exception { - TestExecutor testExecutor = new TestExecutor(); - InputStream resultStream = testExecutor.executeQuery("1+1", OutputFormat.ADM, - new URI("http", null, "127.0.0.1", 19002, Servlets.AQL_QUERY, null, null), Collections.emptyList()); - StringWriter sw = new StringWriter(); - IOUtils.copy(resultStream, sw); - Assert.assertEquals("2", sw.toString().trim()); - } - - @Test - public void test2_stopCluster() throws Exception { - Process process = - new ProcessBuilder(joinPath(LOCAL_SAMPLES_DIR, "bin/stop-sample-cluster.sh")).inheritIO().start(); - Assert.assertEquals(0, process.waitFor()); - try { - new URL("http://127.0.0.1:19002";).openConnection().connect(); - Assert.assertTrue("Expected connection to be refused.", false); - } catch (IOException e) { - // expected - } - } -} http://git-wip-us.apache.org/repos/asf/asterixdb/blob/2530e39d/asterixdb/asterix-server/src/test/java/org/apache/asterix/server/test/SqlppExecutionIT.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-server/src/test/java/org/apache/asterix/server/test/SqlppExecutionIT.java b/asterixdb/asterix-server/src/test/java/org/apache/asterix/server/test/SqlppExecutionIT.java deleted file mode 100644 index 066ddd9..0000000 --- a/asterixdb/asterix-server/src/test/java/org/apache/asterix/server/test/SqlppExecutionIT.java +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Copyright 2009-2013 by The Regents of the University of California - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * you may obtain a copy of the License from - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.asterix.server.test; - -import java.io.File; -import java.util.ArrayList; -import java.util.Collection; - -import org.apache.asterix.testframework.context.TestCaseContext; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; -import org.junit.runners.Parameterized.Parameters; - -/** - * Runs the runtime test cases under 'asterix-app/src/test/resources/runtimets'. - */ -@RunWith(Parameterized.class) -public class SqlppExecutionIT extends AbstractExecutionIT { - - @Parameters(name = "SqlppExecutionIT {index}: {0}") - public static Collection<Object[]> tests() throws Exception { - Collection<Object[]> testArgs = buildTestsInXml("only_sqlpp.xml"); - if (testArgs.size() == 0) { - testArgs = buildTestsInXml("testsuite_sqlpp.xml"); - } - return testArgs; - } - - protected static Collection<Object[]> buildTestsInXml(String xmlfile) throws Exception { - Collection<Object[]> testArgs = new ArrayList<Object[]>(); - TestCaseContext.Builder b = new TestCaseContext.Builder(); - for (TestCaseContext ctx : b.build(new File(PATH_BASE), xmlfile)) { - testArgs.add(new Object[] { ctx }); - } - return testArgs; - - } - - public SqlppExecutionIT(TestCaseContext tcCtx) { - super(tcCtx); - } -} http://git-wip-us.apache.org/repos/asf/asterixdb/blob/2530e39d/asterixdb/asterix-server/src/test/java/org/apache/asterix/test/server/AbstractExecutionIT.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-server/src/test/java/org/apache/asterix/test/server/AbstractExecutionIT.java b/asterixdb/asterix-server/src/test/java/org/apache/asterix/test/server/AbstractExecutionIT.java new file mode 100644 index 0000000..1a2f9ba --- /dev/null +++ b/asterixdb/asterix-server/src/test/java/org/apache/asterix/test/server/AbstractExecutionIT.java @@ -0,0 +1,163 @@ +/* + * Copyright 2009-2013 by The Regents of the University of California + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * you may obtain a copy of the License from + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.asterix.test.server; + +import static org.apache.hyracks.util.file.FileUtil.joinPath; + +import java.io.File; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +import org.apache.asterix.external.util.ExternalDataConstants; +import org.apache.asterix.external.util.IdentitiyResolverFactory; +import org.apache.asterix.test.base.RetainLogsRule; +import org.apache.asterix.test.common.TestExecutor; +import org.apache.asterix.testframework.context.TestCaseContext; +import org.apache.asterix.testframework.context.TestFileContext; +import org.apache.asterix.testframework.xml.TestCase.CompilationUnit; +import org.apache.asterix.testframework.xml.TestGroup; +import org.apache.commons.lang3.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.codehaus.plexus.util.FileUtils; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TestRule; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.junit.runners.Parameterized.Parameters; + +/** + * Runs the runtime test cases under 'asterix-app/src/test/resources/runtimets'. + */ +@RunWith(Parameterized.class) +public abstract class AbstractExecutionIT { + + protected static final Logger LOGGER = LogManager.getLogger(); + + protected static final String PATH_ACTUAL = joinPath("target", "ittest"); + protected static final String PATH_BASE = joinPath("..", "asterix-app", "src", "test", "resources", "runtimets"); + + protected static final String HDFS_BASE = "../asterix-app/"; + + protected static final TestExecutor testExecutor = new TestExecutor(); + + private static final String EXTERNAL_LIBRARY_TEST_GROUP = "lib"; + + private static final List<String> badTestCases = new ArrayList<>(); + + private static String reportPath = new File(joinPath("target", "failsafe-reports")).getAbsolutePath(); + + @Rule + public TestRule retainLogs = new RetainLogsRule(NCServiceExecutionIT.LOG_DIR, reportPath, this); + + @BeforeClass + public static void setUp() throws Exception { + System.out.println("Starting setup"); + if (LOGGER.isInfoEnabled()) { + LOGGER.info("Starting setup"); + } + File outdir = new File(PATH_ACTUAL); + outdir.mkdirs(); + + File externalTestsJar = + new File(StringUtils.join(new String[] { "..", "asterix-external-data", "target" }, File.separator)) + .listFiles((dir, name) -> name.matches("asterix-external-data-.*-tests.jar"))[0]; + + FileUtils.copyFile(externalTestsJar, + new File(NCServiceExecutionIT.APP_HOME + "/repo", externalTestsJar.getName())); + + NCServiceExecutionIT.setUp(); + + FileUtils.copyDirectoryStructure(new File(joinPath("..", "asterix-app", "data")), + new File(NCServiceExecutionIT.ASTERIX_APP_DIR + "/clusters/local/working_dir/data")); + + FileUtils.copyDirectoryStructure(new File(joinPath("..", "asterix-app", "target", "data")), + new File(NCServiceExecutionIT.ASTERIX_APP_DIR + "/clusters/local/working_dir/target/data")); + + // Set the node resolver to be the identity resolver that expects node names + // to be node controller ids; a valid assumption in test environment. + System.setProperty(ExternalDataConstants.NODE_RESOLVER_FACTORY_PROPERTY, + IdentitiyResolverFactory.class.getName()); + + reportPath = new File(joinPath("target", "failsafe-reports")).getAbsolutePath(); + } + + @AfterClass + public static void tearDown() throws Exception { + File outdir = new File(PATH_ACTUAL); + File[] files = outdir.listFiles(); + if ((files == null) || (files.length == 0)) { + outdir.delete(); + } + //AsterixLifecycleIT.tearDown(); + NCServiceExecutionIT.tearDown(); + if (!badTestCases.isEmpty()) { + System.out.println("The following test cases left some data"); + for (String testCase : badTestCases) { + System.out.println(testCase); + } + } + } + + @Parameters + public static Collection<Object[]> tests() throws Exception { + Collection<Object[]> testArgs = new ArrayList<Object[]>(); + TestCaseContext.Builder b = new TestCaseContext.Builder(); + for (TestCaseContext ctx : b.build(new File(PATH_BASE))) { + testArgs.add(new Object[] { ctx }); + } + return testArgs; + } + + private TestCaseContext tcCtx; + + public AbstractExecutionIT(TestCaseContext tcCtx) { + this.tcCtx = tcCtx; + } + + @Test + public void test() throws Exception { + if (skip()) { + return; + } + testExecutor.executeTest(PATH_ACTUAL, tcCtx, null, false); + testExecutor.cleanup(tcCtx.toString(), badTestCases); + } + + protected boolean skip() { + // If the test case contains library commands, we skip them + List<CompilationUnit> cUnits = tcCtx.getTestCase().getCompilationUnit(); + for (CompilationUnit cUnit : cUnits) { + List<TestFileContext> testFileCtxs = tcCtx.getTestFiles(cUnit); + for (TestFileContext ctx : testFileCtxs) { + if (ctx.getType().equals(EXTERNAL_LIBRARY_TEST_GROUP)) { + return true; + } + } + } + // For now we skip api tests. + for (TestGroup group : tcCtx.getTestGroups()) { + if (group != null && "api".equals(group.getName())) { + LOGGER.info("Skipping test: " + tcCtx.toString()); + return true; + } + } + return false; + } +} http://git-wip-us.apache.org/repos/asf/asterixdb/blob/2530e39d/asterixdb/asterix-server/src/test/java/org/apache/asterix/test/server/LicensingIT.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-server/src/test/java/org/apache/asterix/test/server/LicensingIT.java b/asterixdb/asterix-server/src/test/java/org/apache/asterix/test/server/LicensingIT.java new file mode 100644 index 0000000..4ac3abb --- /dev/null +++ b/asterixdb/asterix-server/src/test/java/org/apache/asterix/test/server/LicensingIT.java @@ -0,0 +1,94 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.asterix.test.server; + +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import org.apache.commons.io.FileUtils; +import org.apache.hyracks.util.file.FileUtil; +import org.junit.Assert; +import org.junit.Before; +import org.junit.FixMethodOrder; +import org.junit.Test; +import org.junit.runners.MethodSorters; + +@FixMethodOrder(MethodSorters.JVM) +public class LicensingIT { + + protected String installerDir; + + @Before + public void setup() { + final String pattern = getInstallerDirPattern(); + final String targetDir = getTargetDir(); + final String[] list = new File(targetDir).list((dir, name) -> name.matches(pattern)); + Assert.assertNotNull("installerDir", list); + Assert.assertFalse("Ambiguous install dir (" + pattern + "): " + Arrays.toString(list), list.length > 1); + Assert.assertEquals("Can't find install dir (" + pattern + ")", 1, list.length); + installerDir = FileUtil.joinPath(targetDir, list[0]); + } + + protected String getTargetDir() { + return FileUtil.joinPath("target"); + } + + protected String getInstallerDirPattern() { + return "asterix-server.*-binary-assembly"; + } + + protected String pathToLicensingFiles() { + return ""; + } + + @Test + public void testLicenseNoticeFilesPresent() throws IOException { + for (String name : getRequiredArtifactNames()) { + final String fileName = FileUtil.joinPath(installerDir, pathToLicensingFiles(), name); + Assert.assertTrue(fileName + " missing", new File(fileName).exists()); + } + } + + protected String[] getRequiredArtifactNames() { + return org.apache.commons.lang3.ArrayUtils.add(getLicenseArtifactNames(), "NOTICE"); + } + + @Test + public void ensureNoMissingLicenses() throws IOException { + for (String licenseArtifactName : getLicenseArtifactNames()) { + final File licenseFile = + new File(FileUtil.joinPath(installerDir, pathToLicensingFiles(), licenseArtifactName)); + List<String> badLines = new ArrayList<>(); + for (String line : FileUtils.readLines(licenseFile, StandardCharsets.UTF_8)) { + if (line.matches("^\\s*MISSING:.*")) { + badLines.add(line.trim()); + } + } + Assert.assertEquals("Missing licenses in " + licenseFile + ": " + badLines, 0, badLines.size()); + } + } + + protected String[] getLicenseArtifactNames() { + return new String[] { "LICENSE" }; + } +} http://git-wip-us.apache.org/repos/asf/asterixdb/blob/2530e39d/asterixdb/asterix-server/src/test/java/org/apache/asterix/test/server/NCServiceExecutionIT.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-server/src/test/java/org/apache/asterix/test/server/NCServiceExecutionIT.java b/asterixdb/asterix-server/src/test/java/org/apache/asterix/test/server/NCServiceExecutionIT.java new file mode 100644 index 0000000..3c46be6 --- /dev/null +++ b/asterixdb/asterix-server/src/test/java/org/apache/asterix/test/server/NCServiceExecutionIT.java @@ -0,0 +1,266 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.asterix.test.server; + +import java.io.File; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Random; +import java.util.concurrent.TimeUnit; + +import org.apache.asterix.test.common.TestExecutor; +import org.apache.asterix.test.runtime.HDFSCluster; +import org.apache.asterix.testframework.context.TestCaseContext; +import org.apache.asterix.testframework.xml.TestGroup; +import org.apache.commons.io.FileUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.hyracks.test.server.process.HyracksCCProcess; +import org.apache.hyracks.test.server.process.HyracksNCServiceProcess; +import org.apache.hyracks.test.server.process.HyracksVirtualCluster; +import org.apache.logging.log4j.Level; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.FixMethodOrder; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.MethodSorters; +import org.junit.runners.Parameterized; +import org.junit.runners.Parameterized.Parameters; + +@FixMethodOrder(MethodSorters.NAME_ASCENDING) +@RunWith(Parameterized.class) +public class NCServiceExecutionIT { + + // Important paths and files for this test. + + // The "target" subdirectory of asterix-server. All outputs go here. + public static final String TARGET_DIR = + StringUtils.join(new String[] { "../asterix-server/target" }, File.separator); + + // Directory where the NCs create and store all data, as configured by + // src/test/resources/NCServiceExecutionIT/cc.conf. + public static final String INSTANCE_DIR = StringUtils.join(new String[] { TARGET_DIR, "tmp" }, File.separator); + + // The log directory, where all CC, NCService, and NC logs are written. CC and + // NCService logs are configured on the HyracksVirtualCluster below. NC logs + // are configured in src/test/resources/NCServiceExecutionIT/ncservice*.conf. + public static final String LOG_DIR = + StringUtils.join(new String[] { TARGET_DIR, "failsafe-reports" }, File.separator); + + // Directory where *.conf files are located. + public static final String CONF_DIR = + StringUtils.join(new String[] { TARGET_DIR, "test-classes", "NCServiceExecutionIT" }, File.separator); + + // The app.home specified for HyracksVirtualCluster. The NCService expects + // to find the NC startup script in ${app.home}/bin. + public static final String APP_HOME = StringUtils.join(new String[] { TARGET_DIR, "appassembler" }, File.separator); + + // Path to the asterix-app directory. This is used as the current working + // directory for the CC and NCService processes, which allows relative file + // paths in "load" statements in test queries to find the right data. It is + // also used for HDFSCluster. + public static final String ASTERIX_APP_DIR = StringUtils.join(new String[] { "..", "asterix-app" }, File.separator); + + // Path to the actual AQL test files, which we borrow from asterix-app. This is + // passed to TestExecutor. + protected static final String TESTS_DIR = + StringUtils.join(new String[] { ASTERIX_APP_DIR, "src", "test", "resources", "runtimets" }, File.separator); + + // Path that actual results are written to. We create and clean this directory + // here, and also pass it to TestExecutor which writes the test output there. + public static final String ACTUAL_RESULTS_DIR = + StringUtils.join(new String[] { TARGET_DIR, "ittest" }, File.separator); + + private static final Logger LOGGER = LogManager.getLogger(); + + enum KillCommand { + CC, + NC1, + NC2; + + @Override + public String toString() { + return "<kill " + name().toLowerCase() + ">"; + } + } + + private static HyracksCCProcess cc; + private static HyracksNCServiceProcess nc1; + private static HyracksNCServiceProcess nc2; + + private final TestCaseContext tcCtx; + private static final TestExecutor testExecutor = new TestExecutor(); + + private static final List<String> badTestCases = new ArrayList<>(); + private static HyracksVirtualCluster cluster; + private final KillCommand killType; + private static boolean clusterActive = false; + + @BeforeClass + public static void setUp() throws Exception { + // Create actual-results output directory. + File outDir = new File(ACTUAL_RESULTS_DIR); + outDir.mkdirs(); + + // Remove any instance data from previous runs. + File instanceDir = new File(INSTANCE_DIR); + if (instanceDir.isDirectory()) { + FileUtils.deleteDirectory(instanceDir); + } + + // HDFSCluster requires the input directory to end with a file separator. + HDFSCluster.getInstance().setup(ASTERIX_APP_DIR + File.separator); + + cluster = new HyracksVirtualCluster(new File(APP_HOME), new File(ASTERIX_APP_DIR)); + nc1 = cluster.addNCService(new File(CONF_DIR, "ncservice1.conf"), new File(LOG_DIR, "ncservice1.log")); + + nc2 = cluster.addNCService(new File(CONF_DIR, "ncservice2.conf"), new File(LOG_DIR, "ncservice2.log")); + + // Start CC + cc = cluster.start(new File(CONF_DIR, "cc.conf"), new File(LOG_DIR, "cc.log")); + + testExecutor.waitForClusterActive(30, TimeUnit.SECONDS); + clusterActive = true; + } + + @AfterClass + public static void tearDown() throws Exception { + File outdir = new File(ACTUAL_RESULTS_DIR); + File[] files = outdir.listFiles(); + if (files == null || files.length == 0) { + outdir.delete(); + } + cluster.stop(); + HDFSCluster.getInstance().cleanup(); + if (!badTestCases.isEmpty()) { + System.out.println("The following test cases left some data"); + for (String testCase : badTestCases) { + System.out.println(testCase); + } + } + } + + @Parameters(name = "NCServiceExecutionTest {index}: {0}") + public static Collection<Object[]> tests() throws Exception { + Collection<Object[]> testArgs = new ArrayList<>(); + Random random = getRandom(); + TestCaseContext.Builder b = new TestCaseContext.Builder(); + for (TestCaseContext ctx : b.build(new File(TESTS_DIR))) { + if (!skip(ctx)) { + testArgs.add(new Object[] { ctx, ctx, null }); + } + // let's kill something every 50 tests + if (testArgs.size() % 50 == 0) { + final KillCommand killCommand = KillCommand.values()[random.nextInt(KillCommand.values().length)]; + testArgs.add(new Object[] { killCommand, null, killCommand }); + } + } + return testArgs; + } + + private static Random getRandom() { + Random random; + if (System.getProperty("random.seed") == null) { + random = new Random() { + @Override + public synchronized void setSeed(long seed) { + super.setSeed(seed); + System.err.println("using generated seed: " + seed + "; use -Drandom.seed to use specific seed"); + } + }; + } else { + final long seed = Long.getLong("random.seed"); + System.err.println("using provided seed (-Drandom.seed): " + seed); + random = new Random(seed); + } + return random; + } + + private static boolean skip(TestCaseContext tcCtx) { + // For now we skip feeds tests, external-library, and api tests. + for (TestGroup group : tcCtx.getTestGroups()) { + if (group.getName().startsWith("external-") || group.getName().equals("feeds") + || group.getName().equals("api")) { + LOGGER.info("Skipping test: " + tcCtx.toString()); + return true; + } + } + return false; + } + + public NCServiceExecutionIT(Object description, TestCaseContext tcCtx, KillCommand killType) { + this.tcCtx = tcCtx; + this.killType = killType; + } + + @Before + public void before() { + if (!clusterActive) { + Assert.fail("Skipping test since cluster is not ACTIVE"); + } + } + + @Test + public void test() throws Exception { + if (tcCtx != null) { + testExecutor.executeTest(ACTUAL_RESULTS_DIR, tcCtx, null, false); + testExecutor.cleanup(tcCtx.toString(), badTestCases); + } else { + switch (killType) { + case CC: + LOGGER.info("Killing CC..."); + cc.stop(true); + cc.start(); + break; + + case NC1: + LOGGER.info("Killing NC1..."); + nc1.stop(); // we can't kill due to ASTERIXDB-1941 + testExecutor.waitForClusterState("UNUSABLE", 60, TimeUnit.SECONDS); // wait for missed heartbeats... + nc1.start(); // this restarts the NC service + testExecutor.startNC("asterix_nc1"); + break; + + case NC2: + LOGGER.info("Killing NC2..."); + nc2.stop(); // we can't kill due to ASTERIXDB-1941 + testExecutor.waitForClusterState("UNUSABLE", 60, TimeUnit.SECONDS); // wait for missed heartbeats... + nc2.start(); // this restarts the NC service + testExecutor.startNC("asterix_nc2"); + break; + + default: + Assert.fail("killType: " + killType); + } + try { + testExecutor.waitForClusterActive(30, TimeUnit.SECONDS); + } catch (Exception e) { + // stop executing the rest of the tests since the cluster is not ACTIVE + LOGGER.log(Level.ERROR, "Cannot continue since cluster is not ACTIVE", e); + clusterActive = false; + Assert.fail("Cluster is not ACTIVE"); + } + } + } +} http://git-wip-us.apache.org/repos/asf/asterixdb/blob/2530e39d/asterixdb/asterix-server/src/test/java/org/apache/asterix/test/server/NcLifecycleIT.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-server/src/test/java/org/apache/asterix/test/server/NcLifecycleIT.java b/asterixdb/asterix-server/src/test/java/org/apache/asterix/test/server/NcLifecycleIT.java new file mode 100644 index 0000000..4ae9d96 --- /dev/null +++ b/asterixdb/asterix-server/src/test/java/org/apache/asterix/test/server/NcLifecycleIT.java @@ -0,0 +1,119 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.asterix.test.server; + +import static org.apache.asterix.test.server.NCServiceExecutionIT.APP_HOME; +import static org.apache.asterix.test.server.NCServiceExecutionIT.ASTERIX_APP_DIR; +import static org.apache.asterix.test.server.NCServiceExecutionIT.INSTANCE_DIR; +import static org.apache.asterix.test.server.NCServiceExecutionIT.LOG_DIR; +import static org.apache.asterix.test.server.NCServiceExecutionIT.TARGET_DIR; + +import java.io.File; +import java.util.ArrayList; +import java.util.Collection; +import java.util.concurrent.TimeUnit; + +import org.apache.asterix.test.base.RetainLogsRule; +import org.apache.asterix.test.common.TestExecutor; +import org.apache.asterix.testframework.context.TestCaseContext; +import org.apache.commons.io.FileUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.hyracks.test.server.process.HyracksVirtualCluster; +import org.apache.hyracks.util.file.FileUtil; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TestRule; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +@RunWith(Parameterized.class) +public class NcLifecycleIT { + + private static final String PATH_BASE = + FileUtil.joinPath("src", "test", "resources", "integrationts", "NcLifecycle"); + private static final String CONF_DIR = + StringUtils.join(new String[] { TARGET_DIR, "test-classes", "NcLifecycleIT" }, File.separator); + private static final String PATH_ACTUAL = FileUtil.joinPath("target", "ittest"); + private static final Logger LOGGER = LogManager.getLogger(); + private static String reportPath = new File(FileUtil.joinPath("target", "failsafe-reports")).getAbsolutePath(); + private static final TestExecutor testExecutor = new TestExecutor(); + private static HyracksVirtualCluster cluster; + + private TestCaseContext tcCtx; + + public NcLifecycleIT(TestCaseContext tcCtx) { + this.tcCtx = tcCtx; + } + + @Rule + public TestRule retainLogs = new RetainLogsRule(NCServiceExecutionIT.ASTERIX_APP_DIR, reportPath, this); + + @Before + public void before() throws Exception { + LOGGER.info("Creating new instance..."); + File instanceDir = new File(INSTANCE_DIR); + if (instanceDir.isDirectory()) { + FileUtils.deleteDirectory(instanceDir); + } + + cluster = new HyracksVirtualCluster(new File(APP_HOME), new File(ASTERIX_APP_DIR)); + cluster.addNCService(new File(CONF_DIR, "ncservice1.conf"), new File(LOG_DIR, "ncservice1.log")); + cluster.addNCService(new File(CONF_DIR, "ncservice2.conf"), new File(LOG_DIR, "ncservice2.log")); + + // Start CC + cluster.start(new File(CONF_DIR, "cc.conf"), new File(LOG_DIR, "cc.log")); + LOGGER.info("Instance created."); + testExecutor.waitForClusterActive(30, TimeUnit.SECONDS); + LOGGER.info("Instance is in ACTIVE state."); + } + + @After + public void after() { + LOGGER.info("Destroying instance..."); + cluster.stop(); + LOGGER.info("Instance destroyed."); + } + + @Test + public void test() throws Exception { + testExecutor.executeTest(PATH_ACTUAL, tcCtx, null, false); + } + + @Parameterized.Parameters(name = "NcLifecycleIT {index}: {0}") + public static Collection<Object[]> tests() throws Exception { + Collection<Object[]> testArgs = buildTestsInXml(TestCaseContext.DEFAULT_TESTSUITE_XML_NAME); + if (testArgs.size() == 0) { + testArgs = buildTestsInXml(TestCaseContext.DEFAULT_TESTSUITE_XML_NAME); + } + return testArgs; + } + + private static Collection<Object[]> buildTestsInXml(String xmlfile) throws Exception { + Collection<Object[]> testArgs = new ArrayList<>(); + TestCaseContext.Builder b = new TestCaseContext.Builder(); + for (TestCaseContext ctx : b.build(new File(PATH_BASE), xmlfile)) { + testArgs.add(new Object[] { ctx }); + } + return testArgs; + } +}
