Michael Blow has submitted this change and it was merged. Change subject: [NO ISSUE] Fix logging for ncservice-based tests ......................................................................
[NO ISSUE] Fix logging for ncservice-based tests Change-Id: Id264aede0f62558ad6e34355047c623a1d594692 Reviewed-on: https://asterix-gerrit.ics.uci.edu/3311 Tested-by: Jenkins <[email protected]> Contrib: Jenkins <[email protected]> Integration-Tests: Jenkins <[email protected]> Reviewed-by: Ian Maxon <[email protected]> --- M asterixdb/asterix-app/src/test/java/org/apache/asterix/api/http/servlet/VersionApiServletTest.java M asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/HDFSCluster.java M asterixdb/asterix-server/src/test/java/org/apache/asterix/test/server/AbstractExecutionIT.java M asterixdb/asterix-server/src/test/java/org/apache/asterix/test/server/NCServiceExecutionIT.java M asterixdb/asterix-server/src/test/java/org/apache/asterix/test/server/NcLifecycleIT.java M asterixdb/asterix-server/src/test/java/org/apache/asterix/test/server/RecoveryIT.java M asterixdb/asterix-server/src/test/java/org/apache/asterix/test/server/ReplicationIT.java M asterixdb/asterix-server/src/test/resources/NCServiceExecutionIT/cc.conf M asterixdb/asterix-server/src/test/resources/NCServiceExecutionIT/ncservice1.conf M asterixdb/asterix-server/src/test/resources/NCServiceExecutionIT/ncservice2.conf M asterixdb/asterix-server/src/test/resources/NcLifecycleIT/cc.conf M asterixdb/asterix-server/src/test/resources/NcLifecycleIT/ncservice1.conf M asterixdb/asterix-server/src/test/resources/NcLifecycleIT/ncservice2.conf M asterixdb/asterix-server/src/test/resources/ReplicationIT/cc.conf M asterixdb/asterix-server/src/test/resources/ReplicationIT/ncservice1.conf M asterixdb/asterix-server/src/test/resources/ReplicationIT/ncservice2.conf M hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/CCLogConfigurationFactory.java M hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/NodeControllerData.java M hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/NodeRegistration.java R hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/resources/log4j2.xml M hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NCLogConfigurationFactory.java M hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/HttpServer.java M hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/WebManager.java M hyracks-fullstack/hyracks/hyracks-server/pom.xml M hyracks-fullstack/hyracks/hyracks-server/src/test/java/org/apache/hyracks/test/server/NCServiceIT.java M hyracks-fullstack/hyracks/hyracks-server/src/test/java/org/apache/hyracks/test/server/process/HyracksCCProcess.java M hyracks-fullstack/hyracks/hyracks-server/src/test/java/org/apache/hyracks/test/server/process/HyracksNCServiceProcess.java M hyracks-fullstack/hyracks/hyracks-server/src/test/java/org/apache/hyracks/test/server/process/HyracksServerProcess.java M hyracks-fullstack/hyracks/hyracks-server/src/test/java/org/apache/hyracks/test/server/process/HyracksVirtualCluster.java M hyracks-fullstack/hyracks/hyracks-server/src/test/resources/NCServiceIT/cc.conf M hyracks-fullstack/hyracks/hyracks-server/src/test/resources/NCServiceIT/nc-blue.conf M hyracks-fullstack/hyracks/hyracks-server/src/test/resources/NCServiceIT/nc-red.conf M hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/PidHelper.java M hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/trace/Event.java M hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/trace/Tracer.java 35 files changed, 210 insertions(+), 233 deletions(-) Approvals: Anon. E. Moose #1000171: Jenkins: Verified; ; Verified Ian Maxon: Looks good to me, approved Objections: Jenkins: Violations found diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/http/servlet/VersionApiServletTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/http/servlet/VersionApiServletTest.java index 976dbc3..8dcfa26 100644 --- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/http/servlet/VersionApiServletTest.java +++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/http/servlet/VersionApiServletTest.java @@ -33,13 +33,10 @@ import org.apache.asterix.api.http.server.VersionApiServlet; import org.apache.asterix.common.config.BuildProperties; import org.apache.asterix.runtime.utils.CcApplicationContext; -import org.apache.asterix.test.runtime.SqlppExecutionTest; import org.apache.hyracks.api.client.IHyracksClientConnection; import org.apache.hyracks.http.api.IServletRequest; import org.apache.hyracks.http.api.IServletResponse; -import org.junit.After; import org.junit.Assert; -import org.junit.Before; import org.junit.Test; import com.fasterxml.jackson.databind.ObjectMapper; @@ -47,19 +44,12 @@ import io.netty.handler.codec.http.FullHttpRequest; import io.netty.handler.codec.http.HttpMethod; +import io.netty.util.internal.logging.Log4J2LoggerFactory; public class VersionApiServletTest { - - @Before - public void setup() throws Exception { - // Starts test asterixdb cluster. - SqlppExecutionTest.setUp(); - } - - @After - public void teardown() throws Exception { - // Tears down the asterixdb cluster. - SqlppExecutionTest.tearDown(); + static { + // bootstrap netty with log4j2 logging + io.netty.util.internal.logging.InternalLoggerFactory.setDefaultFactory(Log4J2LoggerFactory.INSTANCE); } @Test diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/HDFSCluster.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/HDFSCluster.java index a89304a..c0b4841 100644 --- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/HDFSCluster.java +++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/HDFSCluster.java @@ -59,17 +59,18 @@ /** * Instantiates the (Mini) DFS Cluster with the configured number of datanodes. - * Post instantiation, data is laoded to HDFS. + * Post instantiation, data is loaded to HDFS. * Called prior to running the Runtime test suite. */ public void setup() throws Exception { - setup(""); + setup(new File(".")); } - public void setup(String basePath) throws Exception { - conf.addResource(new Path(basePath + PATH_TO_HADOOP_CONF + "/core-site.xml")); - conf.addResource(new Path(basePath + PATH_TO_HADOOP_CONF + "/mapred-site.xml")); - conf.addResource(new Path(basePath + PATH_TO_HADOOP_CONF + "/hdfs-site.xml")); + public void setup(File basePath) throws Exception { + File hadoopConfDir = new File(basePath, PATH_TO_HADOOP_CONF); + conf.addResource(new Path(new File(hadoopConfDir, "core-site.xml").getPath())); + conf.addResource(new Path(new File(hadoopConfDir, "mapred-site.xml").getPath())); + conf.addResource(new Path(new File(hadoopConfDir, "hdfs-site.xml").getPath())); cleanupLocal(); conf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, MINIDFS_BASEDIR); MiniDFSCluster.Builder build = new MiniDFSCluster.Builder(conf); @@ -81,10 +82,10 @@ loadData(basePath); } - private void loadData(String localDataRoot) throws IOException { + private void loadData(File localDataRoot) throws IOException { Path destDir = new Path(HDFS_PATH); dfs.mkdirs(destDir); - File srcDir = new File(localDataRoot + DATA_PATH); + File srcDir = new File(localDataRoot, DATA_PATH); if (srcDir.exists()) { File[] listOfFiles = srcDir.listFiles(); for (File srcFile : listOfFiles) { diff --git a/asterixdb/asterix-server/src/test/java/org/apache/asterix/test/server/AbstractExecutionIT.java b/asterixdb/asterix-server/src/test/java/org/apache/asterix/test/server/AbstractExecutionIT.java index 1a2f9ba..35bf7e4 100644 --- a/asterixdb/asterix-server/src/test/java/org/apache/asterix/test/server/AbstractExecutionIT.java +++ b/asterixdb/asterix-server/src/test/java/org/apache/asterix/test/server/AbstractExecutionIT.java @@ -14,12 +14,15 @@ */ package org.apache.asterix.test.server; +import static org.apache.asterix.test.server.NCServiceExecutionIT.APP_HOME; +import static org.apache.asterix.test.server.NCServiceExecutionIT.ASTERIX_APP_DIR; import static org.apache.hyracks.util.file.FileUtil.joinPath; import java.io.File; import java.util.ArrayList; import java.util.Collection; import java.util.List; +import java.util.Objects; import org.apache.asterix.external.util.ExternalDataConstants; import org.apache.asterix.external.util.IdentitiyResolverFactory; @@ -29,7 +32,6 @@ import org.apache.asterix.testframework.context.TestFileContext; import org.apache.asterix.testframework.xml.TestCase.CompilationUnit; import org.apache.asterix.testframework.xml.TestGroup; -import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.codehaus.plexus.util.FileUtils; @@ -75,20 +77,18 @@ File outdir = new File(PATH_ACTUAL); outdir.mkdirs(); - File externalTestsJar = - new File(StringUtils.join(new String[] { "..", "asterix-external-data", "target" }, File.separator)) - .listFiles((dir, name) -> name.matches("asterix-external-data-.*-tests.jar"))[0]; + File externalTestsJar = Objects.requireNonNull(new File(joinPath("..", "asterix-external-data", "target")) + .listFiles((dir, name) -> name.matches("asterix-external-data-.*-tests.jar")))[0]; - FileUtils.copyFile(externalTestsJar, - new File(NCServiceExecutionIT.APP_HOME + "/repo", externalTestsJar.getName())); + FileUtils.copyFile(externalTestsJar, new File(APP_HOME, joinPath("repo", externalTestsJar.getName()))); NCServiceExecutionIT.setUp(); FileUtils.copyDirectoryStructure(new File(joinPath("..", "asterix-app", "data")), - new File(NCServiceExecutionIT.ASTERIX_APP_DIR + "/clusters/local/working_dir/data")); + new File(ASTERIX_APP_DIR, joinPath("clusters", "local", "working_dir", "data"))); FileUtils.copyDirectoryStructure(new File(joinPath("..", "asterix-app", "target", "data")), - new File(NCServiceExecutionIT.ASTERIX_APP_DIR + "/clusters/local/working_dir/target/data")); + new File(ASTERIX_APP_DIR, joinPath("clusters", "local", "working_dir", "target", "data"))); // Set the node resolver to be the identity resolver that expects node names // to be node controller ids; a valid assumption in test environment. diff --git a/asterixdb/asterix-server/src/test/java/org/apache/asterix/test/server/NCServiceExecutionIT.java b/asterixdb/asterix-server/src/test/java/org/apache/asterix/test/server/NCServiceExecutionIT.java index 64506f5..d64dcb5 100644 --- a/asterixdb/asterix-server/src/test/java/org/apache/asterix/test/server/NCServiceExecutionIT.java +++ b/asterixdb/asterix-server/src/test/java/org/apache/asterix/test/server/NCServiceExecutionIT.java @@ -18,19 +18,22 @@ */ package org.apache.asterix.test.server; +import static org.apache.hyracks.util.file.FileUtil.joinPath; + import java.io.File; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; import java.util.List; import java.util.Random; import java.util.concurrent.TimeUnit; +import java.util.stream.Stream; import org.apache.asterix.test.common.TestExecutor; import org.apache.asterix.test.runtime.HDFSCluster; import org.apache.asterix.testframework.context.TestCaseContext; import org.apache.asterix.testframework.xml.TestGroup; import org.apache.commons.io.FileUtils; -import org.apache.commons.lang3.StringUtils; import org.apache.hyracks.test.server.process.HyracksCCProcess; import org.apache.hyracks.test.server.process.HyracksNCServiceProcess; import org.apache.hyracks.test.server.process.HyracksVirtualCluster; @@ -55,45 +58,63 @@ // Important paths and files for this test. // The "target" subdirectory of asterix-server. All outputs go here. - public static final String TARGET_DIR = - StringUtils.join(new String[] { "../asterix-server/target" }, File.separator); + public static final String TARGET_DIR = joinPath("..", "asterix-server", "target"); // Directory where the NCs create and store all data, as configured by // src/test/resources/NCServiceExecutionIT/cc.conf. - public static final String INSTANCE_DIR = StringUtils.join(new String[] { TARGET_DIR, "tmp" }, File.separator); + public static final String INSTANCE_DIR = joinPath(TARGET_DIR, "tmp"); // The log directory, where all CC, NCService, and NC logs are written. CC and // NCService logs are configured on the HyracksVirtualCluster below. NC logs // are configured in src/test/resources/NCServiceExecutionIT/ncservice*.conf. - public static final String LOG_DIR = - StringUtils.join(new String[] { TARGET_DIR, "failsafe-reports" }, File.separator); + public static final String LOG_DIR = joinPath(TARGET_DIR, "failsafe-reports"); // Directory where *.conf files are located. - public static final String CONF_DIR = - StringUtils.join(new String[] { TARGET_DIR, "test-classes", "NCServiceExecutionIT" }, File.separator); + public static final String CONF_DIR = joinPath(TARGET_DIR, "test-classes", "NCServiceExecutionIT"); // The app.home specified for HyracksVirtualCluster. The NCService expects // to find the NC startup script in ${app.home}/bin. - public static final String APP_HOME = StringUtils.join(new String[] { TARGET_DIR, "appassembler" }, File.separator); + public static final File APP_HOME = new File(TARGET_DIR, "appassembler"); // Path to the asterix-app directory. This is used as the current working // directory for the CC and NCService processes, which allows relative file // paths in "load" statements in test queries to find the right data. It is // also used for HDFSCluster. - public static final String ASTERIX_APP_DIR = StringUtils.join(new String[] { "..", "asterix-app" }, File.separator); + public static final File ASTERIX_APP_DIR = new File(joinPath("..", "asterix-app")); // Path to the actual AQL test files, which we borrow from asterix-app. This is // passed to TestExecutor. - protected static final String TESTS_DIR = - StringUtils.join(new String[] { ASTERIX_APP_DIR, "src", "test", "resources", "runtimets" }, File.separator); + protected static final File TESTS_DIR = + new File(ASTERIX_APP_DIR, joinPath("src", "test", "resources", "runtimets")); // Path that actual results are written to. We create and clean this directory // here, and also pass it to TestExecutor which writes the test output there. - public static final String ACTUAL_RESULTS_DIR = - StringUtils.join(new String[] { TARGET_DIR, "ittest" }, File.separator); + public static final File ACTUAL_RESULTS_DIR = new File(TARGET_DIR, "ittest"); private static final Logger LOGGER = LogManager.getLogger(); private static boolean startHdfs; + + @AfterClass + public static void checkLogFiles() { + NCServiceExecutionIT.checkLogFiles(new File(TARGET_DIR, NCServiceExecutionIT.class.getSimpleName()), + "asterix_nc1", "asterix_nc2"); + } + + public static void checkLogFiles(File logsPath, String... ncNames) { + LOGGER.info("verifying server log files"); + Assert.assertTrue("logs dir (" + logsPath + ") missing!", logsPath.exists()); + List<String> allFiles = new ArrayList<>(); + Stream.of(ncNames).map(NCServiceExecutionIT::addNcFiles).forEach(allFiles::addAll); + allFiles.add("access.log"); + allFiles.add("cc.log"); + allFiles.forEach(file -> { + Assert.assertTrue(file + " missing!", new File(logsPath, file).exists()); + }); + } + + private static List<String> addNcFiles(String ncName) { + return Arrays.asList("access-" + ncName + ".log", "nc-" + ncName + ".log", "trace-" + ncName + ".log"); + } enum KillCommand { CC, @@ -126,8 +147,7 @@ public static void setUp(boolean startHdfs) throws Exception { NCServiceExecutionIT.startHdfs = startHdfs; // Create actual-results output directory. - File outDir = new File(ACTUAL_RESULTS_DIR); - outDir.mkdirs(); + ACTUAL_RESULTS_DIR.mkdirs(); // Remove any instance data from previous runs. File instanceDir = new File(INSTANCE_DIR); @@ -136,17 +156,16 @@ } if (startHdfs) { - // HDFSCluster requires the input directory to end with a file separator. - HDFSCluster.getInstance().setup(ASTERIX_APP_DIR + File.separator); + HDFSCluster.getInstance().setup(ASTERIX_APP_DIR); } - cluster = new HyracksVirtualCluster(new File(APP_HOME), new File(ASTERIX_APP_DIR)); - nc1 = cluster.addNCService(new File(CONF_DIR, "ncservice1.conf"), new File(LOG_DIR, "ncservice1.log")); + cluster = new HyracksVirtualCluster(APP_HOME, ASTERIX_APP_DIR); + nc1 = cluster.addNCService(new File(CONF_DIR, "ncservice1.conf"), null); - nc2 = cluster.addNCService(new File(CONF_DIR, "ncservice2.conf"), new File(LOG_DIR, "ncservice2.log")); + nc2 = cluster.addNCService(new File(CONF_DIR, "ncservice2.conf"), null); // Start CC - cc = cluster.start(new File(CONF_DIR, "cc.conf"), new File(LOG_DIR, "asterixcc.out.log")); + cc = cluster.start(new File(CONF_DIR, "cc.conf"), null); testExecutor.waitForClusterActive(30, TimeUnit.SECONDS); clusterActive = true; @@ -154,10 +173,9 @@ @AfterClass public static void tearDown() throws Exception { - File outdir = new File(ACTUAL_RESULTS_DIR); - File[] files = outdir.listFiles(); + File[] files = ACTUAL_RESULTS_DIR.listFiles(); if (files == null || files.length == 0) { - outdir.delete(); + ACTUAL_RESULTS_DIR.delete(); } cluster.stop(); if (startHdfs) { @@ -176,7 +194,7 @@ Collection<Object[]> testArgs = new ArrayList<>(); Random random = getRandom(); TestCaseContext.Builder b = new TestCaseContext.Builder(); - for (TestCaseContext ctx : b.build(new File(TESTS_DIR))) { + for (TestCaseContext ctx : b.build(TESTS_DIR)) { if (!skip(ctx)) { testArgs.add(new Object[] { ctx, ctx, null }); } @@ -234,7 +252,7 @@ @Test public void test() throws Exception { if (tcCtx != null) { - testExecutor.executeTest(ACTUAL_RESULTS_DIR, tcCtx, null, false); + testExecutor.executeTest(ACTUAL_RESULTS_DIR.getPath(), tcCtx, null, false); testExecutor.cleanup(tcCtx.toString(), badTestCases); } else { switch (killType) { diff --git a/asterixdb/asterix-server/src/test/java/org/apache/asterix/test/server/NcLifecycleIT.java b/asterixdb/asterix-server/src/test/java/org/apache/asterix/test/server/NcLifecycleIT.java index 4ae9d96..a31e36f 100644 --- a/asterixdb/asterix-server/src/test/java/org/apache/asterix/test/server/NcLifecycleIT.java +++ b/asterixdb/asterix-server/src/test/java/org/apache/asterix/test/server/NcLifecycleIT.java @@ -18,11 +18,8 @@ */ package org.apache.asterix.test.server; -import static org.apache.asterix.test.server.NCServiceExecutionIT.APP_HOME; -import static org.apache.asterix.test.server.NCServiceExecutionIT.ASTERIX_APP_DIR; -import static org.apache.asterix.test.server.NCServiceExecutionIT.INSTANCE_DIR; -import static org.apache.asterix.test.server.NCServiceExecutionIT.LOG_DIR; -import static org.apache.asterix.test.server.NCServiceExecutionIT.TARGET_DIR; +import static org.apache.asterix.test.server.NCServiceExecutionIT.*; +import static org.apache.hyracks.util.file.FileUtil.joinPath; import java.io.File; import java.util.ArrayList; @@ -33,12 +30,11 @@ import org.apache.asterix.test.common.TestExecutor; import org.apache.asterix.testframework.context.TestCaseContext; import org.apache.commons.io.FileUtils; -import org.apache.commons.lang3.StringUtils; import org.apache.hyracks.test.server.process.HyracksVirtualCluster; -import org.apache.hyracks.util.file.FileUtil; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.junit.After; +import org.junit.AfterClass; import org.junit.Before; import org.junit.Rule; import org.junit.Test; @@ -49,13 +45,11 @@ @RunWith(Parameterized.class) public class NcLifecycleIT { - private static final String PATH_BASE = - FileUtil.joinPath("src", "test", "resources", "integrationts", "NcLifecycle"); - private static final String CONF_DIR = - StringUtils.join(new String[] { TARGET_DIR, "test-classes", "NcLifecycleIT" }, File.separator); - private static final String PATH_ACTUAL = FileUtil.joinPath("target", "ittest"); + private static final String PATH_BASE = joinPath("src", "test", "resources", "integrationts", "NcLifecycle"); + private static final String CONF_DIR = joinPath(TARGET_DIR, "test-classes", "NcLifecycleIT"); + private static final String PATH_ACTUAL = joinPath("target", "ittest"); private static final Logger LOGGER = LogManager.getLogger(); - private static String reportPath = new File(FileUtil.joinPath("target", "failsafe-reports")).getAbsolutePath(); + private static final File reportPath = new File("target", "failsafe-reports"); private static final TestExecutor testExecutor = new TestExecutor(); private static HyracksVirtualCluster cluster; @@ -66,7 +60,7 @@ } @Rule - public TestRule retainLogs = new RetainLogsRule(NCServiceExecutionIT.ASTERIX_APP_DIR, reportPath, this); + public TestRule retainLogs = new RetainLogsRule(ASTERIX_APP_DIR, reportPath, this); @Before public void before() throws Exception { @@ -76,12 +70,12 @@ FileUtils.deleteDirectory(instanceDir); } - cluster = new HyracksVirtualCluster(new File(APP_HOME), new File(ASTERIX_APP_DIR)); - cluster.addNCService(new File(CONF_DIR, "ncservice1.conf"), new File(LOG_DIR, "ncservice1.log")); - cluster.addNCService(new File(CONF_DIR, "ncservice2.conf"), new File(LOG_DIR, "ncservice2.log")); + cluster = new HyracksVirtualCluster(APP_HOME, ASTERIX_APP_DIR); + cluster.addNCService(new File(CONF_DIR, "ncservice1.conf"), null); + cluster.addNCService(new File(CONF_DIR, "ncservice2.conf"), null); // Start CC - cluster.start(new File(CONF_DIR, "cc.conf"), new File(LOG_DIR, "cc.log")); + cluster.start(new File(CONF_DIR, "cc.conf"), null); LOGGER.info("Instance created."); testExecutor.waitForClusterActive(30, TimeUnit.SECONDS); LOGGER.info("Instance is in ACTIVE state."); @@ -94,6 +88,12 @@ LOGGER.info("Instance destroyed."); } + @AfterClass + public static void checkLogFiles() { + NCServiceExecutionIT.checkLogFiles(new File(TARGET_DIR, NcLifecycleIT.class.getSimpleName()), "asterix_nc1", + "asterix_nc2"); + } + @Test public void test() throws Exception { testExecutor.executeTest(PATH_ACTUAL, tcCtx, null, false); diff --git a/asterixdb/asterix-server/src/test/java/org/apache/asterix/test/server/RecoveryIT.java b/asterixdb/asterix-server/src/test/java/org/apache/asterix/test/server/RecoveryIT.java index 9123a5d..214eb5a 100644 --- a/asterixdb/asterix-server/src/test/java/org/apache/asterix/test/server/RecoveryIT.java +++ b/asterixdb/asterix-server/src/test/java/org/apache/asterix/test/server/RecoveryIT.java @@ -18,17 +18,19 @@ */ package org.apache.asterix.test.server; +import static org.apache.hyracks.util.file.FileUtil.joinPath; + import java.io.File; import java.util.ArrayList; import java.util.Collection; import java.util.Map; +import java.util.Objects; import org.apache.asterix.test.base.RetainLogsRule; import org.apache.asterix.test.common.TestExecutor; import org.apache.asterix.test.runtime.HDFSCluster; import org.apache.asterix.testframework.context.TestCaseContext; import org.apache.commons.io.FileUtils; -import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.junit.AfterClass; @@ -44,9 +46,9 @@ public class RecoveryIT { private static final Logger LOGGER = LogManager.getLogger(); - private static final String PATH_ACTUAL = "target" + File.separator + "rttest" + File.separator; + private static final String PATH_ACTUAL = joinPath("target", "rttest"); private static final String PATH_BASE = "src/test/resources/transactionts/"; - private static final String HDFS_BASE = "../asterix-app/"; + private static final File HDFS_BASE = new File("..", "asterix-app"); private TestCaseContext tcCtx; private static File asterixInstallerPath; private static File installerTargetPath; @@ -68,17 +70,17 @@ File outdir = new File(PATH_ACTUAL); outdir.mkdirs(); - File externalTestsJar = - new File(StringUtils.join(new String[] { "..", "asterix-external-data", "target" }, File.separator)) - .listFiles((dir, name) -> name.matches("asterix-external-data-.*-tests.jar"))[0]; + File externalTestsJar = Objects.requireNonNull(new File(joinPath("..", "asterix-external-data", "target")) + .listFiles((dir, name) -> name.matches("asterix-external-data-.*-tests.jar")))[0]; asterixInstallerPath = new File(System.getProperty("user.dir")); installerTargetPath = new File(new File(asterixInstallerPath.getParentFile(), "asterix-server"), "target"); reportPath = new File(installerTargetPath, "failsafe-reports").getAbsolutePath(); - ncServiceSubDirName = - installerTargetPath.list((dir, name) -> name.matches("asterix-server.*binary-assembly"))[0]; + ncServiceSubDirName = Objects.requireNonNull( + installerTargetPath.list((dir, name) -> name.matches("asterix-server.*binary-assembly")))[0]; ncServiceSubPath = new File(installerTargetPath, ncServiceSubDirName).getAbsolutePath(); - ncServiceHomeDirName = new File(ncServiceSubPath).list(((dir, name) -> name.matches("apache-asterixdb.*")))[0]; + ncServiceHomeDirName = Objects.requireNonNull( + new File(ncServiceSubPath).list(((dir, name) -> name.matches("apache-asterixdb.*"))))[0]; ncServiceHomePath = new File(ncServiceSubPath, ncServiceHomeDirName).getAbsolutePath(); LOGGER.info("NCSERVICE_HOME=" + ncServiceHomePath); @@ -89,14 +91,12 @@ env = pb.environment(); env.put("NCSERVICE_HOME", ncServiceHomePath); env.put("JAVA_HOME", System.getProperty("java.home")); - scriptHomePath = asterixInstallerPath + File.separator + "src" + File.separator + "test" + File.separator - + "resources" + File.separator + "transactionts" + File.separator + "scripts"; + scriptHomePath = + joinPath(asterixInstallerPath.getPath(), "src", "test", "resources", "transactionts", "scripts"); env.put("SCRIPT_HOME", scriptHomePath); - TestExecutor.executeScript(pb, - scriptHomePath + File.separator + "setup_teardown" + File.separator + "configure_and_validate.sh"); - TestExecutor.executeScript(pb, - scriptHomePath + File.separator + "setup_teardown" + File.separator + "stop_and_delete.sh"); + TestExecutor.executeScript(pb, joinPath(scriptHomePath, "setup_teardown", "configure_and_validate.sh")); + TestExecutor.executeScript(pb, joinPath(scriptHomePath, "setup_teardown", "stop_and_delete.sh")); HDFSCluster.getInstance().setup(HDFS_BASE); } @@ -104,11 +104,9 @@ public static void tearDown() throws Exception { File outdir = new File(PATH_ACTUAL); FileUtils.deleteDirectory(outdir); - File dataCopyDir = - new File(ncServiceHomePath + File.separator + ".." + File.separator + ".." + File.separator + "data"); + File dataCopyDir = new File(joinPath(ncServiceHomePath, "..", "..", "data")); FileUtils.deleteDirectory(dataCopyDir); - TestExecutor.executeScript(pb, - scriptHomePath + File.separator + "setup_teardown" + File.separator + "stop_and_delete.sh"); + TestExecutor.executeScript(pb, joinPath(scriptHomePath, "setup_teardown", "stop_and_delete.sh")); HDFSCluster.getInstance().cleanup(); } diff --git a/asterixdb/asterix-server/src/test/java/org/apache/asterix/test/server/ReplicationIT.java b/asterixdb/asterix-server/src/test/java/org/apache/asterix/test/server/ReplicationIT.java index 0f058bc..1de71c4 100644 --- a/asterixdb/asterix-server/src/test/java/org/apache/asterix/test/server/ReplicationIT.java +++ b/asterixdb/asterix-server/src/test/java/org/apache/asterix/test/server/ReplicationIT.java @@ -18,11 +18,8 @@ */ package org.apache.asterix.test.server; -import static org.apache.asterix.test.server.NCServiceExecutionIT.APP_HOME; -import static org.apache.asterix.test.server.NCServiceExecutionIT.ASTERIX_APP_DIR; -import static org.apache.asterix.test.server.NCServiceExecutionIT.INSTANCE_DIR; -import static org.apache.asterix.test.server.NCServiceExecutionIT.LOG_DIR; -import static org.apache.asterix.test.server.NCServiceExecutionIT.TARGET_DIR; +import static org.apache.asterix.test.server.NCServiceExecutionIT.*; +import static org.apache.hyracks.util.file.FileUtil.joinPath; import java.io.File; import java.net.InetAddress; @@ -37,12 +34,11 @@ import org.apache.asterix.test.common.TestExecutor; import org.apache.asterix.testframework.context.TestCaseContext; import org.apache.commons.io.FileUtils; -import org.apache.commons.lang3.StringUtils; import org.apache.hyracks.test.server.process.HyracksVirtualCluster; -import org.apache.hyracks.util.file.FileUtil; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.junit.After; +import org.junit.AfterClass; import org.junit.Before; import org.junit.Rule; import org.junit.Test; @@ -53,13 +49,11 @@ @RunWith(Parameterized.class) public class ReplicationIT { - private static final String PATH_BASE = - FileUtil.joinPath("src", "test", "resources", "integrationts", "replication"); - private static final String CONF_DIR = - StringUtils.join(new String[] { TARGET_DIR, "test-classes", "ReplicationIT" }, File.separator); - private static final String PATH_ACTUAL = FileUtil.joinPath("target", "ittest"); + private static final String PATH_BASE = joinPath("src", "test", "resources", "integrationts", "replication"); + private static final String CONF_DIR = joinPath(TARGET_DIR, "test-classes", "ReplicationIT"); + private static final String PATH_ACTUAL = joinPath("target", "ittest"); private static final Logger LOGGER = LogManager.getLogger(); - private static String reportPath = new File(FileUtil.joinPath("target", "failsafe-reports")).getAbsolutePath(); + private static File reportPath = new File("target", "failsafe-reports"); private static final TestExecutor testExecutor = new TestExecutor(); private static HyracksVirtualCluster cluster; @@ -82,7 +76,7 @@ } @Rule - public TestRule retainLogs = new RetainLogsRule(NCServiceExecutionIT.ASTERIX_APP_DIR, reportPath, this); + public TestRule retainLogs = new RetainLogsRule(ASTERIX_APP_DIR, reportPath, this); @Before public void before() throws Exception { @@ -92,12 +86,12 @@ FileUtils.deleteDirectory(instanceDir); } - cluster = new HyracksVirtualCluster(new File(APP_HOME), new File(ASTERIX_APP_DIR)); - cluster.addNCService(new File(CONF_DIR, "ncservice1.conf"), new File(LOG_DIR, "ncservice1.log")); - cluster.addNCService(new File(CONF_DIR, "ncservice2.conf"), new File(LOG_DIR, "ncservice2.log")); + cluster = new HyracksVirtualCluster(APP_HOME, ASTERIX_APP_DIR); + cluster.addNCService(new File(CONF_DIR, "ncservice1.conf"), null); + cluster.addNCService(new File(CONF_DIR, "ncservice2.conf"), null); // Start CC - cluster.start(new File(CONF_DIR, "cc.conf"), new File(LOG_DIR, "cc.log")); + cluster.start(new File(CONF_DIR, "cc.conf"), null); LOGGER.info("Instance created."); testExecutor.waitForClusterActive(30, TimeUnit.SECONDS); LOGGER.info("Instance is in ACTIVE state."); @@ -110,6 +104,12 @@ LOGGER.info("Instance destroyed."); } + @AfterClass + public static void checkLogFiles() { + NCServiceExecutionIT.checkLogFiles(new File(TARGET_DIR, ReplicationIT.class.getSimpleName()), "asterix_nc1", + "asterix_nc2"); + } + @Test public void test() throws Exception { testExecutor.executeTest(PATH_ACTUAL, tcCtx, null, false); diff --git a/asterixdb/asterix-server/src/test/resources/NCServiceExecutionIT/cc.conf b/asterixdb/asterix-server/src/test/resources/NCServiceExecutionIT/cc.conf index 2a92c5d..1c6d574 100644 --- a/asterixdb/asterix-server/src/test/resources/NCServiceExecutionIT/cc.conf +++ b/asterixdb/asterix-server/src/test/resources/NCServiceExecutionIT/cc.conf @@ -46,5 +46,5 @@ heartbeat.max.misses=25 [common] -log.dir = ../asterix-server/target/failsafe-reports/ +log.dir = ../asterix-server/target/NCServiceExecutionIT log.level = INFO \ No newline at end of file diff --git a/asterixdb/asterix-server/src/test/resources/NCServiceExecutionIT/ncservice1.conf b/asterixdb/asterix-server/src/test/resources/NCServiceExecutionIT/ncservice1.conf index ba10142..e178cfa 100644 --- a/asterixdb/asterix-server/src/test/resources/NCServiceExecutionIT/ncservice1.conf +++ b/asterixdb/asterix-server/src/test/resources/NCServiceExecutionIT/ncservice1.conf @@ -16,5 +16,4 @@ ; under the License. [ncservice] -logdir=../asterix-server/target/failsafe-reports - +logdir=- diff --git a/asterixdb/asterix-server/src/test/resources/NCServiceExecutionIT/ncservice2.conf b/asterixdb/asterix-server/src/test/resources/NCServiceExecutionIT/ncservice2.conf index 2036584..43aecf0 100644 --- a/asterixdb/asterix-server/src/test/resources/NCServiceExecutionIT/ncservice2.conf +++ b/asterixdb/asterix-server/src/test/resources/NCServiceExecutionIT/ncservice2.conf @@ -16,6 +16,5 @@ ; under the License. [ncservice] -logdir=../asterix-server/target/failsafe-reports +logdir=- port=9091 - diff --git a/asterixdb/asterix-server/src/test/resources/NcLifecycleIT/cc.conf b/asterixdb/asterix-server/src/test/resources/NcLifecycleIT/cc.conf index dc80e56..a91622b 100644 --- a/asterixdb/asterix-server/src/test/resources/NcLifecycleIT/cc.conf +++ b/asterixdb/asterix-server/src/test/resources/NcLifecycleIT/cc.conf @@ -46,3 +46,4 @@ [common] log.level = INFO +log.dir=../asterix-server/target/NcLifecycleIT diff --git a/asterixdb/asterix-server/src/test/resources/NcLifecycleIT/ncservice1.conf b/asterixdb/asterix-server/src/test/resources/NcLifecycleIT/ncservice1.conf index ba10142..e178cfa 100644 --- a/asterixdb/asterix-server/src/test/resources/NcLifecycleIT/ncservice1.conf +++ b/asterixdb/asterix-server/src/test/resources/NcLifecycleIT/ncservice1.conf @@ -16,5 +16,4 @@ ; under the License. [ncservice] -logdir=../asterix-server/target/failsafe-reports - +logdir=- diff --git a/asterixdb/asterix-server/src/test/resources/NcLifecycleIT/ncservice2.conf b/asterixdb/asterix-server/src/test/resources/NcLifecycleIT/ncservice2.conf index 2036584..43aecf0 100644 --- a/asterixdb/asterix-server/src/test/resources/NcLifecycleIT/ncservice2.conf +++ b/asterixdb/asterix-server/src/test/resources/NcLifecycleIT/ncservice2.conf @@ -16,6 +16,5 @@ ; under the License. [ncservice] -logdir=../asterix-server/target/failsafe-reports +logdir=- port=9091 - diff --git a/asterixdb/asterix-server/src/test/resources/ReplicationIT/cc.conf b/asterixdb/asterix-server/src/test/resources/ReplicationIT/cc.conf index 19e951f..b88cbd4 100644 --- a/asterixdb/asterix-server/src/test/resources/ReplicationIT/cc.conf +++ b/asterixdb/asterix-server/src/test/resources/ReplicationIT/cc.conf @@ -47,6 +47,7 @@ [common] log.level = INFO +log.dir=../asterix-server/target/ReplicationIT replication.enabled=true replication.strategy=all replication.factor=2 diff --git a/asterixdb/asterix-server/src/test/resources/ReplicationIT/ncservice1.conf b/asterixdb/asterix-server/src/test/resources/ReplicationIT/ncservice1.conf index ba10142..e178cfa 100644 --- a/asterixdb/asterix-server/src/test/resources/ReplicationIT/ncservice1.conf +++ b/asterixdb/asterix-server/src/test/resources/ReplicationIT/ncservice1.conf @@ -16,5 +16,4 @@ ; under the License. [ncservice] -logdir=../asterix-server/target/failsafe-reports - +logdir=- diff --git a/asterixdb/asterix-server/src/test/resources/ReplicationIT/ncservice2.conf b/asterixdb/asterix-server/src/test/resources/ReplicationIT/ncservice2.conf index 2036584..43aecf0 100644 --- a/asterixdb/asterix-server/src/test/resources/ReplicationIT/ncservice2.conf +++ b/asterixdb/asterix-server/src/test/resources/ReplicationIT/ncservice2.conf @@ -16,6 +16,5 @@ ; under the License. [ncservice] -logdir=../asterix-server/target/failsafe-reports +logdir=- port=9091 - diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/CCLogConfigurationFactory.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/CCLogConfigurationFactory.java index 22dea9f..a635084 100644 --- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/CCLogConfigurationFactory.java +++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/CCLogConfigurationFactory.java @@ -23,6 +23,8 @@ import org.apache.hyracks.control.common.controllers.CCConfig; import org.apache.logging.log4j.Level; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.core.LoggerContext; import org.apache.logging.log4j.core.config.Configuration; import org.apache.logging.log4j.core.config.ConfigurationFactory; @@ -34,6 +36,7 @@ import org.apache.logging.log4j.core.config.builder.impl.BuiltConfiguration; public class CCLogConfigurationFactory extends ConfigurationFactory { + private static final Logger LOGGER = LogManager.getLogger(); private CCConfig config; public CCLogConfigurationFactory(CCConfig config) { @@ -42,6 +45,8 @@ public Configuration createConfiguration(ConfigurationBuilder<BuiltConfiguration> builder) { File logDir = new File(config.getLogDir()); + File ccLog = new File(logDir, "cc.log"); + LOGGER.warn("logs are being redirected to: {}", ccLog::getAbsolutePath); builder.setStatusLevel(Level.WARN); builder.setConfigurationName("RollingBuilder"); // create a rolling file appender @@ -50,10 +55,9 @@ ComponentBuilder triggeringPolicy = builder.newComponent("Policies") .addComponent(builder.newComponent("CronTriggeringPolicy").addAttribute("schedule", "0 0 0 * * ?")) .addComponent(builder.newComponent("SizeBasedTriggeringPolicy").addAttribute("size", "50M")); - AppenderComponentBuilder defaultRoll = - builder.newAppender("default", "RollingFile").addAttribute("fileName", new File(logDir, "cc.log")) - .addAttribute("filePattern", new File(logDir, "cc-%d{MM-dd-yy}.log.gz")).add(defaultLayout) - .addComponent(triggeringPolicy); + AppenderComponentBuilder defaultRoll = builder.newAppender("default", "RollingFile") + .addAttribute("fileName", ccLog).addAttribute("filePattern", new File(logDir, "cc-%d{MM-dd-yy}.log.gz")) + .add(defaultLayout).addComponent(triggeringPolicy); builder.add(defaultRoll); // create the new logger diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/NodeControllerData.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/NodeControllerData.java index 24a3e57..c37acab 100644 --- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/NodeControllerData.java +++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/NodeControllerData.java @@ -79,7 +79,7 @@ private final Map<String, String> systemProperties; - private final long pid; + private final int pid; private final HeartbeatSchema hbSchema; diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/NodeRegistration.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/NodeRegistration.java index 474bc0a..437b001 100644 --- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/NodeRegistration.java +++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/NodeRegistration.java @@ -75,7 +75,7 @@ private final NetworkAddress messagingPort; - private final long pid; + private final int pid; private final NodeCapacity capacity; @@ -190,7 +190,7 @@ return messagingPort; } - public long getPid() { + public int getPid() { return pid; } } diff --git a/asterixdb/asterix-app/src/main/resources/log4j2.xml b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/resources/log4j2.xml similarity index 100% rename from asterixdb/asterix-app/src/main/resources/log4j2.xml rename to hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/resources/log4j2.xml diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NCLogConfigurationFactory.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NCLogConfigurationFactory.java index a657d6b..fad6b3e 100644 --- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NCLogConfigurationFactory.java +++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NCLogConfigurationFactory.java @@ -23,6 +23,8 @@ import org.apache.hyracks.control.common.controllers.NCConfig; import org.apache.logging.log4j.Level; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.core.LoggerContext; import org.apache.logging.log4j.core.config.Configuration; import org.apache.logging.log4j.core.config.ConfigurationFactory; @@ -34,6 +36,7 @@ import org.apache.logging.log4j.core.config.builder.impl.BuiltConfiguration; public class NCLogConfigurationFactory extends ConfigurationFactory { + private static final Logger LOGGER = LogManager.getLogger(); private NCConfig config; public NCLogConfigurationFactory(NCConfig config) { @@ -43,6 +46,8 @@ public Configuration createConfiguration(ConfigurationBuilder<BuiltConfiguration> builder) { String nodeId = config.getNodeId(); File logDir = new File(config.getLogDir()); + File ncLog = new File(logDir, "nc-" + nodeId + ".log"); + LOGGER.warn("logs are being redirected to: {}", ncLog::getAbsolutePath); builder.setStatusLevel(Level.WARN); builder.setConfigurationName("RollingBuilder"); // create a rolling file appender @@ -51,11 +56,11 @@ ComponentBuilder triggeringPolicy = builder.newComponent("Policies") .addComponent(builder.newComponent("CronTriggeringPolicy").addAttribute("schedule", "0 0 0 * * ?")) .addComponent(builder.newComponent("SizeBasedTriggeringPolicy").addAttribute("size", "50M")); - AppenderComponentBuilder defaultRoll = builder.newAppender("default", "RollingFile") - .addAttribute("fileName", new File(logDir, "nc-" + nodeId + ".log").getAbsolutePath()) - .addAttribute("filePattern", - new File(logDir, "nc-" + nodeId + "-%d{MM-dd-yy-ss}.log.gz").getAbsolutePath()) - .add(defaultLayout).addComponent(triggeringPolicy); + AppenderComponentBuilder defaultRoll = + builder.newAppender("default", "RollingFile").addAttribute("fileName", ncLog.getAbsolutePath()) + .addAttribute("filePattern", + new File(logDir, "nc-" + nodeId + "-%d{MM-dd-yy-ss}.log.gz").getAbsolutePath()) + .add(defaultLayout).addComponent(triggeringPolicy); builder.add(defaultRoll); // create the new logger @@ -74,9 +79,9 @@ LayoutComponentBuilder traceLayout = builder.newLayout("PatternLayout").addAttribute("pattern", "%m,%n") .addAttribute("header", "[").addAttribute("footer", "]"); AppenderComponentBuilder traceRoll = builder.newAppender("trace", "RollingFile") - .addAttribute("fileName", logDir + "trace-" + nodeId + ".log") - .addAttribute("filePattern", logDir + "trace-" + nodeId + "-%d{MM-dd-yy-ss}.log.gz").add(traceLayout) - .addComponent(triggeringPolicy); + .addAttribute("fileName", new File(logDir, "trace-" + nodeId + ".log")) + .addAttribute("filePattern", new File(logDir, "trace-" + nodeId + "-%d{MM-dd-yy-ss}.log.gz")) + .add(traceLayout).addComponent(triggeringPolicy); builder.add(traceRoll); builder.add(builder.newLogger("org.apache.hyracks.util.trace.Tracer.Traces", Level.forName("TRACER", 570)) .add(builder.newAppenderRef("trace")).addAttribute("additivity", false)); diff --git a/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/HttpServer.java b/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/HttpServer.java index 94a6440..152f6c7 100644 --- a/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/HttpServer.java +++ b/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/HttpServer.java @@ -50,8 +50,6 @@ import io.netty.handler.codec.http.FullHttpRequest; import io.netty.handler.logging.LogLevel; import io.netty.handler.logging.LoggingHandler; -import io.netty.util.internal.logging.InternalLoggerFactory; -import io.netty.util.internal.logging.Log4J2LoggerFactory; public class HttpServer { // Constants @@ -83,10 +81,6 @@ private volatile Channel channel; private Throwable cause; private HttpServerConfig config; - - static { - InternalLoggerFactory.setDefaultFactory(Log4J2LoggerFactory.INSTANCE); - } public HttpServer(EventLoopGroup bossGroup, EventLoopGroup workerGroup, int port, HttpServerConfig config) { this(bossGroup, workerGroup, new InetSocketAddress(port), config, null); diff --git a/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/WebManager.java b/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/WebManager.java index 55741e4..b8347ae 100644 --- a/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/WebManager.java +++ b/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/WebManager.java @@ -24,12 +24,18 @@ import io.netty.channel.EventLoopGroup; import io.netty.channel.nio.NioEventLoopGroup; +import io.netty.util.internal.logging.Log4J2LoggerFactory; public class WebManager { private final List<HttpServer> servers; private final EventLoopGroup bosses; private final EventLoopGroup workers; + static { + // bootstrap netty with log4j2 logging + io.netty.util.internal.logging.InternalLoggerFactory.setDefaultFactory(Log4J2LoggerFactory.INSTANCE); + } + /** * Create a web manager with number of bosses = 1 * and number of workers = MultithreadEventLoopGroup.DEFAULT_EVENT_LOOP_THREADS diff --git a/hyracks-fullstack/hyracks/hyracks-server/pom.xml b/hyracks-fullstack/hyracks/hyracks-server/pom.xml index 7e566d6..6c5639f 100644 --- a/hyracks-fullstack/hyracks/hyracks-server/pom.xml +++ b/hyracks-fullstack/hyracks/hyracks-server/pom.xml @@ -185,9 +185,5 @@ <groupId>org.apache.logging.log4j</groupId> <artifactId>log4j-api</artifactId> </dependency> - <dependency> - <groupId>commons-io</groupId> - <artifactId>commons-io</artifactId> - </dependency> </dependencies> </project> diff --git a/hyracks-fullstack/hyracks/hyracks-server/src/test/java/org/apache/hyracks/test/server/NCServiceIT.java b/hyracks-fullstack/hyracks/hyracks-server/src/test/java/org/apache/hyracks/test/server/NCServiceIT.java index 5cbc5b4..cd0359f 100644 --- a/hyracks-fullstack/hyracks/hyracks-server/src/test/java/org/apache/hyracks/test/server/NCServiceIT.java +++ b/hyracks-fullstack/hyracks/hyracks-server/src/test/java/org/apache/hyracks/test/server/NCServiceIT.java @@ -21,11 +21,8 @@ import java.io.File; import java.io.IOException; import java.net.InetAddress; -import java.nio.file.Files; -import java.nio.file.StandardOpenOption; import java.util.Iterator; -import org.apache.commons.io.FileUtils; import org.apache.http.HttpResponse; import org.apache.http.HttpStatus; import org.apache.http.client.HttpClient; @@ -58,13 +55,9 @@ @BeforeClass public static void setUp() throws Exception { cluster = new HyracksVirtualCluster(new File(APP_HOME), null); - File tempConf = new File(TARGET_DIR, "cc.conf"); - FileUtils.copyFile(new File(RESOURCE_DIR, "cc.conf"), tempConf); - Files.write(tempConf.toPath(), ("log.dir: " + LOG_DIR).getBytes(), StandardOpenOption.APPEND); - File log4jPath = new File(FileUtil.joinPath("..", "..", "src", "test", "resources", "log4j2-hyracks-test.xml")); - cluster.addNCService(new File(RESOURCE_DIR, "nc-red.conf"), new File(LOG_DIR, "nc-red.log"), log4jPath); - cluster.addNCService(new File(RESOURCE_DIR, "nc-blue.conf"), new File(LOG_DIR, "nc-blue.log"), log4jPath); + cluster.addNCService(new File(RESOURCE_DIR, "nc-red.conf"), null); + cluster.addNCService(new File(RESOURCE_DIR, "nc-blue.conf"), null); try { Thread.sleep(2000); @@ -72,7 +65,7 @@ } // Start CC - cluster.start(tempConf, new File(LOG_DIR, "cc.log"), log4jPath); + cluster.start(new File(RESOURCE_DIR, "cc.conf"), null); try { Thread.sleep(10000); diff --git a/hyracks-fullstack/hyracks/hyracks-server/src/test/java/org/apache/hyracks/test/server/process/HyracksCCProcess.java b/hyracks-fullstack/hyracks/hyracks-server/src/test/java/org/apache/hyracks/test/server/process/HyracksCCProcess.java index a79d033..0b529fb 100644 --- a/hyracks-fullstack/hyracks/hyracks-server/src/test/java/org/apache/hyracks/test/server/process/HyracksCCProcess.java +++ b/hyracks-fullstack/hyracks/hyracks-server/src/test/java/org/apache/hyracks/test/server/process/HyracksCCProcess.java @@ -25,16 +25,12 @@ public class HyracksCCProcess extends HyracksServerProcess { - public HyracksCCProcess(File configFile, File logFile, File appHome, File workingDir) { + HyracksCCProcess(File configFile, File logFile, File appHome, File workingDir) { + super(" cc"); this.configFile = configFile; this.logFile = logFile; this.appHome = appHome; this.workingDir = workingDir; - } - - public HyracksCCProcess(File configFile, File logFile, File appHome, File workingDir, File log4jPath) { - this(configFile, logFile, appHome, workingDir); - args.add("-Dlog4j.configurationFile=file://" + log4jPath.getAbsolutePath()); } @Override diff --git a/hyracks-fullstack/hyracks/hyracks-server/src/test/java/org/apache/hyracks/test/server/process/HyracksNCServiceProcess.java b/hyracks-fullstack/hyracks/hyracks-server/src/test/java/org/apache/hyracks/test/server/process/HyracksNCServiceProcess.java index d0e0244..958eb9a 100644 --- a/hyracks-fullstack/hyracks/hyracks-server/src/test/java/org/apache/hyracks/test/server/process/HyracksNCServiceProcess.java +++ b/hyracks-fullstack/hyracks/hyracks-server/src/test/java/org/apache/hyracks/test/server/process/HyracksNCServiceProcess.java @@ -20,21 +20,19 @@ import java.io.File; import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; import org.apache.hyracks.control.nc.service.NCService; public class HyracksNCServiceProcess extends HyracksServerProcess { + private static final AtomicInteger ncServiceCounter = new AtomicInteger(); - public HyracksNCServiceProcess(File configFile, File logFile, File appHome, File workingDir) { + HyracksNCServiceProcess(File configFile, File logFile, File appHome, File workingDir) { + super("nc" + ncServiceCounter.incrementAndGet()); this.configFile = configFile; this.logFile = logFile; this.appHome = appHome; this.workingDir = workingDir; - } - - public HyracksNCServiceProcess(File configFile, File logFile, File appHome, File workingDir, File log4jPath) { - this(configFile, logFile, appHome, workingDir); - args.add("-Dlog4j.configurationFile=file://" + log4jPath.getAbsolutePath()); } @Override diff --git a/hyracks-fullstack/hyracks/hyracks-server/src/test/java/org/apache/hyracks/test/server/process/HyracksServerProcess.java b/hyracks-fullstack/hyracks/hyracks-server/src/test/java/org/apache/hyracks/test/server/process/HyracksServerProcess.java index a8c363b..59ab4f7 100644 --- a/hyracks-fullstack/hyracks/hyracks-server/src/test/java/org/apache/hyracks/test/server/process/HyracksServerProcess.java +++ b/hyracks-fullstack/hyracks/hyracks-server/src/test/java/org/apache/hyracks/test/server/process/HyracksServerProcess.java @@ -18,9 +18,12 @@ */ package org.apache.hyracks.test.server.process; +import java.io.BufferedReader; import java.io.File; import java.io.FileWriter; import java.io.IOException; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -32,14 +35,21 @@ abstract class HyracksServerProcess { private static final Logger LOGGER = LogManager.getLogger(); + protected final String processName; protected Process process; + protected Thread pipeThread; protected File configFile = null; protected File logFile = null; protected File appHome = null; protected File workingDir = null; protected List<String> args = new ArrayList<>(); + protected HyracksServerProcess(String processName) { + this.processName = processName; + } + public void start() throws IOException { + String[] cmd = buildCommand(); if (LOGGER.isInfoEnabled()) { LOGGER.info("Starting command: " + Arrays.toString(cmd)); @@ -47,33 +57,44 @@ ProcessBuilder pb = new ProcessBuilder(cmd); pb.redirectErrorStream(true); + pb.directory(workingDir); if (logFile != null) { - if (LOGGER.isInfoEnabled()) { - LOGGER.info("Logging to: " + logFile.getCanonicalPath()); - } + LOGGER.info("Logging to: " + logFile.getCanonicalPath()); logFile.getParentFile().mkdirs(); try (FileWriter writer = new FileWriter(logFile, true)) { writer.write("---------------------\n"); } pb.redirectOutput(ProcessBuilder.Redirect.appendTo(logFile)); + process = pb.start(); } else { - if (LOGGER.isInfoEnabled()) { - LOGGER.info("Logfile not set, subprocess will output to stdout"); - } + pb.redirectOutput(ProcessBuilder.Redirect.PIPE); + process = pb.start(); + pipeThread = new Thread(() -> { + try (BufferedReader reader = + new BufferedReader(new InputStreamReader(process.getInputStream(), StandardCharsets.UTF_8))) { + String line; + while ((line = reader.readLine()) != null) { + System.out.println(processName + ": " + line); + } + } catch (IOException e) { + LOGGER.debug("exception reading process pipe", e); + } + }); + pipeThread.start(); } - pb.directory(workingDir); - process = pb.start(); } public void stop() { process.destroy(); try { boolean success = process.waitFor(30, TimeUnit.SECONDS); - if (LOGGER.isWarnEnabled()) { - LOGGER.warn("Killing unresponsive NC Process"); - } if (!success) { + LOGGER.warn("Killing unresponsive NC Process"); process.destroyForcibly(); + } + if (pipeThread != null) { + pipeThread.interrupt(); + pipeThread.join(); } } catch (InterruptedException e) { Thread.currentThread().interrupt(); @@ -106,7 +127,7 @@ cList.add(configFile.getAbsolutePath()); } addCmdLineArgs(cList); - return cList.toArray(new String[cList.size()]); + return cList.toArray(new String[0]); } protected void addJvmArgs(List<String> cList) { diff --git a/hyracks-fullstack/hyracks/hyracks-server/src/test/java/org/apache/hyracks/test/server/process/HyracksVirtualCluster.java b/hyracks-fullstack/hyracks/hyracks-server/src/test/java/org/apache/hyracks/test/server/process/HyracksVirtualCluster.java index 6c77628..062d429 100644 --- a/hyracks-fullstack/hyracks/hyracks-server/src/test/java/org/apache/hyracks/test/server/process/HyracksVirtualCluster.java +++ b/hyracks-fullstack/hyracks/hyracks-server/src/test/java/org/apache/hyracks/test/server/process/HyracksVirtualCluster.java @@ -65,22 +65,6 @@ } /** - * Creates and starts an NCService. - * - * @param configFile - * - full path to an ncservice.conf. May be null to accept all defaults. - * @throws IOException - * - if there are errors starting the process. - */ - public HyracksNCServiceProcess addNCService(File configFile, File logFile, File log4jConfig) throws IOException { - HyracksNCServiceProcess proc = - new HyracksNCServiceProcess(configFile, logFile, appHome, workingDir, log4jConfig); - proc.start(); - ncProcs.add(proc); - return proc; - } - - /** * Starts the CC, initializing the cluster. Expects that any NCs referenced * in the cluster configuration have already been started with addNCService(). * @@ -92,22 +76,6 @@ */ public HyracksCCProcess start(File ccConfigFile, File logFile) throws IOException { ccProc = new HyracksCCProcess(ccConfigFile, logFile, appHome, workingDir); - ccProc.start(); - return ccProc; - } - - /** - * Starts the CC, initializing the cluster. Expects that any NCs referenced - * in the cluster configuration have already been started with addNCService(). - * - * @param ccConfigFile - * - full path to a cluster conf file. May be null to accept all - * defaults, although this is seldom useful since there are no NCs. - * @throws IOException - * - if there are errors starting the process. - */ - public HyracksCCProcess start(File ccConfigFile, File logFile, File log4jConfig) throws IOException { - ccProc = new HyracksCCProcess(ccConfigFile, logFile, appHome, workingDir, log4jConfig); ccProc.start(); return ccProc; } diff --git a/hyracks-fullstack/hyracks/hyracks-server/src/test/resources/NCServiceIT/cc.conf b/hyracks-fullstack/hyracks/hyracks-server/src/test/resources/NCServiceIT/cc.conf index 9c80c7d..9b1a1cd 100644 --- a/hyracks-fullstack/hyracks/hyracks-server/src/test/resources/NCServiceIT/cc.conf +++ b/hyracks-fullstack/hyracks/hyracks-server/src/test/resources/NCServiceIT/cc.conf @@ -30,3 +30,4 @@ console.listen.port = 12345 [common] +log.dir=target/NCServiceIT diff --git a/hyracks-fullstack/hyracks/hyracks-server/src/test/resources/NCServiceIT/nc-blue.conf b/hyracks-fullstack/hyracks/hyracks-server/src/test/resources/NCServiceIT/nc-blue.conf index 6eb38dd..baccd46 100644 --- a/hyracks-fullstack/hyracks/hyracks-server/src/test/resources/NCServiceIT/nc-blue.conf +++ b/hyracks-fullstack/hyracks/hyracks-server/src/test/resources/NCServiceIT/nc-blue.conf @@ -18,3 +18,4 @@ [ncservice] address=127.0.0.1 port=9091 +logdir=- \ No newline at end of file diff --git a/hyracks-fullstack/hyracks/hyracks-server/src/test/resources/NCServiceIT/nc-red.conf b/hyracks-fullstack/hyracks/hyracks-server/src/test/resources/NCServiceIT/nc-red.conf index 286bd32..7616b37 100644 --- a/hyracks-fullstack/hyracks/hyracks-server/src/test/resources/NCServiceIT/nc-red.conf +++ b/hyracks-fullstack/hyracks/hyracks-server/src/test/resources/NCServiceIT/nc-red.conf @@ -18,3 +18,4 @@ [ncservice] address=127.0.0.1 port=9090 +logdir=- \ No newline at end of file diff --git a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/PidHelper.java b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/PidHelper.java index 46e77e3..5a8edbd 100644 --- a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/PidHelper.java +++ b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/PidHelper.java @@ -21,7 +21,6 @@ import java.lang.management.ManagementFactory; import java.lang.management.RuntimeMXBean; import java.lang.reflect.Field; -import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import org.apache.logging.log4j.Level; @@ -35,20 +34,11 @@ private PidHelper() { } - public static long getPid() { + public static int getPid() { return getPid(ManagementFactory.getRuntimeMXBean()); } - public static long getPid(RuntimeMXBean runtimeMXBean) { - // TODO: replace with direct invoke of getPid() once compatibility is at JDK 10 or higher - try { - Method getPidMethod = runtimeMXBean.getClass().getMethod("getPid"); - return (Long) getPidMethod.invoke(runtimeMXBean); - } catch (NoSuchMethodException e) { - LOGGER.debug("ignoring exception trying to find getPid() (expected pre-JDK 10)", e); - } catch (IllegalAccessException | InvocationTargetException e) { - LOGGER.debug("ignoring exception trying to execute getPid()", e); - } + public static int getPid(RuntimeMXBean runtimeMXBean) { try { Field jvmField = runtimeMXBean.getClass().getDeclaredField("jvm"); jvmField.setAccessible(true); diff --git a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/trace/Event.java b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/trace/Event.java index b744198..b5fe3d3 100644 --- a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/trace/Event.java +++ b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/trace/Event.java @@ -28,14 +28,14 @@ public final long cat; public final ITracer.Phase ph; public final long ts; - public final long pid; + public final int pid; public final long tid; public final ITracer.Scope scope; public final String args; public final TraceCategoryRegistry registry; - private Event(String name, long cat, ITracer.Phase ph, long ts, long pid, long tid, ITracer.Scope scope, - String args, TraceCategoryRegistry registry) { + private Event(String name, long cat, ITracer.Phase ph, long ts, int pid, long tid, ITracer.Scope scope, String args, + TraceCategoryRegistry registry) { this.name = name; this.cat = cat; this.ph = ph; @@ -51,7 +51,7 @@ return (System.nanoTime() - NANOTIME_DELTA_TO_EPOCH) / 1000; } - public static Event create(String name, long cat, ITracer.Phase ph, long pid, long tid, ITracer.Scope scope, + public static Event create(String name, long cat, ITracer.Phase ph, int pid, long tid, ITracer.Scope scope, String args, TraceCategoryRegistry registry) { return new Event(name, cat, ph, timestamp(), pid, tid, scope, args, registry); } diff --git a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/trace/Tracer.java b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/trace/Tracer.java index c1c38cf..9019fdf 100644 --- a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/trace/Tracer.java +++ b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/trace/Tracer.java @@ -44,7 +44,7 @@ private long categories; private final TraceCategoryRegistry registry; - private static final long pid = PidHelper.getPid(); + private static final int pid = PidHelper.getPid(); public Tracer(String name, long categories, TraceCategoryRegistry registry) { final String traceLoggerName = Tracer.class.getName() + ".Traces." + name; -- To view, visit https://asterix-gerrit.ics.uci.edu/3311 To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-MessageType: merged Gerrit-Change-Id: Id264aede0f62558ad6e34355047c623a1d594692 Gerrit-PatchSet: 11 Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Owner: Michael Blow <[email protected]> Gerrit-Reviewer: Anon. E. Moose #1000171 Gerrit-Reviewer: Ian Maxon <[email protected]> Gerrit-Reviewer: Jenkins <[email protected]> Gerrit-Reviewer: Michael Blow <[email protected]> Gerrit-Reviewer: Murtadha Hubail <[email protected]> Gerrit-Reviewer: Till Westmann <[email protected]>
