Repository: sqoop Updated Branches: refs/heads/sqoop2 87515c56e -> 9aec8f965
SQOOP-2832: Sqoop2: Precommit: Create log files for individual tests (Jarek Jarcec Cecho via Colin Ma) Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/9aec8f96 Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/9aec8f96 Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/9aec8f96 Branch: refs/heads/sqoop2 Commit: 9aec8f965eacde9be04fd40eb53a664873bee4df Parents: 87515c5 Author: Colin Ma <[email protected]> Authored: Mon Feb 22 15:24:43 2016 +0800 Committer: Colin Ma <[email protected]> Committed: Mon Feb 22 15:24:43 2016 +0800 ---------------------------------------------------------------------- test/pom.xml | 1 + .../test/testng/ReconfigureLogListener.java | 118 +++++++++++++++++++ .../resources/connector-loading-tests-suite.xml | 1 + test/src/test/resources/hive-tests-suite.xml | 1 + .../test/resources/integration-tests-suite.xml | 1 + .../resources/new-integration-tests-suite.xml | 1 + test/src/test/resources/shell-tests-suite.xml | 1 + test/src/test/resources/tools-tests-suite.xml | 1 + test/src/test/resources/upgrade-tests-suite.xml | 1 + 9 files changed, 126 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/sqoop/blob/9aec8f96/test/pom.xml ---------------------------------------------------------------------- diff --git a/test/pom.xml b/test/pom.xml index 134bca1..1e88b34 100644 --- a/test/pom.xml +++ b/test/pom.xml @@ -209,6 +209,7 @@ limitations under the License. </excludes> <systemPropertyVariables> <sqoop.integration.tmpdir>${project.build.directory}</sqoop.integration.tmpdir> + <sqoop.integration.log>${project.build.directory}/surefire-reports</sqoop.integration.log> </systemPropertyVariables> </configuration> <executions> http://git-wip-us.apache.org/repos/asf/sqoop/blob/9aec8f96/test/src/main/java/org/apache/sqoop/test/testng/ReconfigureLogListener.java ---------------------------------------------------------------------- diff --git a/test/src/main/java/org/apache/sqoop/test/testng/ReconfigureLogListener.java b/test/src/main/java/org/apache/sqoop/test/testng/ReconfigureLogListener.java new file mode 100644 index 0000000..52e9bb9 --- /dev/null +++ b/test/src/main/java/org/apache/sqoop/test/testng/ReconfigureLogListener.java @@ -0,0 +1,118 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sqoop.test.testng; + +import org.apache.log4j.FileAppender; +import org.apache.log4j.Level; +import org.apache.log4j.Logger; +import org.apache.log4j.PatternLayout; +import org.testng.ITestResult; +import org.testng.TestListenerAdapter; + +import java.util.HashMap; +import java.util.Map; + +/** + * Sqoop is running as much tests as possible inside one suite to safe time starting + * miniclusters which is time consuming exercise (~40 seconds per single test class). + * That however means that we have one output log file that recently grown to more then + * 1GB in side and hence the usability has decreased. + * + * This listener will intercept each test and will reconfigure log4j to log directly + * into files rather then to console (that would be forwarded by maven surefire plugin to + * the normal log file). Each test will get it's own file which is easier for human to + * read. + * + * We're using a counter to order log files per execution order rather then per name as + * we can't guarantee log isolation entirely (e.g. some information relevant to test N + * can be in log file for test N-1). It's easier to open previous log if you immediately + * know what is the previous log. + */ [email protected]({"ST_WRITE_TO_STATIC_FROM_INSTANCE_METHOD"}) +public class ReconfigureLogListener extends TestListenerAdapter { + + /** + * Directory into which we should put all log files. + */ + private static final String TEST_LOG_DIRECTORY = System.getProperty("sqoop.integration.log"); + + /** + * We're purposely configuring some of the loggers into a different level then DEBUG + * as they are more spamming the log then being actually helpful. As with all logs, + * it can happen that they might be needed at some point, so rather then hardcoding + * the rules in code, we've chosen path to expose ability to override them on the + * command line. + */ + private static final String TEST_LOGGERS = System.getProperty("sqoop.integration.log.loggers", + "org.eclipse.jetty=INFO," + + "org.apache.directory=INFO," + + "org.apache.hadoop.ipc.Server=INFO," + + "org.apache.hadoop.hdfs=INFO," + + "org.apache.hadoop.security.SaslInputStream=INFO," + + "org.apache.hadoop.security.SaslRpcClient=INFO," + + "org.apache.hadoop.ipc.Client=INFO," + + "org.apache.hadoop.conf.Configuration=INFO" + ); + + // Parsed and cached variant of TEST_LOGGERS + private static Map<String, Level> loggerConfiguration; + static { + loggerConfiguration = new HashMap<>(); + for(String rule : TEST_LOGGERS.split(",")) { + String []split = rule.split("="); + if(split.length != 2) { + throw new RuntimeException("Incorrect rule, expected logger=level: " + rule); + } + + loggerConfiguration.put(split[0].trim(), Level.toLevel(split[1].trim())); + } + } + + /** + * Counter is increased for each test execution + */ + private static int counter = 0; + + /** + * On every test start, we'll start logging into different file + */ + @Override + public void onTestStart(ITestResult tr) { + // Reset log4j configuration on every test run + Logger.getRootLogger().getLoggerRepository().resetConfiguration(); + + // Get test # so that our resulting files are sorted by execution and not by name + int ourCounter = counter++; + + // Usual File appender + FileAppender appender = new FileAppender(); + appender.setName("Sqoop test dynamic logger"); + appender.setFile(TEST_LOG_DIRECTORY + "/" + String.format("%05d", ourCounter) + "_" + tr.getTestClass().getName() + "." + tr.getName() + ".txt"); + appender.setLayout(new PatternLayout("%d{ISO8601} [%t] %-5p %c %x - %m%n")); + appender.setImmediateFlush(true); + appender.setThreshold(Level.DEBUG); + appender.activateOptions(); + + // Different levels for various not-so important loggers + Logger.getRootLogger().addAppender(appender); + for(Map.Entry<String, Level> entry : loggerConfiguration.entrySet()) { + Logger.getLogger(entry.getKey()).setLevel(entry.getValue()); + } + } + +} http://git-wip-us.apache.org/repos/asf/sqoop/blob/9aec8f96/test/src/test/resources/connector-loading-tests-suite.xml ---------------------------------------------------------------------- diff --git a/test/src/test/resources/connector-loading-tests-suite.xml b/test/src/test/resources/connector-loading-tests-suite.xml index 02c1df3..c03fb4f 100644 --- a/test/src/test/resources/connector-loading-tests-suite.xml +++ b/test/src/test/resources/connector-loading-tests-suite.xml @@ -22,6 +22,7 @@ limitations under the License. <listeners> <listener class-name="org.apache.sqoop.test.testng.SqoopTestListener" /> + <listener class-name="org.apache.sqoop.test.testng.ReconfigureLogListener" /> </listeners> <test name="ConnectorLoadingTests"> http://git-wip-us.apache.org/repos/asf/sqoop/blob/9aec8f96/test/src/test/resources/hive-tests-suite.xml ---------------------------------------------------------------------- diff --git a/test/src/test/resources/hive-tests-suite.xml b/test/src/test/resources/hive-tests-suite.xml index 3e2b328..c65db04 100644 --- a/test/src/test/resources/hive-tests-suite.xml +++ b/test/src/test/resources/hive-tests-suite.xml @@ -22,6 +22,7 @@ limitations under the License. <listeners> <listener class-name="org.apache.sqoop.test.testng.SqoopTestListener" /> + <listener class-name="org.apache.sqoop.test.testng.ReconfigureLogListener" /> </listeners> <test name="HiveTests"> http://git-wip-us.apache.org/repos/asf/sqoop/blob/9aec8f96/test/src/test/resources/integration-tests-suite.xml ---------------------------------------------------------------------- diff --git a/test/src/test/resources/integration-tests-suite.xml b/test/src/test/resources/integration-tests-suite.xml index 73e0a77..91e47e4 100644 --- a/test/src/test/resources/integration-tests-suite.xml +++ b/test/src/test/resources/integration-tests-suite.xml @@ -22,6 +22,7 @@ limitations under the License. <listeners> <listener class-name="org.apache.sqoop.test.testng.SqoopTestListener" /> + <listener class-name="org.apache.sqoop.test.testng.ReconfigureLogListener" /> </listeners> <test name="ConnectorTests"> http://git-wip-us.apache.org/repos/asf/sqoop/blob/9aec8f96/test/src/test/resources/new-integration-tests-suite.xml ---------------------------------------------------------------------- diff --git a/test/src/test/resources/new-integration-tests-suite.xml b/test/src/test/resources/new-integration-tests-suite.xml index 96a1320..b470b62 100644 --- a/test/src/test/resources/new-integration-tests-suite.xml +++ b/test/src/test/resources/new-integration-tests-suite.xml @@ -22,6 +22,7 @@ limitations under the License. <listeners> <listener class-name="org.apache.sqoop.test.testng.SqoopTestListener" /> + <listener class-name="org.apache.sqoop.test.testng.ReconfigureLogListener" /> </listeners> <test name="ServerTests"> http://git-wip-us.apache.org/repos/asf/sqoop/blob/9aec8f96/test/src/test/resources/shell-tests-suite.xml ---------------------------------------------------------------------- diff --git a/test/src/test/resources/shell-tests-suite.xml b/test/src/test/resources/shell-tests-suite.xml index e2c0b09..565d2fd 100644 --- a/test/src/test/resources/shell-tests-suite.xml +++ b/test/src/test/resources/shell-tests-suite.xml @@ -22,6 +22,7 @@ limitations under the License. <listeners> <listener class-name="org.apache.sqoop.test.testng.SqoopTestListener" /> + <listener class-name="org.apache.sqoop.test.testng.ReconfigureLogListener" /> </listeners> <test name="ShellTests"> http://git-wip-us.apache.org/repos/asf/sqoop/blob/9aec8f96/test/src/test/resources/tools-tests-suite.xml ---------------------------------------------------------------------- diff --git a/test/src/test/resources/tools-tests-suite.xml b/test/src/test/resources/tools-tests-suite.xml index fee2121..ceb192e 100644 --- a/test/src/test/resources/tools-tests-suite.xml +++ b/test/src/test/resources/tools-tests-suite.xml @@ -22,6 +22,7 @@ limitations under the License. <listeners> <listener class-name="org.apache.sqoop.test.testng.SqoopTestListener" /> + <listener class-name="org.apache.sqoop.test.testng.ReconfigureLogListener" /> </listeners> <test name="ToolsTests"> http://git-wip-us.apache.org/repos/asf/sqoop/blob/9aec8f96/test/src/test/resources/upgrade-tests-suite.xml ---------------------------------------------------------------------- diff --git a/test/src/test/resources/upgrade-tests-suite.xml b/test/src/test/resources/upgrade-tests-suite.xml index 3318e67..f97a2d2 100644 --- a/test/src/test/resources/upgrade-tests-suite.xml +++ b/test/src/test/resources/upgrade-tests-suite.xml @@ -22,6 +22,7 @@ limitations under the License. <listeners> <listener class-name="org.apache.sqoop.test.testng.SqoopTestListener" /> + <listener class-name="org.apache.sqoop.test.testng.ReconfigureLogListener" /> </listeners> <test name="RepositoryTests">
