GEODE-2416: Collect together artifacts from individual servers into a single zip file
- GEODE-2414: Determine a mechanism to stream a zip file from server to locator - GEODE-2415: Write a function to return a zip file for a single server Project: http://git-wip-us.apache.org/repos/asf/geode/repo Commit: http://git-wip-us.apache.org/repos/asf/geode/commit/4c6f3695 Tree: http://git-wip-us.apache.org/repos/asf/geode/tree/4c6f3695 Diff: http://git-wip-us.apache.org/repos/asf/geode/diff/4c6f3695 Branch: refs/heads/develop Commit: 4c6f3695db9ca69f5c1952128eda91ea5dfe3ede Parents: e5121ab Author: Jared Stewart <[email protected]> Authored: Sun Feb 12 11:30:58 2017 -0800 Committer: Jinmei Liao <[email protected]> Committed: Fri Mar 3 16:03:49 2017 -0800 ---------------------------------------------------------------------- .../geode/management/internal/cli/CliUtil.java | 29 +- .../cli/commands/MiscellaneousCommands.java | 368 ++++--------------- .../cli/functions/ExportLogsFunction.java | 203 ++++++++++ .../cli/util/ExportLogsCacheWriter.java | 78 ++++ .../internal/cli/util/ExportLogsRepository.java | 39 ++ .../internal/cli/util/LogExporter.java | 136 +++++++ .../management/internal/cli/util/LogFilter.java | 113 ++++++ .../internal/cli/util/LogLevelExtractor.java | 65 ++++ .../management/internal/cli/util/MergeLogs.java | 89 ++++- .../internal/configuration/utils/ZipUtils.java | 8 +- .../internal/cli/commands/ExportLogsDUnit.java | 363 ++++++++++++++++++ ...laneousCommandsExportLogsPart1DUnitTest.java | 138 ------- ...laneousCommandsExportLogsPart2DUnitTest.java | 141 ------- ...laneousCommandsExportLogsPart3DUnitTest.java | 156 -------- ...laneousCommandsExportLogsPart4DUnitTest.java | 138 ------- .../ExportLogsFunctionIntegrationTest.java | 143 +++++++ .../internal/cli/util/LogExporterTest.java | 104 ++++++ .../internal/cli/util/LogFilterTest.java | 126 +++++++ .../cli/util/LogLevelExtractorTest.java | 61 +++ .../internal/cli/util/MergeLogsTest.java | 114 ++++++ .../configuration/EventTestCacheWriter.java | 36 ++ .../configuration/ZipUtilsJUnitTest.java | 16 + .../dunit/rules/GfshShellConnectionRule.java | 4 + .../test/dunit/rules/LocatorStarterRule.java | 1 + .../apache/geode/test/dunit/rules/Member.java | 2 +- .../cli/commands/CommandOverHttpDUnitTest.java | 6 +- 26 files changed, 1772 insertions(+), 905 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/geode/blob/4c6f3695/geode-core/src/main/java/org/apache/geode/management/internal/cli/CliUtil.java ---------------------------------------------------------------------- diff --git a/geode-core/src/main/java/org/apache/geode/management/internal/cli/CliUtil.java b/geode-core/src/main/java/org/apache/geode/management/internal/cli/CliUtil.java index 8cd098d..87a07a6 100755 --- a/geode-core/src/main/java/org/apache/geode/management/internal/cli/CliUtil.java +++ b/geode-core/src/main/java/org/apache/geode/management/internal/cli/CliUtil.java @@ -301,7 +301,27 @@ public class CliUtil { return matchingMembers; } + /** + * Finds all Members (including both servers and locators) which belong to the given arrays of groups or members. + */ + public static Set<DistributedMember> findMembersIncludingLocators(String[] groups, String[] members) { + Cache cache = CacheFactory.getAnyInstance(); + Set<DistributedMember> allMembers = getAllMembers(cache); + + return findMembers(allMembers, groups, members); + } + + /** + * Finds all Servers which belong to the given arrays of groups or members. Does not include locators. + */ public static Set<DistributedMember> findMembers(String[] groups, String[] members) { + Cache cache = CacheFactory.getAnyInstance(); + Set<DistributedMember> allNormalMembers = getAllNormalMembers(cache); + + return findMembers(allNormalMembers, groups, members); + } + + private static Set<DistributedMember> findMembers(Set<DistributedMember> membersToConsider, String[] groups, String[] members) { if (groups == null) { groups = new String[] {}; } @@ -310,21 +330,18 @@ public class CliUtil { members = new String[] {}; } - Cache cache = CacheFactory.getAnyInstance(); - if ((members.length > 0) && (groups.length > 0)) { throw new IllegalArgumentException(CliStrings.PROVIDE_EITHER_MEMBER_OR_GROUP_MESSAGE); } - Set<DistributedMember> allNormalMembers = getAllNormalMembers(cache); if (members.length == 0 && groups.length == 0) { - return allNormalMembers; + return membersToConsider; } Set<DistributedMember> matchingMembers = new HashSet<DistributedMember>(); // it will either go into this loop or the following loop, not both. for (String memberNameOrId : members) { - for (DistributedMember member : allNormalMembers) { + for (DistributedMember member : membersToConsider) { if (memberNameOrId.equalsIgnoreCase(member.getId()) || memberNameOrId.equals(member.getName())) { matchingMembers.add(member); @@ -333,7 +350,7 @@ public class CliUtil { } for (String group : groups) { - for (DistributedMember member : allNormalMembers) { + for (DistributedMember member : membersToConsider) { if (member.getGroups().contains(group)) { matchingMembers.add(member); } http://git-wip-us.apache.org/repos/asf/geode/blob/4c6f3695/geode-core/src/main/java/org/apache/geode/management/internal/cli/commands/MiscellaneousCommands.java ---------------------------------------------------------------------- diff --git a/geode-core/src/main/java/org/apache/geode/management/internal/cli/commands/MiscellaneousCommands.java b/geode-core/src/main/java/org/apache/geode/management/internal/cli/commands/MiscellaneousCommands.java index bbf0542..e720d09 100644 --- a/geode-core/src/main/java/org/apache/geode/management/internal/cli/commands/MiscellaneousCommands.java +++ b/geode-core/src/main/java/org/apache/geode/management/internal/cli/commands/MiscellaneousCommands.java @@ -24,10 +24,10 @@ import java.io.InputStream; import java.io.InputStreamReader; import java.io.OutputStream; import java.io.PrintWriter; -import java.sql.Time; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; import java.text.MessageFormat; -import java.text.ParseException; -import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -38,7 +38,6 @@ import java.util.List; import java.util.Map; import java.util.Map.Entry; import java.util.Set; -import java.util.StringTokenizer; import java.util.concurrent.Callable; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; @@ -50,9 +49,11 @@ import java.util.zip.DataFormatException; import java.util.zip.GZIPInputStream; import javax.management.ObjectName; +import org.apache.commons.io.FileUtils; import org.apache.geode.LogWriter; import org.apache.geode.cache.Cache; import org.apache.geode.cache.CacheFactory; +import org.apache.geode.cache.Region; import org.apache.geode.cache.execute.Execution; import org.apache.geode.cache.execute.Function; import org.apache.geode.cache.execute.FunctionException; @@ -65,8 +66,7 @@ import org.apache.geode.distributed.internal.deadlock.Dependency; import org.apache.geode.distributed.internal.deadlock.DependencyGraph; import org.apache.geode.distributed.internal.deadlock.GemFireDeadlockDetector; import org.apache.geode.internal.cache.GemFireCacheImpl; -import org.apache.geode.internal.i18n.LocalizedStrings; -import org.apache.geode.internal.logging.InternalLogWriter; +import org.apache.geode.internal.logging.LogService; import org.apache.geode.internal.logging.LogWriterImpl; import org.apache.geode.management.CacheServerMXBean; import org.apache.geode.management.DistributedRegionMXBean; @@ -89,9 +89,9 @@ import org.apache.geode.management.internal.cli.GfshParser; import org.apache.geode.management.internal.cli.LogWrapper; import org.apache.geode.management.internal.cli.domain.StackTracesPerMember; import org.apache.geode.management.internal.cli.functions.ChangeLogLevelFunction; +import org.apache.geode.management.internal.cli.functions.ExportLogsFunction; import org.apache.geode.management.internal.cli.functions.GarbageCollectionFunction; import org.apache.geode.management.internal.cli.functions.GetStackTracesFunction; -import org.apache.geode.management.internal.cli.functions.LogFileFunction; import org.apache.geode.management.internal.cli.functions.NetstatFunction; import org.apache.geode.management.internal.cli.functions.NetstatFunction.NetstatFunctionArgument; import org.apache.geode.management.internal.cli.functions.NetstatFunction.NetstatFunctionResult; @@ -109,17 +109,19 @@ import org.apache.geode.management.internal.cli.result.ResultDataException; import org.apache.geode.management.internal.cli.result.TabularResultData; import org.apache.geode.management.internal.cli.shell.Gfsh; import org.apache.geode.management.internal.cli.util.MergeLogs; +import org.apache.geode.management.internal.cli.util.ExportLogsCacheWriter; +import org.apache.geode.management.internal.configuration.utils.ZipUtils; import org.apache.geode.management.internal.security.ResourceOperation; import org.apache.geode.security.ResourcePermission.Operation; import org.apache.geode.security.ResourcePermission.Resource; +import org.apache.logging.log4j.Logger; import org.springframework.shell.core.CommandMarker; import org.springframework.shell.core.annotation.CliAvailabilityIndicator; import org.springframework.shell.core.annotation.CliCommand; import org.springframework.shell.core.annotation.CliOption; /** - * * @since GemFire 7.0 */ public class MiscellaneousCommands implements CommandMarker { @@ -145,8 +147,6 @@ public class MiscellaneousCommands implements CommandMarker { logger.info("Gfsh executing shutdown on members " + includeMembers); - - Callable<String> shutdownNodes = new Callable<String>() { @Override @@ -210,8 +210,6 @@ public class MiscellaneousCommands implements CommandMarker { locators.removeAll(dataNodes); - - if (!shutdownLocators && numDataNodes == 0) { return ResultBuilder.createInfoResult(CliStrings.SHUTDOWN__MSG__NO_DATA_NODE_FOUND); } @@ -248,8 +246,8 @@ public class MiscellaneousCommands implements CommandMarker { } if (locators.contains(manager) && !shutdownLocators) { // This means manager is a locator and - // shutdownLocators is false. Hence we - // should not stop the manager + // shutdownLocators is false. Hence we + // should not stop the manager return ResultBuilder.createInfoResult("Shutdown is triggered"); } // now shut down this manager @@ -272,12 +270,9 @@ public class MiscellaneousCommands implements CommandMarker { } /** - * * @param timeout user specified timeout * @param nodesToBeStopped list of nodes to be stopped * @return Elapsed time to shutdown the given nodes; - * @throws ExecutionException - * @throws InterruptedException */ private long shutDownNodeWithTimeOut(long timeout, Set<DistributedMember> nodesToBeStopped) throws TimeoutException, InterruptedException, ExecutionException { @@ -290,9 +285,9 @@ public class MiscellaneousCommands implements CommandMarker { long timeElapsed = shutDownTimeEnd - shutDownTimeStart; if (timeElapsed > timeout || Boolean.getBoolean("ThrowTimeoutException")) { // The second check - // for - // ThrowTimeoutException - // is a test hook + // for + // ThrowTimeoutException + // is a test hook throw new TimeoutException(); } return timeElapsed; @@ -532,7 +527,7 @@ public class MiscellaneousCommands implements CommandMarker { } resultData.addAsFile(saveToFile, resultInfo.toString(), CliStrings.NETSTAT__MSG__SAVED_OUTPUT_IN_0, false); // Note: substitution for {0} will - // happen on client side. + // happen on client side. } else { resultData.addLine(resultInfo.toString()); } @@ -694,87 +689,6 @@ public class MiscellaneousCommands implements CommandMarker { return result; } - Result exportLogsPreprocessing(String dirName, String[] groups, String memberId, String logLevel, - boolean onlyLogLevel, boolean mergeLog, String start, String end, - int numOfLogFilesForTesting) { - Result result = null; - try { - LogWrapper.getInstance().fine("Exporting logs"); - Cache cache = CacheFactory.getAnyInstance(); - Set<DistributedMember> dsMembers = new HashSet<DistributedMember>(); - Time startTime = null, endTime = null; - - if (logLevel == null || logLevel.length() == 0) { - // set default log level - logLevel = LogWriterImpl.levelToString(InternalLogWriter.INFO_LEVEL); - } - if (start != null && end == null) { - startTime = parseDate(start); - endTime = new Time(System.currentTimeMillis()); - } - - if (end != null && start == null) { - endTime = parseDate(end); - startTime = new Time(0); - } - if (start != null && end != null) { - startTime = parseDate(start); - endTime = parseDate(end); - if (endTime.getTime() - startTime.getTime() <= 0) { - result = - ResultBuilder.createUserErrorResult(CliStrings.EXPORT_LOGS__MSG__INVALID_TIMERANGE); - } - } - if (end == null && start == null) { - // set default time range as 1 day. - endTime = new Time(System.currentTimeMillis()); - startTime = new Time(endTime.getTime() - 24 * 60 * 60 * 1000); - } - LogWrapper.getInstance().fine( - "Exporting logs startTime=" + startTime.toGMTString() + " " + startTime.toLocaleString()); - LogWrapper.getInstance() - .fine("Exporting logs endTime=" + endTime.toGMTString() + " " + endTime.toLocaleString()); - if (groups != null && memberId != null) { - result = - ResultBuilder.createUserErrorResult(CliStrings.EXPORT_LOGS__MSG__SPECIFY_ONE_OF_OPTION); - } else if (groups != null && groups.length > 0) { - for (String group : groups) { - Set<DistributedMember> groupMembers = cache.getDistributedSystem().getGroupMembers(group); - if (groupMembers != null && groupMembers.size() > 0) { - dsMembers.addAll(groupMembers); - } - } - if (dsMembers.size() == 0) { - result = ResultBuilder - .createUserErrorResult(CliStrings.EXPORT_LOGS__MSG__NO_GROUPMEMBER_FOUND); - } - result = export(cache, dsMembers, dirName, logLevel, onlyLogLevel ? "true" : "false", - mergeLog, startTime, endTime, numOfLogFilesForTesting); - } else if (memberId != null) { - DistributedMember member = CliUtil.getDistributedMemberByNameOrId(memberId); - if (member == null) { - result = ResultBuilder.createUserErrorResult( - CliStrings.format(CliStrings.EXPORT_LOGS__MSG__INVALID_MEMBERID, memberId)); - } - dsMembers.add(member); - result = export(cache, dsMembers, dirName, logLevel, onlyLogLevel ? "true" : "false", - mergeLog, startTime, endTime, numOfLogFilesForTesting); - } else { - // run in entire DS members and get all including locators - dsMembers.addAll(CliUtil.getAllMembers(cache)); - result = export(cache, dsMembers, dirName, logLevel, onlyLogLevel ? "true" : "false", - mergeLog, startTime, endTime, numOfLogFilesForTesting); - } - } catch (ParseException ex) { - LogWrapper.getInstance().fine(ex.getMessage()); - result = ResultBuilder.createUserErrorResult(ex.getMessage()); - } catch (Exception ex) { - LogWrapper.getInstance().fine(ex.getMessage()); - result = ResultBuilder.createUserErrorResult(ex.getMessage()); - } - return result; - } - @CliCommand(value = CliStrings.EXPORT_LOGS, help = CliStrings.EXPORT_LOGS__HELP) @CliMetaData(shellOnly = false, relatedTopic = {CliStrings.TOPIC_GEODE_SERVER, CliStrings.TOPIC_GEODE_DEBUG_UTIL}) @@ -789,7 +703,7 @@ public class MiscellaneousCommands implements CommandMarker { @CliOption(key = CliStrings.EXPORT_LOGS__MEMBER, unspecifiedDefaultValue = CliMetaData.ANNOTATION_NULL_VALUE, optionContext = ConverterHint.ALL_MEMBER_IDNAME, - help = CliStrings.EXPORT_LOGS__MEMBER__HELP) String memberId, + help = CliStrings.EXPORT_LOGS__MEMBER__HELP) String[] memberIds, @CliOption(key = CliStrings.EXPORT_LOGS__LOGLEVEL, unspecifiedDefaultValue = CliMetaData.ANNOTATION_NULL_VALUE, optionContext = ConverterHint.LOG_LEVEL, @@ -805,207 +719,63 @@ public class MiscellaneousCommands implements CommandMarker { unspecifiedDefaultValue = CliMetaData.ANNOTATION_NULL_VALUE, help = CliStrings.EXPORT_LOGS__ENDTIME__HELP) String end) { Result result = null; - try { - result = exportLogsPreprocessing(dirName, groups, memberId, logLevel, onlyLogLevel, mergeLog, - start, end, 0); - } catch (Exception ex) { - LogWrapper.getInstance().fine(ex.getMessage()); - result = ResultBuilder.createUserErrorResult(ex.getMessage()); - } - LogWrapper.getInstance().fine("Exporting logs returning =" + result); - return result; - } + Logger logger = LogService.getLogger(); - Time parseDate(String dateString) throws ParseException { - Time time = null; try { - SimpleDateFormat df = new SimpleDateFormat(MiscellaneousCommands.FORMAT); - time = new Time(df.parse(dateString).getTime()); - } catch (Exception e) { - SimpleDateFormat df = new SimpleDateFormat(MiscellaneousCommands.ONLY_DATE_FORMAT); - time = new Time(df.parse(dateString).getTime()); - } - return time; - } + GemFireCacheImpl cache = GemFireCacheImpl.getInstance(); - Result export(Cache cache, Set<DistributedMember> dsMembers, String dirName, String logLevel, - String onlyLogLevel, boolean mergeLog, Time startTime, Time endTime, - int numOfLogFilesForTesting) { - LogWrapper.getInstance() - .fine("Exporting logs in export membersize = " + dsMembers.size() + " dirname=" + dirName - + " logLevel=" + logLevel + " onlyLogLevel=" + onlyLogLevel + " mergeLog=" + mergeLog - + " startTime=" + startTime.toGMTString() + "endTime=" + endTime.toGMTString()); - Function function = new LogFileFunction(); - FunctionService.registerFunction(function); - try { - List<?> resultList = null; + Set<DistributedMember> targetMembers = CliUtil.findMembersIncludingLocators(groups, memberIds); - List<String> logsToMerge = new ArrayList<String>(); - - Iterator<DistributedMember> it = dsMembers.iterator(); - Object[] args = new Object[6]; - args[0] = dirName; - args[1] = logLevel; - args[2] = onlyLogLevel; - args[3] = startTime.getTime(); - args[4] = endTime.getTime(); - args[5] = numOfLogFilesForTesting; - - while (it.hasNext()) { - boolean toContinueForRestOfmembers = false; - DistributedMember member = it.next(); - LogWrapper.getInstance().fine("Exporting logs copy the logs for member=" + member.getId()); - try { - resultList = (ArrayList<?>) CliUtil.executeFunction(function, args, member).getResult(); - } catch (Exception ex) { - LogWrapper.getInstance() - .fine(CliStrings.format( - CliStrings.EXPORT_LOGS__MSG__FAILED_TO_EXPORT_LOG_FILES_FOR_MEMBER_0, - member.getId()), ex); - // try for other members - continue; - } - if (resultList != null && !resultList.isEmpty()) { - for (int i = 0; i < resultList.size(); i++) { - Object object = resultList.get(i); - if (object instanceof Exception) { - ResultBuilder.createGemFireErrorResult(CliStrings.format( - CliStrings.EXPORT_LOGS__MSG__FAILED_TO_EXPORT_LOG_FILES_FOR_MEMBER_0, - member.getId())); - LogWrapper.getInstance().fine("Exporting logs for member=" + member.getId() - + " exception=" + ((Throwable) object).getMessage(), ((Throwable) object)); - toContinueForRestOfmembers = true; - break; - } else if (object instanceof Throwable) { - ResultBuilder.createGemFireErrorResult(CliStrings.format( - CliStrings.EXPORT_LOGS__MSG__FAILED_TO_EXPORT_LOG_FILES_FOR_MEMBER_0, - member.getId())); - - Throwable th = (Throwable) object; - LogWrapper.getInstance().fine(CliUtil.stackTraceAsString((th))); - LogWrapper.getInstance().fine("Exporting logs for member=" + member.getId() - + " exception=" + ((Throwable) object).getMessage(), ((Throwable) object)); - toContinueForRestOfmembers = true; - break; - } - } - } else { - LogWrapper.getInstance().fine("Exporting logs for member=" + member.getId() - + " resultList is either null or empty"); - continue; - } + Map<String, Path> zipFilesFromMembers = new HashMap<>(); + for (DistributedMember server : targetMembers) { + Region region = ExportLogsFunction.createOrGetExistingExportLogsRegion(true); - if (toContinueForRestOfmembers == true) { - LogWrapper.getInstance().fine("Exporting logs for member=" + member.getId() - + " toContinueForRestOfmembers=" + toContinueForRestOfmembers); - // proceed for rest of the member - continue; - } + ExportLogsCacheWriter cacheWriter = + (ExportLogsCacheWriter) region.getAttributes().getCacheWriter(); - String rstList = (String) resultList.get(0); - LogWrapper.getInstance().fine("for member=" + member.getId() - + "Successfully exported to directory=" + dirName + " rstList=" + rstList); - if (rstList == null || rstList.length() == 0) { - ResultBuilder.createGemFireErrorResult(CliStrings.format( - CliStrings.EXPORT_LOGS__MSG__FAILED_TO_EXPORT_LOG_FILES_FOR_MEMBER_0, - member.getId())); - LogWrapper.getInstance().fine("for member=" + member.getId() + "rstList is null"); - continue; - } else if (rstList.contains("does not exist or cannot be created")) { - LogWrapper.getInstance() - .fine("for member=" + member.getId() + " does not exist or cannot be created"); - return ResultBuilder.createInfoResult(CliStrings - .format(CliStrings.EXPORT_LOGS__MSG__TARGET_DIR_CANNOT_BE_CREATED, dirName)); - } else if (rstList.contains( - LocalizedStrings.InternalDistributedSystem_THIS_CONNECTION_TO_A_DISTRIBUTED_SYSTEM_HAS_BEEN_DISCONNECTED - .toLocalizedString())) { - LogWrapper.getInstance() - .fine("for member=" + member.getId() - + LocalizedStrings.InternalDistributedSystem_THIS_CONNECTION_TO_A_DISTRIBUTED_SYSTEM_HAS_BEEN_DISCONNECTED - .toLocalizedString()); - // proceed for rest of the members - continue; - } + cacheWriter.startFile(server.getName()); - // maintain list of log files to merge only when merge option is true. - if (mergeLog == true) { - StringTokenizer st = new StringTokenizer(rstList, ";"); - while (st.hasMoreTokens()) { - logsToMerge.add(st.nextToken()); - } - } + CliUtil.executeFunction(new ExportLogsFunction(), + new ExportLogsFunction.Args(start, end, logLevel, onlyLogLevel), server) + .getResult(); + Path zipFile = cacheWriter.endFile(); + ExportLogsFunction.destroyExportLogsRegion(); + logger.info("Recieved zip file from member " + server.getId() + ": " + zipFile.toString()); + zipFilesFromMembers.put(server.getId(), zipFile); } - // merge log files - if (mergeLog == true) { - LogWrapper.getInstance().fine("Successfully exported to directory=" + dirName - + " and now merging logsToMerge=" + logsToMerge.size()); - mergeLogs(logsToMerge); - return ResultBuilder - .createInfoResult("Successfully exported and merged in directory " + dirName); + + Path tempDir = Files.createTempDirectory("exportedLogs"); + Path exportedLogsDir = tempDir.resolve("exportedLogs"); + + for (Path zipFile : zipFilesFromMembers.values()) { + Path unzippedMemberDir = + exportedLogsDir.resolve(zipFile.getFileName().toString().replace(".zip", "")); + ZipUtils.unzip(zipFile.toAbsolutePath().toString(), unzippedMemberDir.toString()); + FileUtils.deleteQuietly(zipFile.toFile()); } - LogWrapper.getInstance().fine("Successfully exported to directory without merging" + dirName); - return ResultBuilder.createInfoResult("Successfully exported to directory " + dirName); + + Path workingDir = Paths.get(System.getProperty("user.dir")); + Path exportedLogsZipFile = workingDir.resolve("exportedLogs[" + System.currentTimeMillis() + "].zip").toAbsolutePath(); + + logger.info("Zipping into: " + exportedLogsZipFile.toString()); + ZipUtils.zipDirectory(exportedLogsDir, exportedLogsZipFile); + FileUtils.deleteDirectory(tempDir.toFile()); + result = ResultBuilder.createInfoResult("File exported to: " + exportedLogsZipFile.toString()); } catch (Exception ex) { - LogWrapper.getInstance().info(ex.getMessage(), ex); - return ResultBuilder.createUserErrorResult( - CliStrings.EXPORT_LOGS__MSG__FUNCTION_EXCEPTION + ((LogFileFunction) function).getId()); + ex.printStackTrace(); + logger.error(ex, ex); + result = ResultBuilder.createUserErrorResult(ex.getMessage()); + } finally { + ExportLogsFunction.destroyExportLogsRegion(); } - } - Result mergeLogs(List<String> logsToMerge) { - // create a new process for merging - LogWrapper.getInstance().fine("Exporting logs merging logs" + logsToMerge.size()); - if (logsToMerge.size() > 1) { - List<String> commandList = new ArrayList<String>(); - commandList.add(System.getProperty("java.home") + File.separatorChar + "bin" - + File.separatorChar + "java"); - commandList.add("-classpath"); - commandList.add(System.getProperty("java.class.path", ".")); - commandList.add(MergeLogs.class.getName()); - - commandList - .add(logsToMerge.get(0).substring(0, logsToMerge.get(0).lastIndexOf(File.separator) + 1)); - - ProcessBuilder procBuilder = new ProcessBuilder(commandList); - StringBuilder output = new StringBuilder(); - String errorString = new String(); - try { - LogWrapper.getInstance().fine("Exporting logs now merging logs"); - Process mergeProcess = procBuilder.redirectErrorStream(true).start(); - - mergeProcess.waitFor(); - - InputStream inputStream = mergeProcess.getInputStream(); - BufferedReader br = new BufferedReader(new InputStreamReader(inputStream)); - String line = null; - - while ((line = br.readLine()) != null) { - output.append(line).append(GfshParser.LINE_SEPARATOR); - } - mergeProcess.destroy(); - } catch (Exception e) { - LogWrapper.getInstance().fine(e.getMessage()); - return ResultBuilder.createUserErrorResult( - CliStrings.EXPORT_LOGS__MSG__FUNCTION_EXCEPTION + "Could not merge"); - } finally { - if (errorString != null) { - output.append(errorString).append(GfshParser.LINE_SEPARATOR); - LogWrapper.getInstance().fine("Exporting logs after merging logs " + output); - } - } - if (output.toString().contains("Sucessfully merged logs")) { - LogWrapper.getInstance().fine("Exporting logs Sucessfully merged logs"); - return ResultBuilder.createInfoResult("Successfully merged"); - } else { - LogWrapper.getInstance().fine("Could not merge"); - return ResultBuilder.createUserErrorResult( - CliStrings.EXPORT_LOGS__MSG__FUNCTION_EXCEPTION + "Could not merge"); - } - } - return ResultBuilder.createInfoResult("Only one log file, nothing to merge"); + LogWrapper.getInstance().fine("Exporting logs returning =" + result); + return result; } + + /**** * Current implementation supports writing it to a file and returning the location of the file * @@ -1112,7 +882,7 @@ public class MiscellaneousCommands implements CommandMarker { /*** * Writes the Stack traces member-wise to a text file - * + * * @param dumps - Map containing key : member , value : zipped stack traces * @param fileName - Name of the file to which the stack-traces are written to * @return Canonical path of the file which contains the stack-traces @@ -1266,7 +1036,6 @@ public class MiscellaneousCommands implements CommandMarker { csvBuilder.append('\n'); } - CompositeResultData crd = ResultBuilder.createCompositeResultData(); SectionResultData section = crd.addSection(); TabularResultData metricsTable = section.addTable(); @@ -1410,7 +1179,6 @@ public class MiscellaneousCommands implements CommandMarker { Set<String> checkSet = new HashSet<String>(categoriesMap.keySet()); Set<String> userCategories = getSetDifference(categories, checkSet); - // Checking if the categories specified by the user are valid or not if (userCategories.isEmpty()) { for (String category : checkSet) { @@ -1604,7 +1372,7 @@ public class MiscellaneousCommands implements CommandMarker { memberMxBean.getDiskFlushAvgLatency(), csvBuilder); writeToTableAndCsv(metricsTable, "", "totalQueueSize", memberMxBean.getTotalDiskTasksWaiting(), csvBuilder); // deadcoded to workaround bug - // 46397 + // 46397 writeToTableAndCsv(metricsTable, "", "totalBackupInProgress", memberMxBean.getTotalBackupInProgress(), csvBuilder); } @@ -2059,7 +1827,7 @@ public class MiscellaneousCommands implements CommandMarker { /*** * Writes an entry to a TabularResultData and writes a comma separated entry to a string builder - * + * * @param metricsTable * @param type * @param metricName @@ -2116,7 +1884,7 @@ public class MiscellaneousCommands implements CommandMarker { /**** * Defines and returns map of categories for System metrics. - * + * * @return map with categories for system metrics and display flag set to true */ private Map<String, Boolean> getSystemMetricsCategories() { @@ -2130,7 +1898,7 @@ public class MiscellaneousCommands implements CommandMarker { /**** * Defines and returns map of categories for Region Metrics - * + * * @return map with categories for region metrics and display flag set to true */ private Map<String, Boolean> getRegionMetricsCategories() { @@ -2149,7 +1917,7 @@ public class MiscellaneousCommands implements CommandMarker { /**** * Defines and returns map of categories for system-wide region metrics - * + * * @return map with categories for system wide region metrics and display flag set to true */ private Map<String, Boolean> getSystemRegionMetricsCategories() { @@ -2160,7 +1928,7 @@ public class MiscellaneousCommands implements CommandMarker { /***** * Defines and returns map of categories for member metrics - * + * * @return map with categories for member metrics and display flag set to true */ private Map<String, Boolean> getMemberMetricsCategories() { @@ -2214,7 +1982,7 @@ public class MiscellaneousCommands implements CommandMarker { /*** * Writes to a TabularResultData and also appends a CSV string to a String builder - * + * * @param metricsTable * @param type * @param metricName @@ -2280,7 +2048,6 @@ public class MiscellaneousCommands implements CommandMarker { Set<DistributedMember> dsMembers = new HashSet<DistributedMember>(); Set<DistributedMember> ds = CliUtil.getAllMembers(cache); - if (grps != null && grps.length > 0) { for (String grp : grps) { dsMembers.addAll(cache.getDistributedSystem().getGroupMembers(grp)); @@ -2359,7 +2126,6 @@ public class MiscellaneousCommands implements CommandMarker { } - @CliAvailabilityIndicator({CliStrings.SHUTDOWN, CliStrings.GC, CliStrings.SHOW_DEADLOCK, CliStrings.SHOW_METRICS, CliStrings.SHOW_LOG, CliStrings.EXPORT_STACKTRACE, CliStrings.NETSTAT, CliStrings.EXPORT_LOGS, CliStrings.CHANGE_LOGLEVEL}) http://git-wip-us.apache.org/repos/asf/geode/blob/4c6f3695/geode-core/src/main/java/org/apache/geode/management/internal/cli/functions/ExportLogsFunction.java ---------------------------------------------------------------------- diff --git a/geode-core/src/main/java/org/apache/geode/management/internal/cli/functions/ExportLogsFunction.java b/geode-core/src/main/java/org/apache/geode/management/internal/cli/functions/ExportLogsFunction.java new file mode 100644 index 0000000..1dc89e9 --- /dev/null +++ b/geode-core/src/main/java/org/apache/geode/management/internal/cli/functions/ExportLogsFunction.java @@ -0,0 +1,203 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with this work for additional information regarding + * copyright ownership. The ASF licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. You may obtain a + * copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + * + */ + +package org.apache.geode.management.internal.cli.functions; + +import static java.util.stream.Collectors.toSet; +import static org.apache.geode.distributed.internal.DistributionManager.LOCATOR_DM_TYPE; + +import org.apache.geode.cache.AttributesFactory; +import org.apache.geode.cache.DataPolicy; +import org.apache.geode.cache.Region; +import org.apache.geode.cache.Scope; +import org.apache.geode.cache.execute.Function; +import org.apache.geode.cache.execute.FunctionContext; +import org.apache.geode.internal.InternalEntity; +import org.apache.geode.internal.cache.GemFireCacheImpl; +import org.apache.geode.internal.cache.InternalRegionArguments; +import org.apache.geode.internal.lang.StringUtils; +import org.apache.geode.internal.logging.InternalLogWriter; +import org.apache.geode.internal.logging.LogService; +import org.apache.geode.internal.logging.LogWriterImpl; +import org.apache.geode.management.internal.cli.commands.MiscellaneousCommands; +import org.apache.geode.management.internal.cli.util.ExportLogsCacheWriter; +import org.apache.geode.management.internal.cli.util.LogExporter; +import org.apache.geode.management.internal.cli.util.LogFilter; +import org.apache.geode.management.internal.configuration.domain.Configuration; +import org.apache.logging.log4j.Logger; + +import java.io.FileInputStream; +import java.io.IOException; +import java.io.Serializable; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.util.Arrays; +import java.util.Set; +import java.util.stream.Stream; + +public class ExportLogsFunction implements Function, InternalEntity { + public static final String EXPORT_LOGS_REGION = "__exportLogsRegion"; + private static final Logger LOGGER = LogService.getLogger(); + private static final long serialVersionUID = 1L; + private static final int BUFFER_SIZE = 1024; + + + @Override + public void execute(final FunctionContext context) { + try { + // TODO: change this to get cache from FunctionContext when it becomes available + GemFireCacheImpl cache = GemFireCacheImpl.getInstance(); + + String memberId = cache.getDistributedSystem().getMemberId(); + LOGGER.info("ExportLogsFunction started for member {}", memberId); + + Region exportLogsRegion = createOrGetExistingExportLogsRegion(false); + + Args args = (Args) context.getArguments(); + LogFilter logFilter = + new LogFilter(args.getPermittedLogLevels(), args.getStartTime(), args.getEndTime()); + Path workingDir = Paths.get(System.getProperty("user.dir")); + + Path exportedZipFile = new LogExporter(logFilter).export(workingDir); + + LOGGER.info("Streaming zipped file: " + exportedZipFile.toString()); + try (FileInputStream inputStream = new FileInputStream(exportedZipFile.toFile())) { + byte[] buffer = new byte[BUFFER_SIZE]; + + int bytesRead; + while ((bytesRead = inputStream.read(buffer)) > 0) { + if (bytesRead == BUFFER_SIZE) { + exportLogsRegion.put(memberId, buffer); + } else { + exportLogsRegion.put(memberId, Arrays.copyOfRange(buffer, 0, bytesRead)); + } + } + } + + context.getResultSender().lastResult(null); + + } catch (Exception e) { + LOGGER.error(e); + context.getResultSender().sendException(e); + } + } + + protected static boolean isLocator(GemFireCacheImpl cache) { + return cache.getMyId().getVmKind() == LOCATOR_DM_TYPE; + } + + public static Region createOrGetExistingExportLogsRegion(boolean isInitiatingMember) + throws IOException, ClassNotFoundException { + GemFireCacheImpl cache = GemFireCacheImpl.getInstance(); + + Region exportLogsRegion = cache.getRegion(EXPORT_LOGS_REGION); + if (exportLogsRegion == null) { + AttributesFactory<String, Configuration> regionAttrsFactory = + new AttributesFactory<String, Configuration>(); + regionAttrsFactory.setDataPolicy(DataPolicy.EMPTY); + regionAttrsFactory.setScope(Scope.DISTRIBUTED_ACK); + + if (isInitiatingMember) { + regionAttrsFactory.setCacheWriter(new ExportLogsCacheWriter()); + } + InternalRegionArguments internalArgs = new InternalRegionArguments(); + internalArgs.setIsUsedForMetaRegion(true); + exportLogsRegion = + cache.createVMRegion(EXPORT_LOGS_REGION, regionAttrsFactory.create(), internalArgs); + } + + return exportLogsRegion; + } + + public static void destroyExportLogsRegion() { + GemFireCacheImpl cache = GemFireCacheImpl.getInstance(); + + Region exportLogsRegion = cache.getRegion(EXPORT_LOGS_REGION); + if (exportLogsRegion == null) { + return; + } + + exportLogsRegion.destroyRegion(); + + } + + @Override + public boolean isHA() { + return false; + } + + public static class Args implements Serializable { + private String startTime; + private String endTime; + private String logLevel; + private boolean logLevelOnly; + + public Args(String startTime, String endTime, String logLevel, boolean logLevelOnly) { + this.startTime = startTime; + this.endTime = endTime; + this.logLevel = logLevel; + this.logLevelOnly = logLevelOnly; + } + + public LocalDateTime getStartTime() { + return parseTime(startTime); + } + + public LocalDateTime getEndTime() { + return parseTime(endTime); + } + + public Set<String> getPermittedLogLevels() { + if (logLevel == null || StringUtils.isBlank(logLevel)) { + return LogFilter.allLogLevels(); + } + + if (logLevelOnly) { + return Stream.of(logLevel).collect(toSet()); + } + + // Return all log levels lower than or equal to the specified logLevel + return Arrays.stream(InternalLogWriter.levelNames).filter((String level) -> { + int logLevelCode = LogWriterImpl.levelNameToCode(level); + int logLevelCodeThreshold = LogWriterImpl.levelNameToCode(logLevel); + + return logLevelCode >= logLevelCodeThreshold; + }).collect(toSet()); + } + + private static LocalDateTime parseTime(String dateString) { + if (dateString == null) { + return null; + } + + try { + SimpleDateFormat df = new SimpleDateFormat(MiscellaneousCommands.FORMAT); + return df.parse(dateString).toInstant().atZone(ZoneId.systemDefault()).toLocalDateTime(); + } catch (ParseException e) { + try { + SimpleDateFormat df = new SimpleDateFormat(MiscellaneousCommands.ONLY_DATE_FORMAT); + return df.parse(dateString).toInstant().atZone(ZoneId.systemDefault()).toLocalDateTime(); + } catch (ParseException e1) { + return null; + } + } + } + } +} http://git-wip-us.apache.org/repos/asf/geode/blob/4c6f3695/geode-core/src/main/java/org/apache/geode/management/internal/cli/util/ExportLogsCacheWriter.java ---------------------------------------------------------------------- diff --git a/geode-core/src/main/java/org/apache/geode/management/internal/cli/util/ExportLogsCacheWriter.java b/geode-core/src/main/java/org/apache/geode/management/internal/cli/util/ExportLogsCacheWriter.java new file mode 100644 index 0000000..a8b7225 --- /dev/null +++ b/geode-core/src/main/java/org/apache/geode/management/internal/cli/util/ExportLogsCacheWriter.java @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with this work for additional information regarding + * copyright ownership. The ASF licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. You may obtain a + * copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + * + */ + +package org.apache.geode.management.internal.cli.util; + +import org.apache.commons.io.IOUtils; +import org.apache.geode.cache.CacheWriterException; +import org.apache.geode.cache.EntryEvent; +import org.apache.geode.cache.util.CacheWriterAdapter; + +import java.io.BufferedOutputStream; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.io.Serializable; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Map; + +public class ExportLogsCacheWriter extends CacheWriterAdapter implements Serializable { + private Path currentFile; + private BufferedOutputStream currentOutputStream; + + @Override + public void beforeCreate(EntryEvent event) throws CacheWriterException { + if (currentFile.getFileName().endsWith("server-2.zip")) { + System.out.println("We got data from server 2"); + } + if (currentOutputStream == null) { + throw new IllegalStateException("No outputStream is open. You must call startFile before sending data."); + } + + try { + Object newValue = event.getNewValue(); + if (!(newValue instanceof byte[])) { + throw new IllegalArgumentException( + "Value must be a byte[]. Recieved " + newValue.getClass().getCanonicalName()); + } + currentOutputStream.write((byte[]) newValue); + } catch (IOException e) { + throw new CacheWriterException(e); + } + } + + public void startFile(String memberId) throws IOException { + if (currentFile != null || currentOutputStream != null) { + throw new IllegalStateException("Cannot open more than one file at once"); + } + + currentFile = Files.createTempDirectory(memberId).resolve(memberId + ".zip"); + currentOutputStream = new BufferedOutputStream(new FileOutputStream(currentFile.toFile())); + } + + public Path endFile() { + Path completedFile = currentFile; + + IOUtils.closeQuietly(currentOutputStream); + currentOutputStream = null; + currentFile = null; + + return completedFile; + } +} http://git-wip-us.apache.org/repos/asf/geode/blob/4c6f3695/geode-core/src/main/java/org/apache/geode/management/internal/cli/util/ExportLogsRepository.java ---------------------------------------------------------------------- diff --git a/geode-core/src/main/java/org/apache/geode/management/internal/cli/util/ExportLogsRepository.java b/geode-core/src/main/java/org/apache/geode/management/internal/cli/util/ExportLogsRepository.java new file mode 100644 index 0000000..9a79fc0 --- /dev/null +++ b/geode-core/src/main/java/org/apache/geode/management/internal/cli/util/ExportLogsRepository.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.geode.management.internal.cli.util; + +import java.util.Map; + +public class ExportLogsRepository { + + + Map exportFiles; + + public void addFile() { + + } + + public void deleteFile() { + + } + + private void cleanUpExpiredFiles() { + + } +} http://git-wip-us.apache.org/repos/asf/geode/blob/4c6f3695/geode-core/src/main/java/org/apache/geode/management/internal/cli/util/LogExporter.java ---------------------------------------------------------------------- diff --git a/geode-core/src/main/java/org/apache/geode/management/internal/cli/util/LogExporter.java b/geode-core/src/main/java/org/apache/geode/management/internal/cli/util/LogExporter.java new file mode 100644 index 0000000..f30d686 --- /dev/null +++ b/geode-core/src/main/java/org/apache/geode/management/internal/cli/util/LogExporter.java @@ -0,0 +1,136 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with this work for additional information regarding + * copyright ownership. The ASF licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. You may obtain a + * copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + * + */ + +package org.apache.geode.management.internal.cli.util; + +import static java.util.stream.Collectors.toList; + +import org.apache.commons.io.FileUtils; +import org.apache.geode.cache.AttributesFactory; +import org.apache.geode.cache.DataPolicy; +import org.apache.geode.cache.Region; +import org.apache.geode.cache.Scope; +import org.apache.geode.internal.cache.GemFireCacheImpl; +import org.apache.geode.internal.cache.InternalRegionArguments; +import org.apache.geode.internal.logging.LogService; +import org.apache.geode.management.internal.configuration.domain.Configuration; +import org.apache.geode.management.internal.configuration.utils.ZipUtils; +import org.apache.logging.log4j.Logger; + +import java.io.BufferedReader; +import java.io.BufferedWriter; +import java.io.FileReader; +import java.io.FileWriter; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.text.ParseException; +import java.util.List; +import java.util.function.Predicate; +import java.util.stream.Stream; + +public class LogExporter { + private static final Logger LOGGER = LogService.getLogger(); + + private final LogFilter logFilter; + + public LogExporter(LogFilter logFilter) throws ParseException { + this.logFilter = logFilter; + } + + public Path export(Path workingDir) throws IOException { + LOGGER.debug("Working directory is {}", workingDir); + + Path tempDirectory = Files.createTempDirectory("exportLogs"); + + for (Path logFile : findLogFiles(workingDir)) { + Path filteredLogFile = tempDirectory.resolve(logFile.getFileName()); + + if (this.logFilter == null) { + Files.copy(logFile, filteredLogFile); + } else { + writeFilteredLogFile(logFile, filteredLogFile); + } + } + + for (Path statFile : findStatFiles(workingDir)) { + Files.copy(statFile, tempDirectory); + } + + Path zipFile = Files.createTempFile("logExport", ".zip"); + ZipUtils.zipDirectory(tempDirectory, zipFile); + LOGGER.info("Zipped files to: " + zipFile); + + + FileUtils.deleteDirectory(tempDirectory.toFile()); + + return zipFile; + } + + protected void writeFilteredLogFile(Path originalLogFile, Path filteredLogFile) + throws IOException { + this.logFilter.startNewFile(); + + try (BufferedReader reader = new BufferedReader(new FileReader(originalLogFile.toFile()))) { + try (BufferedWriter writer = new BufferedWriter(new FileWriter(filteredLogFile.toFile()))) { + + String line; + while ((line = reader.readLine()) != null) { + LogFilter.LineFilterResult result = this.logFilter.acceptsLine(line); + + if (result == LogFilter.LineFilterResult.REMAINDER_OF_FILE_REJECTED) { + break; + } + + if (result == LogFilter.LineFilterResult.LINE_ACCEPTED) { + writeLine(line, writer); + } + } + } + } + } + + private void writeLine(String line, BufferedWriter writer) { + try { + writer.write(line); + writer.newLine(); + } catch (IOException e) { + throw new RuntimeException("Unable to write to log file", e); + } + } + + protected List<Path> findLogFiles(Path workingDir) throws IOException { + Predicate<Path> logFileSelector = (Path file) -> file.toString().toLowerCase().endsWith(".log"); + return findFiles(workingDir, logFileSelector); + } + + + protected List<Path> findStatFiles(Path workingDir) throws IOException { + Predicate<Path> statFileSelector = + (Path file) -> file.toString().toLowerCase().endsWith(".gfs"); + return findFiles(workingDir, statFileSelector); + } + + private List<Path> findFiles(Path workingDir, Predicate<Path> fileSelector) throws IOException { + Stream<Path> selectedFiles = + Files.list(workingDir).filter(fileSelector).filter(this.logFilter::acceptsFile); + + return selectedFiles.collect(toList()); + } + + +} http://git-wip-us.apache.org/repos/asf/geode/blob/4c6f3695/geode-core/src/main/java/org/apache/geode/management/internal/cli/util/LogFilter.java ---------------------------------------------------------------------- diff --git a/geode-core/src/main/java/org/apache/geode/management/internal/cli/util/LogFilter.java b/geode-core/src/main/java/org/apache/geode/management/internal/cli/util/LogFilter.java new file mode 100644 index 0000000..2436530 --- /dev/null +++ b/geode-core/src/main/java/org/apache/geode/management/internal/cli/util/LogFilter.java @@ -0,0 +1,113 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with this work for additional information regarding + * copyright ownership. The ASF licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. You may obtain a + * copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + * + */ + +package org.apache.geode.management.internal.cli.util; + +import static java.util.stream.Collectors.toSet; + +import org.apache.geode.internal.logging.InternalLogWriter; +import org.apache.geode.internal.logging.LogService; +import org.apache.logging.log4j.Logger; + +import java.io.IOException; +import java.nio.file.Path; +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.util.Arrays; +import java.util.Set; + +public class LogFilter { + public enum LineFilterResult { + LINE_ACCEPTED, LINE_REJECTED, REMAINDER_OF_FILE_REJECTED + } + + private static final Logger LOGGER = LogService.getLogger(); + + private final Set<String> permittedLogLevels; + private final LocalDateTime startDate; + private final LocalDateTime endDate; + + private LineFilterResult resultOfPreviousLine = LineFilterResult.LINE_ACCEPTED; + + public LogFilter(Set<String> permittedLogLevels, LocalDateTime startDate, LocalDateTime endDate) { + this.permittedLogLevels = (permittedLogLevels == null || permittedLogLevels.isEmpty()) + ? allLogLevels() : permittedLogLevels; + this.startDate = startDate; + this.endDate = endDate; + } + + public void startNewFile() { + this.resultOfPreviousLine = LineFilterResult.LINE_ACCEPTED; + } + + public LineFilterResult acceptsLine(String logLine) { + LogLevelExtractor.Result result = LogLevelExtractor.extract(logLine); + + return acceptsLogEntry(result); + } + + protected LineFilterResult acceptsLogEntry(LogLevelExtractor.Result result) { + if (result == null) { + return resultOfPreviousLine; + } + + return acceptsLogEntry(result.getLogLevel(), result.getLogTimestamp()); + } + + protected LineFilterResult acceptsLogEntry(String logLevel, LocalDateTime logTimestamp) { + if (logTimestamp == null || logLevel == null) { + throw new IllegalArgumentException(); + } + + LineFilterResult result; + + if (endDate != null && logTimestamp.isAfter(endDate)) { + result = LineFilterResult.REMAINDER_OF_FILE_REJECTED; + } else if (startDate != null && logTimestamp.isBefore(startDate)) { + result = LineFilterResult.LINE_REJECTED; + } else { + result = permittedLogLevels.contains(logLevel) ? LineFilterResult.LINE_ACCEPTED + : LineFilterResult.LINE_REJECTED; + } + + resultOfPreviousLine = result; + + return result; + } + + public boolean acceptsFile(Path file) { + if (startDate == null) { + return true; + } + try { + return (getEndTimeOf(file).isAfter(startDate)); + } catch (IOException e) { + LOGGER.error("Unable to determine lastModified time", e); + return true; + } + } + + private static LocalDateTime getEndTimeOf(Path file) throws IOException { + long lastModifiedMillis = file.toFile().lastModified(); + return Instant.ofEpochMilli(lastModifiedMillis).atZone(ZoneId.systemDefault()) + .toLocalDateTime(); + } + + public static Set<String> allLogLevels() { + return Arrays.stream(InternalLogWriter.levelNames).collect(toSet()); + } +} http://git-wip-us.apache.org/repos/asf/geode/blob/4c6f3695/geode-core/src/main/java/org/apache/geode/management/internal/cli/util/LogLevelExtractor.java ---------------------------------------------------------------------- diff --git a/geode-core/src/main/java/org/apache/geode/management/internal/cli/util/LogLevelExtractor.java b/geode-core/src/main/java/org/apache/geode/management/internal/cli/util/LogLevelExtractor.java new file mode 100644 index 0000000..6c15d15 --- /dev/null +++ b/geode-core/src/main/java/org/apache/geode/management/internal/cli/util/LogLevelExtractor.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with this work for additional information regarding + * copyright ownership. The ASF licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. You may obtain a + * copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + * + */ + +package org.apache.geode.management.internal.cli.util; + +import java.time.LocalDateTime; +import java.time.format.DateTimeFormatter; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public class LogLevelExtractor { + private static Pattern LOG_PATTERN = + Pattern.compile("^\\[(\\S*)\\s+([\\d\\/]+)\\s+([\\d:\\.]+)\\s+(\\S+)"); + + private static DateTimeFormatter LOG_TIMESTAMP_FORMATTER = + DateTimeFormatter.ofPattern("yyyy/MM/dd HH:mm:ss.SSS zzz"); + private static final String SPACE = " "; + + public static Result extract(String logLine) { + Matcher m = LOG_PATTERN.matcher(logLine); + if (!m.find()) { + return null; + } + + String logLevel = m.group(1); + String logTimestamp = m.group(2) + SPACE + m.group(3) + SPACE + m.group(4); + + LocalDateTime timestamp = LocalDateTime.parse(logTimestamp, LOG_TIMESTAMP_FORMATTER); + + return new Result(logLevel, timestamp); + } + + public static class Result { + private String logLevel; + private LocalDateTime logTimestamp; + + public Result(String logLevel, LocalDateTime logTimestamp) { + this.logLevel = logLevel; + this.logTimestamp = logTimestamp; + } + + public String getLogLevel() { + return logLevel; + } + + public LocalDateTime getLogTimestamp() { + return logTimestamp; + } + + } +} + http://git-wip-us.apache.org/repos/asf/geode/blob/4c6f3695/geode-core/src/main/java/org/apache/geode/management/internal/cli/util/MergeLogs.java ---------------------------------------------------------------------- diff --git a/geode-core/src/main/java/org/apache/geode/management/internal/cli/util/MergeLogs.java b/geode-core/src/main/java/org/apache/geode/management/internal/cli/util/MergeLogs.java index 8d2ef45..67d5473 100644 --- a/geode-core/src/main/java/org/apache/geode/management/internal/cli/util/MergeLogs.java +++ b/geode-core/src/main/java/org/apache/geode/management/internal/cli/util/MergeLogs.java @@ -14,17 +14,30 @@ */ package org.apache.geode.management.internal.cli.util; +import static java.util.stream.Collectors.toList; + +import java.io.BufferedReader; import java.io.File; import java.io.FileInputStream; import java.io.FileNotFoundException; import java.io.FilenameFilter; import java.io.InputStream; +import java.io.InputStreamReader; import java.io.PrintWriter; +import java.nio.file.Path; +import java.nio.file.Paths; import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.List; +import org.apache.commons.io.FileUtils; import org.apache.geode.internal.logging.MergeLogFiles; +import org.apache.geode.management.cli.Result; +import org.apache.geode.management.internal.cli.GfshParser; +import org.apache.geode.management.internal.cli.LogWrapper; import org.apache.geode.management.internal.cli.i18n.CliStrings; +import org.apache.geode.management.internal.cli.result.ResultBuilder; /** * @@ -32,17 +45,54 @@ import org.apache.geode.management.internal.cli.i18n.CliStrings; */ public class MergeLogs { - /** - * @param args - */ + + public static void mergeLogsInNewProcess(Path logDirectory) { + // create a new process for merging + LogWrapper.getInstance().fine("Exporting logs merging logs" + logDirectory); + List<String> commandList = new ArrayList<String>(); + commandList.add(System.getProperty("java.home") + File.separatorChar + "bin" + + File.separatorChar + "java"); + commandList.add("-classpath"); + commandList.add(System.getProperty("java.class.path", ".")); + commandList.add(MergeLogs.class.getName()); + + commandList + .add(logDirectory.toAbsolutePath().toString()); + + ProcessBuilder procBuilder = new ProcessBuilder(commandList); + StringBuilder output = new StringBuilder(); + String errorString = new String(); + try { + LogWrapper.getInstance().fine("Exporting logs now merging logs"); + Process mergeProcess = procBuilder.redirectErrorStream(true).start(); + + mergeProcess.waitFor(); + + InputStream inputStream = mergeProcess.getInputStream(); + BufferedReader br = new BufferedReader(new InputStreamReader(inputStream)); + String line = null; + + while ((line = br.readLine()) != null) { + output.append(line).append(GfshParser.LINE_SEPARATOR); + } + mergeProcess.destroy(); + } catch (Exception e) { + LogWrapper.getInstance().severe(e.getMessage()); + } + if (output.toString().contains("Merged logs to: ")) { + LogWrapper.getInstance().fine("Exporting logs Sucessfully merged logs"); + } else { + LogWrapper.getInstance().severe("Could not merge"); + } + } public static void main(String[] args) { if (args.length < 1 || args.length > 1) { throw new IllegalArgumentException("Requires only 1 arguments : <targetDirName>"); } try { - String result = mergeLogFile(args[0]); - System.out.println(result); + String result = mergeLogFile(args[0]).getCanonicalPath(); + System.out.println("Merged logs to: " + result); } catch (Exception e) { System.out.println(e.getMessage()); } @@ -50,27 +100,32 @@ public class MergeLogs { } - static String mergeLogFile(String dirName) throws Exception { - SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd_HH-mm-ss"); - File dir = new File(dirName); - String[] logsToMerge = dir.list(); - InputStream[] logFiles = new FileInputStream[logsToMerge.length]; + protected static List<File> findLogFilesToMerge (File dir) { + return FileUtils.listFiles(dir, new String[]{"log"}, true).stream().collect(toList()); + } + + static File mergeLogFile(String dirName) throws Exception { + Path dir = Paths.get(dirName); + List<File> logsToMerge = findLogFilesToMerge(dir.toFile()); + InputStream[] logFiles = new FileInputStream[logsToMerge.size()]; String[] logFileNames = new String[logFiles.length]; - for (int i = 0; i < logsToMerge.length; i++) { + for (int i = 0; i < logsToMerge.size(); i++) { try { - logFiles[i] = new FileInputStream(dirName + File.separator + logsToMerge[i]); - logFileNames[i] = dirName + File.separator + logsToMerge[i]; + logFiles[i] = new FileInputStream(logsToMerge.get(i)); + logFileNames[i] = dir.relativize(logsToMerge.get(i).toPath()).toString(); } catch (FileNotFoundException e) { throw new Exception( - logsToMerge[i] + " " + CliStrings.EXPORT_LOGS__MSG__FILE_DOES_NOT_EXIST); + logsToMerge.get(i) + " " + CliStrings.EXPORT_LOGS__MSG__FILE_DOES_NOT_EXIST); } } PrintWriter mergedLog = null; + File mergedLogFile = null; try { String mergeLog = - dirName + File.separator + "merge_" + sdf.format(new java.util.Date()) + ".log"; - mergedLog = new PrintWriter(mergeLog); + dirName + File.separator + "merge_" + new SimpleDateFormat("yyyy-MM-dd_HH-mm-ss").format(new java.util.Date()) + ".log"; + mergedLogFile = new File(mergeLog); + mergedLog = new PrintWriter(mergedLogFile); boolean flag = MergeLogFiles.mergeLogFiles(logFiles, logFileNames, mergedLog); } catch (FileNotFoundException e) { throw new Exception( @@ -79,7 +134,7 @@ public class MergeLogs { throw new Exception("Exception in creating PrintWriter in MergeLogFiles" + e.getMessage()); } - return "Sucessfully merged logs"; + return mergedLogFile; } http://git-wip-us.apache.org/repos/asf/geode/blob/4c6f3695/geode-core/src/main/java/org/apache/geode/management/internal/configuration/utils/ZipUtils.java ---------------------------------------------------------------------- diff --git a/geode-core/src/main/java/org/apache/geode/management/internal/configuration/utils/ZipUtils.java b/geode-core/src/main/java/org/apache/geode/management/internal/configuration/utils/ZipUtils.java index 81161d6..e32803a 100644 --- a/geode-core/src/main/java/org/apache/geode/management/internal/configuration/utils/ZipUtils.java +++ b/geode-core/src/main/java/org/apache/geode/management/internal/configuration/utils/ZipUtils.java @@ -41,8 +41,12 @@ import org.apache.commons.io.IOUtils; public class ZipUtils { public static void zipDirectory(Path sourceDirectory, Path targetFile) throws IOException { - Path p = Files.createFile(targetFile); - try (ZipOutputStream zs = new ZipOutputStream(Files.newOutputStream(p))) { + Path parentDir = targetFile.getParent(); + if (parentDir != null && !parentDir.toFile().exists()) { + parentDir.toFile().mkdirs(); + } + + try (ZipOutputStream zs = new ZipOutputStream(Files.newOutputStream(targetFile))) { Files.walk(sourceDirectory).filter(path -> !Files.isDirectory(path)).forEach(path -> { ZipEntry zipEntry = new ZipEntry(sourceDirectory.relativize(path).toString()); try { http://git-wip-us.apache.org/repos/asf/geode/blob/4c6f3695/geode-core/src/test/java/org/apache/geode/management/internal/cli/commands/ExportLogsDUnit.java ---------------------------------------------------------------------- diff --git a/geode-core/src/test/java/org/apache/geode/management/internal/cli/commands/ExportLogsDUnit.java b/geode-core/src/test/java/org/apache/geode/management/internal/cli/commands/ExportLogsDUnit.java new file mode 100644 index 0000000..43ee742 --- /dev/null +++ b/geode-core/src/test/java/org/apache/geode/management/internal/cli/commands/ExportLogsDUnit.java @@ -0,0 +1,363 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with this work for additional information regarding + * copyright ownership. The ASF licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. You may obtain a + * copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + * + */ + +package org.apache.geode.management.internal.cli.commands; + +import static java.util.stream.Collectors.joining; +import static java.util.stream.Collectors.toList; +import static java.util.stream.Collectors.toSet; +import static org.apache.geode.management.internal.cli.commands.MiscellaneousCommands.FORMAT; +import static org.apache.geode.management.internal.cli.commands.MiscellaneousCommands.ONLY_DATE_FORMAT; +import static org.assertj.core.api.Assertions.assertThat; + +import org.apache.commons.io.FileUtils; +import org.apache.geode.cache.Cache; +import org.apache.geode.distributed.ConfigurationProperties; +import org.apache.geode.internal.cache.GemFireCacheImpl; +import org.apache.geode.internal.logging.LogService; +import org.apache.geode.management.internal.cli.functions.ExportLogsFunction; +import org.apache.geode.management.internal.cli.result.CommandResult; +import org.apache.geode.management.internal.cli.util.CommandStringBuilder; +import org.apache.geode.management.internal.configuration.utils.ZipUtils; +import org.apache.geode.test.dunit.IgnoredException; +import org.apache.geode.test.dunit.rules.GfshShellConnectionRule; +import org.apache.geode.test.dunit.rules.Locator; +import org.apache.geode.test.dunit.rules.LocatorServerStartupRule; +import org.apache.geode.test.dunit.rules.Member; +import org.apache.geode.test.dunit.rules.Server; +import org.apache.logging.log4j.Logger; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; + +import java.io.File; +import java.io.IOException; +import java.io.Serializable; +import java.nio.charset.Charset; +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.time.ZonedDateTime; +import java.time.format.DateTimeFormatter; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Properties; +import java.util.Set; +import java.util.stream.Stream; + + +public class ExportLogsDUnit { + private static final String ERROR_LOG_PREFIX = "[IGNORE]"; + + @Rule + public LocatorServerStartupRule lsRule = new LocatorServerStartupRule(); + + @Rule + public GfshShellConnectionRule gfshConnector = new GfshShellConnectionRule(); + + private Locator locator; + private Server server1; + private Server server2; + + private Map<Member, List<LogLine>> expectedMessages; + + @Before + public void setup() throws Exception { + Properties properties = new Properties(); + properties.setProperty(ConfigurationProperties.LOG_LEVEL, "debug"); + + locator = lsRule.startLocatorVM(0, properties); + server1 = lsRule.startServerVM(1, properties, locator.getPort()); + server2 = lsRule.startServerVM(2, properties, locator.getPort()); + + IgnoredException.addIgnoredException(ERROR_LOG_PREFIX); + + expectedMessages = new HashMap<>(); + expectedMessages.put(locator, listOfLogLines(locator, "info", "error", "debug")); + expectedMessages.put(server1, listOfLogLines(server1, "info", "error", "debug")); + expectedMessages.put(server2, listOfLogLines(server2, "info", "error", "debug")); + + // log the messages in each of the members + for (Member member : expectedMessages.keySet()) { + List<LogLine> logLines = expectedMessages.get(member); + + member.invoke(() -> { + Logger logger = LogService.getLogger(); + logLines.forEach((LogLine logLine) -> logLine.writeLog(logger)); + }); + } + + gfshConnector.connectAndVerify(locator); + } + + @Test + public void startAndEndDateCanExcludeLogs() throws Exception { + ZonedDateTime now = LocalDateTime.now().atZone(ZoneId.systemDefault()); + ZonedDateTime yesterday = now.minusDays(1); + ZonedDateTime twoDaysAgo = now.minusDays(2); + + DateTimeFormatter dateTimeFormatter = DateTimeFormatter.ofPattern(ONLY_DATE_FORMAT); + + CommandStringBuilder commandStringBuilder = new CommandStringBuilder("export logs"); + commandStringBuilder.addOption("start-time", dateTimeFormatter.format(twoDaysAgo)); + commandStringBuilder.addOption("end-time", dateTimeFormatter.format(yesterday)); + commandStringBuilder.addOption("log-level", "debug"); + commandStringBuilder.addOption("dir", "someDir"); + + gfshConnector.executeAndVerifyCommand(commandStringBuilder.toString()); + + Set<String> acceptedLogLevels = new HashSet<>(); + verifyZipFileContents(acceptedLogLevels); + } + + @Test + public void startAndEndDateCanIncludeLogs() throws Exception { + ZonedDateTime now = LocalDateTime.now().atZone(ZoneId.systemDefault()); + ZonedDateTime yesterday = now.minusDays(1); + ZonedDateTime tomorrow = now.plusDays(1); + + DateTimeFormatter dateTimeFormatter = DateTimeFormatter.ofPattern(ONLY_DATE_FORMAT); + + CommandStringBuilder commandStringBuilder = new CommandStringBuilder("export logs"); + commandStringBuilder.addOption("start-time", dateTimeFormatter.format(yesterday)); + commandStringBuilder.addOption("end-time", dateTimeFormatter.format(tomorrow)); + commandStringBuilder.addOption("log-level", "debug"); + commandStringBuilder.addOption("dir", "someDir"); + + gfshConnector.executeAndVerifyCommand(commandStringBuilder.toString()); + + Set<String> acceptedLogLevels = Stream.of("info", "error", "debug").collect(toSet()); + verifyZipFileContents(acceptedLogLevels); + } + + @Test + public void testExportWithStartAndEndDateTimeFiltering() throws Exception { + ZonedDateTime cutoffTime = LocalDateTime.now().atZone(ZoneId.systemDefault()); + + String messageAfterCutoffTime = "[this message should not show up since it is after cutoffTime]"; + LogLine logLineAfterCutoffTime = new LogLine(messageAfterCutoffTime, "info", true); + server1.invoke(() -> { + Logger logger = LogService.getLogger(); + logLineAfterCutoffTime.writeLog(logger); + }); + + DateTimeFormatter dateTimeFormatter = DateTimeFormatter.ofPattern(FORMAT); + String cutoffTimeString = dateTimeFormatter.format(cutoffTime); + + CommandStringBuilder commandStringBuilder = new CommandStringBuilder("export logs"); + commandStringBuilder.addOption("start-time", dateTimeFormatter.format(cutoffTime.minusHours(1))); + commandStringBuilder.addOption("end-time", cutoffTimeString); + commandStringBuilder.addOption("log-level", "debug"); + commandStringBuilder.addOption("dir", "someDir"); + + gfshConnector.executeAndVerifyCommand(commandStringBuilder.toString()); + + expectedMessages.get(server1).add(logLineAfterCutoffTime); + Set<String> acceptedLogLevels = Stream.of("info", "error", "debug").collect(toSet()); + verifyZipFileContents(acceptedLogLevels); + } + + @Test + public void testExportWithThresholdLogLevelFilter() throws Exception { + + CommandResult result = gfshConnector.executeAndVerifyCommand( + "export logs --log-level=info --only-log-level=false --dir=" + lsRule.getTempFolder() + .getRoot().getCanonicalPath()); + + Set<String> acceptedLogLevels = Stream.of("info", "error").collect(toSet()); + verifyZipFileContents(acceptedLogLevels); + + } + + @Test + public void testExportWithExactLogLevelFilter() throws Exception { + CommandResult result = gfshConnector.executeAndVerifyCommand( + "export logs --log-level=info --only-log-level=true --dir=" + lsRule.getTempFolder() + .getRoot().getCanonicalPath()); + + + Set<String> acceptedLogLevels = Stream.of("info").collect(toSet()); + verifyZipFileContents(acceptedLogLevels); + } + + @Test + public void testExportWithNoFilters() throws Exception { + CommandResult result = gfshConnector.executeAndVerifyCommand( + "export logs --dir=" + "someDir" /* lsRule.getTempFolder().getRoot().getCanonicalPath() */); + + Set<String> acceptedLogLevels = Stream.of("info", "error", "debug").collect(toSet()); + verifyZipFileContents(acceptedLogLevels); + + // Ensure export logs region gets cleaned up + server1.invoke(ExportLogsDUnit::verifyExportLogsRegionWasDestroyed); + server2.invoke(ExportLogsDUnit::verifyExportLogsRegionWasDestroyed); + locator.invoke(ExportLogsDUnit::verifyExportLogsRegionWasDestroyed); + } + +@Test +public void exportLogsRegionIsCleanedUpProperly() throws IOException, ClassNotFoundException { + locator.invoke(() -> { + ExportLogsFunction.createOrGetExistingExportLogsRegion(true); + Cache cache = GemFireCacheImpl.getInstance(); + assertThat(cache.getRegion(ExportLogsFunction.EXPORT_LOGS_REGION)).isNotNull(); + }); + + server1.invoke(() -> { + ExportLogsFunction.createOrGetExistingExportLogsRegion(false); + Cache cache = GemFireCacheImpl.getInstance(); + assertThat(cache.getRegion(ExportLogsFunction.EXPORT_LOGS_REGION)).isNotNull(); + }); + + locator.invoke(() -> { + ExportLogsFunction.destroyExportLogsRegion(); + + Cache cache = GemFireCacheImpl.getInstance(); + assertThat(cache.getRegion(ExportLogsFunction.EXPORT_LOGS_REGION)).isNull(); + }); + + server1.invoke(() -> { + Cache cache = GemFireCacheImpl.getInstance(); + assertThat(cache.getRegion(ExportLogsFunction.EXPORT_LOGS_REGION)).isNull(); + }); +} + + + public void verifyZipFileContents(Set<String> acceptedLogLevels) + throws IOException { + File unzippedLogFileDir = unzipExportedLogs(); + + Set<File> dirsFromZipFile = + Stream.of(unzippedLogFileDir.listFiles()).filter(File::isDirectory).collect(toSet()); + assertThat(dirsFromZipFile).hasSize(expectedMessages.keySet().size()); + + Set<String> expectedDirNames = + expectedMessages.keySet().stream().map(Member::getName).collect(toSet()); + Set<String> actualDirNames = dirsFromZipFile.stream().map(File::getName).collect(toSet()); + assertThat(actualDirNames).isEqualTo(expectedDirNames); + + System.out.println("Unzipped artifacts:"); + for (File dir : dirsFromZipFile) { + verifyLogFileContents(acceptedLogLevels, dir); + } + } + + public void verifyLogFileContents(Set<String> acceptedLogLevels, File dirForMember) + throws IOException { + + String memberName = dirForMember.getName(); + Member member = expectedMessages.keySet().stream() + .filter((Member aMember) -> aMember.getName().equals(memberName)) + .findFirst() + .get(); + + assertThat(member).isNotNull(); + + Set<String> fileNamesInDir = + Stream.of(dirForMember.listFiles()).map(File::getName).collect(toSet()); + + System.out.println(dirForMember.getCanonicalPath() + " : " + fileNamesInDir); + + File logFileForMember = new File(dirForMember, memberName + ".log"); + assertThat(logFileForMember).exists(); + assertThat(fileNamesInDir).hasSize(1); + + String logFileContents = + FileUtils.readLines(logFileForMember, Charset.defaultCharset()).stream() + .collect(joining("\n")); + + for (LogLine logLine : expectedMessages.get(member)) { + boolean shouldExpectLogLine = acceptedLogLevels.contains(logLine.level) && !logLine.shouldBeIgnoredDueToTimestamp; + + if (shouldExpectLogLine) { + assertThat(logFileContents).contains(logLine.getMessage()); + } else { + assertThat(logFileContents).doesNotContain(logLine.getMessage()); + } + } + + } + + private File unzipExportedLogs() throws IOException { + File locatorWorkingDir = locator.getWorkingDir(); + List<File> filesInDir = Stream.of(locatorWorkingDir.listFiles()).collect(toList()); + assertThat(filesInDir).isNotEmpty(); + + + List<File> zipFilesInDir = Stream.of(locatorWorkingDir.listFiles()) + .filter(f -> f.getName().endsWith(".zip")).collect(toList()); + assertThat(zipFilesInDir).describedAs(filesInDir.stream().map(File::getAbsolutePath).collect(joining(","))).hasSize(1); + + File unzippedLogFileDir = lsRule.getTempFolder().newFolder("unzippedLogs"); + ZipUtils.unzip(zipFilesInDir.get(0).getCanonicalPath(), unzippedLogFileDir.getCanonicalPath()); + return unzippedLogFileDir; + } + + private List<LogLine> listOfLogLines(Member member, String... levels) { + return Stream.of(levels).map(level -> new LogLine(member, level)).collect(toList()); + } + + private static void verifyExportLogsRegionWasDestroyed() { + Cache cache = GemFireCacheImpl.getInstance(); + assertThat(cache.getRegion(ExportLogsFunction.EXPORT_LOGS_REGION)).isNull(); + } + + public static class LogLine implements Serializable { + String level; + String message; + boolean shouldBeIgnoredDueToTimestamp; + + public LogLine(String message, String level, boolean shouldBeIgnoredDueToTimestamp) { + this.message = message; + this.level = level; + this.shouldBeIgnoredDueToTimestamp = shouldBeIgnoredDueToTimestamp; + } + + public LogLine(Member member, String level) { + this.level = level; + this.message = buildMessage(member.getName()); + } + + public String getMessage() { + return message; + } + + private String buildMessage(String memberName) { + StringBuilder stringBuilder = new StringBuilder(); + if (Objects.equals(level, "error")) { + stringBuilder.append(ERROR_LOG_PREFIX); + } + stringBuilder.append(level); + + return stringBuilder.append(memberName).toString(); + } + + + public void writeLog(Logger logger) { + switch (this.level) { + case "info": + logger.info(getMessage()); + break; + case "error": + logger.error(getMessage()); + break; + case "debug": + logger.debug(getMessage()); + } + } + } +} http://git-wip-us.apache.org/repos/asf/geode/blob/4c6f3695/geode-core/src/test/java/org/apache/geode/management/internal/cli/commands/MiscellaneousCommandsExportLogsPart1DUnitTest.java ---------------------------------------------------------------------- diff --git a/geode-core/src/test/java/org/apache/geode/management/internal/cli/commands/MiscellaneousCommandsExportLogsPart1DUnitTest.java b/geode-core/src/test/java/org/apache/geode/management/internal/cli/commands/MiscellaneousCommandsExportLogsPart1DUnitTest.java deleted file mode 100644 index cb9a8c6..0000000 --- a/geode-core/src/test/java/org/apache/geode/management/internal/cli/commands/MiscellaneousCommandsExportLogsPart1DUnitTest.java +++ /dev/null @@ -1,138 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license - * agreements. See the NOTICE file distributed with this work for additional information regarding - * copyright ownership. The ASF licenses this file to You under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. You may obtain a - * copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License - * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. See the License for the specific language governing permissions and limitations under - * the License. - */ -package org.apache.geode.management.internal.cli.commands; - -import static org.apache.geode.test.dunit.Assert.assertEquals; -import static org.apache.geode.test.dunit.Assert.fail; -import static org.apache.geode.test.dunit.LogWriterUtils.getLogWriter; - -import org.apache.commons.io.FileUtils; -import org.apache.geode.cache.Cache; -import org.apache.geode.cache.Region; -import org.apache.geode.cache.RegionFactory; -import org.apache.geode.cache.RegionShortcut; -import org.apache.geode.internal.logging.LogWriterImpl; -import org.apache.geode.management.cli.Result; -import org.apache.geode.management.internal.cli.result.CommandResult; -import org.apache.geode.test.dunit.Host; -import org.apache.geode.test.dunit.SerializableRunnable; -import org.apache.geode.test.dunit.VM; -import org.apache.geode.test.junit.categories.DistributedTest; -import org.apache.geode.test.junit.categories.FlakyTest; -import org.junit.Test; -import org.junit.experimental.categories.Category; - -import java.io.File; -import java.io.IOException; -import java.text.SimpleDateFormat; -import java.util.Date; - -/** - * Dunit class for testing gemfire function commands : export logs - */ -@Category(DistributedTest.class) -public class MiscellaneousCommandsExportLogsPart1DUnitTest extends CliCommandTestBase { - - private static final long serialVersionUID = 1L; - - void setupForExportLogs() { - final VM vm1 = Host.getHost(0).getVM(1); - setUpJmxManagerOnVm0ThenConnect(null); - - vm1.invoke(new SerializableRunnable() { - public void run() { - // no need to close cache as it will be closed as part of teardown2 - Cache cache = getCache(); - - RegionFactory<Integer, Integer> dataRegionFactory = - cache.createRegionFactory(RegionShortcut.PARTITION); - Region region = dataRegionFactory.create("testRegion"); - for (int i = 0; i < 5; i++) { - region.put("key" + (i + 200), "value" + (i + 200)); - } - } - }); - } - - String getCurrentTimeString() { - SimpleDateFormat sf = new SimpleDateFormat("yyyy_MM_dd_HH_mm_ss_SSS_z"); - Date startDate = new Date(System.currentTimeMillis()); - String formattedStartDate = sf.format(startDate); - return ("_" + formattedStartDate); - } - - @Test - public void testExportLogs() throws IOException { - Date startDate = new Date(System.currentTimeMillis() - 2 * 60 * 1000); - SimpleDateFormat sf = new SimpleDateFormat("yyyy/MM/dd"); - String start = sf.format(startDate); - - Date enddate = new Date(System.currentTimeMillis() + 2 * 60 * 60 * 1000); - String end = sf.format(enddate); - String dir = getCurrentTimeString(); - - setupForExportLogs(); - String logLevel = LogWriterImpl.levelToString(LogWriterImpl.INFO_LEVEL); - - MiscellaneousCommands misc = new MiscellaneousCommands(); - getCache(); - - Result cmdResult = misc.exportLogsPreprocessing("./testExportLogs" + dir, null, null, logLevel, - false, false, start, end, 1); - - getLogWriter().info("testExportLogs command result =" + cmdResult); - - if (cmdResult != null) { - String cmdStringRsult = commandResultToString((CommandResult) cmdResult); - getLogWriter().info("testExportLogs cmdStringRsult=" + cmdStringRsult); - assertEquals(Result.Status.OK, cmdResult.getStatus()); - } else { - fail("testExportLogs failed as did not get CommandResult"); - } - FileUtils.deleteDirectory(new File("./testExportLogs" + dir)); - } - - @Category(FlakyTest.class) // GEODE-1477 (http) - @Test - public void testExportLogsForMerge() throws IOException { - setupForExportLogs(); - Date startDate = new Date(System.currentTimeMillis() - 2 * 60 * 1000); - SimpleDateFormat sf = new SimpleDateFormat("yyyy/MM/dd"); - String start = sf.format(startDate); - - Date enddate = new Date(System.currentTimeMillis() + 2 * 60 * 60 * 1000); - String end = sf.format(enddate); - String dir = getCurrentTimeString(); - - String logLevel = LogWriterImpl.levelToString(LogWriterImpl.INFO_LEVEL); - - MiscellaneousCommands misc = new MiscellaneousCommands(); - getCache(); - - Result cmdResult = misc.exportLogsPreprocessing("./testExportLogsForMerge" + dir, null, null, - logLevel, false, true, start, end, 1); - getLogWriter().info("testExportLogsForMerge command=" + cmdResult); - - if (cmdResult != null) { - String cmdStringRsult = commandResultToString((CommandResult) cmdResult); - getLogWriter().info("testExportLogsForMerge cmdStringRsult=" + cmdStringRsult); - - assertEquals(Result.Status.OK, cmdResult.getStatus()); - } else { - fail("testExportLogsForMerge failed as did not get CommandResult"); - } - FileUtils.deleteDirectory(new File("./testExportLogsForMerge" + dir)); - } -}
