Updated Branches: refs/heads/1.4.5-SNAPSHOT 07a0433bd -> 335f693a4
http://git-wip-us.apache.org/repos/asf/accumulo/blob/335f693a/src/server/src/main/java/org/apache/accumulo/server/test/continuous/UndefinedAnalyzer.java ---------------------------------------------------------------------- diff --git a/src/server/src/main/java/org/apache/accumulo/server/test/continuous/UndefinedAnalyzer.java b/src/server/src/main/java/org/apache/accumulo/server/test/continuous/UndefinedAnalyzer.java index 491b8ec..2caf7cd 100644 --- a/src/server/src/main/java/org/apache/accumulo/server/test/continuous/UndefinedAnalyzer.java +++ b/src/server/src/main/java/org/apache/accumulo/server/test/continuous/UndefinedAnalyzer.java @@ -49,96 +49,100 @@ import org.apache.hadoop.io.Text; * */ public class UndefinedAnalyzer { - + static class UndefinedNode { - + public UndefinedNode(String undef2, String ref2) { this.undef = undef2; this.ref = ref2; } - + String undef; String ref; } - + static class IngestInfo { - + Map<String,TreeMap<Long,Long>> flushes = new HashMap<String,TreeMap<Long,Long>>(); - + public IngestInfo(String logDir) throws Exception { File dir = new File(logDir); File[] ingestLogs = dir.listFiles(new FilenameFilter() { + @Override public boolean accept(java.io.File dir, String name) { return name.endsWith("ingest.out"); } }); - + for (File log : ingestLogs) { parseLog(log); } } - + private void parseLog(File log) throws Exception { BufferedReader reader = new BufferedReader(new FileReader(log)); - - String line; - TreeMap<Long,Long> tm = null; - - while ((line = reader.readLine()) != null) { - if (!line.startsWith("UUID")) - continue; - String[] tokens = line.split("\\s"); - String time = tokens[1]; - String uuid = tokens[2]; - - if (flushes.containsKey(uuid)) { - System.err.println("WARN Duplicate uuid " + log); + try { + String line; + TreeMap<Long,Long> tm = null; + + while ((line = reader.readLine()) != null) { + if (!line.startsWith("UUID")) + continue; + String[] tokens = line.split("\\s"); + String time = tokens[1]; + String uuid = tokens[2]; + + if (flushes.containsKey(uuid)) { + System.err.println("WARN Duplicate uuid " + log); + return; + } + + tm = new TreeMap<Long,Long>(Collections.reverseOrder()); + tm.put(0l, Long.parseLong(time)); + flushes.put(uuid, tm); + break; + + } + + if (tm == null) { + System.err.println("WARN Bad ingest log " + log); return; } - - tm = new TreeMap<Long,Long>(Collections.reverseOrder()); - tm.put(0l, Long.parseLong(time)); - flushes.put(uuid, tm); - break; - - } - - if (tm == null) { - System.err.println("WARN Bad ingest log " + log); - return; - } - - while ((line = reader.readLine()) != null) { - String[] tokens = line.split("\\s"); - - if (!tokens[0].equals("FLUSH")) - continue; - - String time = tokens[1]; - String count = tokens[4]; - - tm.put(Long.parseLong(count), Long.parseLong(time)); + + while ((line = reader.readLine()) != null) { + String[] tokens = line.split("\\s"); + + if (!tokens[0].equals("FLUSH")) + continue; + + String time = tokens[1]; + String count = tokens[4]; + + tm.put(Long.parseLong(count), Long.parseLong(time)); + } + } finally { + if (reader != null) + reader.close(); } - } - + Iterator<Long> getTimes(String uuid, long count) { TreeMap<Long,Long> tm = flushes.get(uuid); - + if (tm == null) return null; - + return tm.tailMap(count).values().iterator(); } } - + static class TabletAssignment { String tablet; String endRow; String prevEndRow; String server; long time; - + TabletAssignment(String tablet, String er, String per, String server, long time) { this.tablet = new String(tablet); this.endRow = new String(er); @@ -146,133 +150,139 @@ public class UndefinedAnalyzer { this.server = new String(server); this.time = time; } - + public boolean contains(String row) { return prevEndRow.compareTo(row) < 0 && endRow.compareTo(row) >= 0; } } - + static class TabletHistory { - + List<TabletAssignment> assignments = new ArrayList<TabletAssignment>(); - + TabletHistory(String tableId, String acuLogDir) throws Exception { File dir = new File(acuLogDir); File[] masterLogs = dir.listFiles(new FilenameFilter() { + @Override public boolean accept(java.io.File dir, String name) { return name.matches("master.*debug.log.*"); } }); - + SimpleDateFormat sdf = new SimpleDateFormat("dd HH:mm:ss,SSS yyyy MM"); String currentYear = (Calendar.getInstance().get(Calendar.YEAR)) + ""; String currentMonth = (Calendar.getInstance().get(Calendar.MONTH) + 1) + ""; - + for (File masterLog : masterLogs) { - + BufferedReader reader = new BufferedReader(new FileReader(masterLog)); - String line; - while ((line = reader.readLine()) != null) { - if (line.contains("TABLET_LOADED")) { - String[] tokens = line.split("\\s+"); - String tablet = tokens[8]; - String server = tokens[10]; - - int pos1 = -1; - int pos2 = -1; - int pos3 = -1; - - for (int i = 0; i < tablet.length(); i++) { - if (tablet.charAt(i) == '<' || tablet.charAt(i) == ';') { - if (pos1 == -1) { - pos1 = i; - } else if (pos2 == -1) { - pos2 = i; - } else { - pos3 = i; + try { + String line; + while ((line = reader.readLine()) != null) { + if (line.contains("TABLET_LOADED")) { + String[] tokens = line.split("\\s+"); + String tablet = tokens[8]; + String server = tokens[10]; + + int pos1 = -1; + int pos2 = -1; + int pos3 = -1; + + for (int i = 0; i < tablet.length(); i++) { + if (tablet.charAt(i) == '<' || tablet.charAt(i) == ';') { + if (pos1 == -1) { + pos1 = i; + } else if (pos2 == -1) { + pos2 = i; + } else { + pos3 = i; + } } } - } - - if (pos1 > 0 && pos2 > 0 && pos3 == -1) { - String tid = tablet.substring(0, pos1); - String endRow = tablet.charAt(pos1) == '<' ? "8000000000000000" : tablet.substring(pos1 + 1, pos2); - String prevEndRow = tablet.charAt(pos2) == '<' ? "" : tablet.substring(pos2 + 1); - if (tid.equals(tableId)) { - // System.out.println(" "+server+" "+tid+" "+endRow+" "+prevEndRow); - Date date = sdf.parse(tokens[0] + " " + tokens[1] + " " + currentYear + " " + currentMonth); - // System.out.println(" "+date); - - assignments.add(new TabletAssignment(tablet, endRow, prevEndRow, server, date.getTime())); - + + if (pos1 > 0 && pos2 > 0 && pos3 == -1) { + String tid = tablet.substring(0, pos1); + String endRow = tablet.charAt(pos1) == '<' ? "8000000000000000" : tablet.substring(pos1 + 1, pos2); + String prevEndRow = tablet.charAt(pos2) == '<' ? "" : tablet.substring(pos2 + 1); + if (tid.equals(tableId)) { + // System.out.println(" "+server+" "+tid+" "+endRow+" "+prevEndRow); + Date date = sdf.parse(tokens[0] + " " + tokens[1] + " " + currentYear + " " + currentMonth); + // System.out.println(" "+date); + + assignments.add(new TabletAssignment(tablet, endRow, prevEndRow, server, date.getTime())); + + } + } else if (!tablet.startsWith("!0")) { + System.err.println("Cannot parse tablet " + tablet); } - } else if (!tablet.startsWith("!0")) { - System.err.println("Cannot parse tablet " + tablet); + } - } + } finally { + if (reader != null) + reader.close(); } } } - + TabletAssignment findMostRecentAssignment(String row, long time1, long time2) { - + long latest = Long.MIN_VALUE; TabletAssignment ret = null; - + for (TabletAssignment assignment : assignments) { if (assignment.contains(row) && assignment.time <= time2 && assignment.time > latest) { latest = assignment.time; ret = assignment; } } - + return ret; } } - + public static void main(String[] args) throws Exception { - + if (args.length != 7) { System.err.println("Usage : " + UndefinedAnalyzer.class.getName() + " <instance> <zoo> <user> <pass> <table> <ci log dir> <acu log dir>"); return; } - + String instanceName = args[0]; String zooKeepers = args[1]; - + String user = args[2]; String password = args[3]; - + String table = args[4]; String logDir = args[5]; String acuLogDir = args[6]; - + List<UndefinedNode> undefs = new ArrayList<UndefinedNode>(); - + BufferedReader reader = new BufferedReader(new InputStreamReader(System.in)); String line; while ((line = reader.readLine()) != null) { String[] tokens = line.split("\\s"); String undef = tokens[0]; String ref = tokens[1]; - + undefs.add(new UndefinedNode(undef, ref)); } - + ZooKeeperInstance zki = new ZooKeeperInstance(instanceName, zooKeepers); Connector conn = zki.getConnector(user, password.getBytes()); BatchScanner bscanner = conn.createBatchScanner(table, Constants.NO_AUTHS, 20); - + List<Range> refs = new ArrayList<Range>(); - + for (UndefinedNode undefinedNode : undefs) refs.add(new Range(new Text(undefinedNode.ref))); - + bscanner.setRanges(refs); - + HashMap<String,List<String>> refInfo = new HashMap<String,List<String>>(); - + for (Entry<Key,Value> entry : bscanner) { String ref = entry.getKey().getRow().toString(); List<String> vals = refInfo.get(ref); @@ -280,32 +290,32 @@ public class UndefinedAnalyzer { vals = new ArrayList<String>(); refInfo.put(ref, vals); } - + vals.add(entry.getValue().toString()); } - + bscanner.close(); - + IngestInfo ingestInfo = new IngestInfo(logDir); TabletHistory tabletHistory = new TabletHistory(Tables.getTableId(zki, table), acuLogDir); - + SimpleDateFormat sdf = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss"); - + for (UndefinedNode undefinedNode : undefs) { - + List<String> refVals = refInfo.get(undefinedNode.ref); if (refVals != null) { for (String refVal : refVals) { TabletAssignment ta = null; - + String[] tokens = refVal.split(":"); - + String uuid = tokens[0]; String count = tokens[1]; - + String t1 = ""; String t2 = ""; - + Iterator<Long> times = ingestInfo.getTimes(uuid, Long.parseLong(count, 16)); if (times != null) { if (times.hasNext()) { @@ -318,19 +328,19 @@ public class UndefinedAnalyzer { } } } - + if (ta == null) System.out.println(undefinedNode.undef + " " + undefinedNode.ref + " " + uuid + " " + t1 + " " + t2); else System.out.println(undefinedNode.undef + " " + undefinedNode.ref + " " + ta.tablet + " " + ta.server + " " + uuid + " " + t1 + " " + t2); - + } } else { System.out.println(undefinedNode.undef + " " + undefinedNode.ref); } - + } - + } - + } http://git-wip-us.apache.org/repos/asf/accumulo/blob/335f693a/src/server/src/main/java/org/apache/accumulo/server/util/SendLogToChainsaw.java ---------------------------------------------------------------------- diff --git a/src/server/src/main/java/org/apache/accumulo/server/util/SendLogToChainsaw.java b/src/server/src/main/java/org/apache/accumulo/server/util/SendLogToChainsaw.java index 3c3d8d7..4a0f5ec 100644 --- a/src/server/src/main/java/org/apache/accumulo/server/util/SendLogToChainsaw.java +++ b/src/server/src/main/java/org/apache/accumulo/server/util/SendLogToChainsaw.java @@ -18,8 +18,10 @@ package org.apache.accumulo.server.util; import java.io.BufferedReader; import java.io.File; +import java.io.FileNotFoundException; import java.io.FileReader; import java.io.FilenameFilter; +import java.io.IOException; import java.net.Socket; import java.net.URLEncoder; import java.text.SimpleDateFormat; @@ -48,33 +50,33 @@ import org.apache.log4j.varia.LevelRangeFilter; import org.apache.log4j.xml.XMLLayout; public class SendLogToChainsaw extends XMLLayout { - + private static Pattern logPattern = Pattern.compile( "^(\\d\\d)\\s(\\d\\d):(\\d\\d):(\\d\\d),(\\d\\d\\d)\\s\\[(.*)\\]\\s(TRACE|DEBUG|INFO|WARN|FATAL|ERROR)\\s*?:(.*)$", Pattern.UNIX_LINES); - + private File[] logFiles = null; - + private SocketFactory factory = SocketFactory.getDefault(); - + private WildcardFileFilter fileFilter = null; - + private Socket socket = null; - + private Pattern lineFilter = null; - + private LongRange dateFilter = null; - + private LevelRangeFilter levelFilter = null; - + public SendLogToChainsaw(String directory, String fileNameFilter, String host, int port, Date start, Date end, String regex, String level) throws Exception { - + // Set up the file name filter if (null != fileNameFilter) { fileFilter = new WildcardFileFilter(fileNameFilter); } else { fileFilter = new WildcardFileFilter("*"); } - + // Get the list of files that match File dir = new File(directory); if (dir.isDirectory()) { @@ -82,22 +84,22 @@ public class SendLogToChainsaw extends XMLLayout { } else { throw new IllegalArgumentException(directory + " is not a directory or is not readable."); } - + if (logFiles.length == 0) { throw new IllegalArgumentException("No files match the supplied filter."); } - + socket = factory.createSocket(host, port); - + lineFilter = Pattern.compile(regex); - + // Create Date Filter if (null != start) { if (end == null) end = new Date(System.currentTimeMillis()); dateFilter = new LongRange(start.getTime(), end.getTime()); } - + if (null != level) { Level base = Level.toLevel(level.toUpperCase()); levelFilter = new LevelRangeFilter(); @@ -106,43 +108,60 @@ public class SendLogToChainsaw extends XMLLayout { levelFilter.setLevelMax(Level.FATAL); } } - + public void processLogFiles() throws Exception { - for (File log : logFiles) { - // Parse the server type and name from the log file name - String threadName = log.getName().substring(0, log.getName().indexOf(".")); - FileReader fReader = new FileReader(log); - BufferedReader reader = new BufferedReader(fReader); - - String line = reader.readLine(); - while (null != line) { - String out = null; + String line = null; + String out = null; + FileReader fReader = null; + BufferedReader reader = null; + try { + for (File log : logFiles) { + // Parse the server type and name from the log file name + String threadName = log.getName().substring(0, log.getName().indexOf(".")); + try { + fReader = new FileReader(log); + } catch (FileNotFoundException e) { + System.out.println("Unable to find file: " + log.getAbsolutePath()); + throw e; + } + reader = new BufferedReader(fReader); + try { - out = convertLine(line, threadName); - if (null != out) { - if (socket != null && socket.isConnected()) - socket.getOutputStream().write(out.getBytes()); - else - System.err.println("Unable to send data to transport"); + line = reader.readLine(); + while (null != line) { + out = convertLine(line, threadName); + if (null != out) { + if (socket != null && socket.isConnected()) + socket.getOutputStream().write(out.getBytes()); + else + System.err.println("Unable to send data to transport"); + } + line = reader.readLine(); } - } catch (Exception e) { + } catch (IOException e) { System.out.println("Error processing line: " + line + ". Output was " + out); throw e; + } finally { + if (reader != null) { + reader.close(); + } + if (fReader != null) { + fReader.close(); + } } - line = reader.readLine(); } - reader.close(); - fReader.close(); + } finally { + if (socket != null && socket.isConnected()) { + socket.close(); + } } - if (socket != null && socket.isConnected()) - socket.close(); } - + private String convertLine(String line, String threadName) throws Exception { String result = null; Matcher m = logPattern.matcher(line); if (m.matches()) { - + Calendar cal = Calendar.getInstance(); cal.setTime(new Date(System.currentTimeMillis())); Integer date = Integer.parseInt(m.group(1)); @@ -188,54 +207,54 @@ public class SendLogToChainsaw extends XMLLayout { } return result; } - + private static Options getOptions() { Options opts = new Options(); - + Option dirOption = new Option("d", "logDirectory", true, "ACCUMULO log directory path"); dirOption.setArgName("dir"); dirOption.setRequired(true); opts.addOption(dirOption); - + Option fileFilterOption = new Option("f", "fileFilter", true, "filter to apply to names of logs"); fileFilterOption.setArgName("filter"); fileFilterOption.setRequired(false); opts.addOption(fileFilterOption); - + Option hostOption = new Option("h", "host", true, "host where chainsaw is running"); hostOption.setArgName("hostname"); hostOption.setRequired(true); opts.addOption(hostOption); - + Option portOption = new Option("p", "port", true, "port where XMLSocketReceiver is listening"); portOption.setArgName("portnum"); portOption.setRequired(true); opts.addOption(portOption); - + Option startOption = new Option("s", "start", true, "start date filter (yyyyMMddHHmmss)"); startOption.setArgName("date"); startOption.setRequired(true); opts.addOption(startOption); - + Option endOption = new Option("e", "end", true, "end date filter (yyyyMMddHHmmss)"); endOption.setArgName("date"); endOption.setRequired(true); opts.addOption(endOption); - + Option levelOption = new Option("l", "level", true, "filter log level"); levelOption.setArgName("level"); levelOption.setRequired(false); opts.addOption(levelOption); - + Option msgFilter = new Option("m", "messageFilter", true, "regex filter for log messages"); msgFilter.setArgName("regex"); msgFilter.setRequired(false); opts.addOption(msgFilter); - + return opts; - + } - + /** * * @param args @@ -245,11 +264,11 @@ public class SendLogToChainsaw extends XMLLayout { * @throws Exception */ public static void main(String[] args) throws Exception { - + Options o = getOptions(); CommandLine cl = null; cl = new BasicParser().parse(o, args); - + String logDir = cl.getOptionValue(o.getOption("d").getOpt()); String fileNameFilter = null; if (cl.hasOption(o.getOption("f").getOpt())) @@ -277,9 +296,9 @@ public class SendLogToChainsaw extends XMLLayout { String levelFilter = null; if (cl.hasOption(o.getOption("l").getOpt())) levelFilter = cl.getOptionValue(o.getOption("l").getOpt()); - + SendLogToChainsaw c = new SendLogToChainsaw(logDir, fileNameFilter, chainsawHost, chainsawPort, startDate, endDate, msgFilter, levelFilter); c.processLogFiles(); } - + }
