http://git-wip-us.apache.org/repos/asf/zookeeper/blob/63aaf0a1/zookeeper-contrib/zookeeper-contrib-loggraph/src/main/java/org/apache/zookeeper/graph/Log4JSource.java ---------------------------------------------------------------------- diff --git a/zookeeper-contrib/zookeeper-contrib-loggraph/src/main/java/org/apache/zookeeper/graph/Log4JSource.java b/zookeeper-contrib/zookeeper-contrib-loggraph/src/main/java/org/apache/zookeeper/graph/Log4JSource.java new file mode 100644 index 0000000..84a9d98 --- /dev/null +++ b/zookeeper-contrib/zookeeper-contrib-loggraph/src/main/java/org/apache/zookeeper/graph/Log4JSource.java @@ -0,0 +1,391 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zookeeper.graph; + +import java.io.File; +import java.io.InputStreamReader; +import java.io.BufferedReader; +import java.io.FileReader; +import java.io.IOException; +import java.util.regex.Pattern; +import java.util.regex.Matcher; +import java.util.ArrayList; +import java.util.Date; +import java.text.SimpleDateFormat; +import java.text.ParseException; +import java.util.Calendar; +import java.util.GregorianCalendar; + +import java.io.EOFException; +import java.io.Closeable; +import java.io.FileNotFoundException; +import java.util.Iterator; +import java.util.NoSuchElementException; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class Log4JSource implements LogSource { + private static final Logger LOG = LoggerFactory.getLogger(Log4JSource.class); + + private static final int skipN = 10000; + private static final String DATE_FORMAT = "yyyy-MM-dd HH:mm:ss,SSS"; + + private LogSkipList skiplist = null; + + private String file = null; + private long starttime = 0; + private long endtime = 0; + private int serverid = 0; + private long size = 0; + + private Pattern timep; + + public boolean overlapsRange(long starttime, long endtime) { + return (starttime <= this.endtime && endtime >= this.starttime); + } + + public long size() { return size; } + public long getStartTime() { return starttime; } + public long getEndTime() { return endtime; } + public LogSkipList getSkipList() { return skiplist; } + + private class Log4JSourceIterator implements LogIterator { + private RandomAccessFileReader in; + private LogEntry next = null; + private long starttime = 0; + private long endtime = 0; + private String buf = ""; + private Log4JSource src = null; + private long skippedAtStart = 0; + private SimpleDateFormat dateformat = null; + private FilterOp filter = null; + + public Log4JSourceIterator(Log4JSource src, long starttime, long endtime) throws IllegalArgumentException, FilterException { + this(src, starttime, endtime, null); + } + + public Log4JSourceIterator(Log4JSource src, long starttime, long endtime, FilterOp filter) throws IllegalArgumentException, FilterException { + + this.dateformat = new SimpleDateFormat(DATE_FORMAT); + this.src = src; + this.starttime = starttime; + this.endtime = endtime; + + File f = new File(src.file); + try { + in = new RandomAccessFileReader(f); + } catch (FileNotFoundException e) { + throw new IllegalArgumentException("Bad file passed in (" + src.file +") cannot open:" + e); + } + + // skip to the offset of latest skip point before starttime + LogSkipList.Mark start = src.getSkipList().findMarkBefore(starttime); + try { + in.seek(start.getBytes()); + skippedAtStart = start.getEntriesSkipped(); + } catch (IOException ioe) { + // if we can't skip, we should just read from the start + } + + LogEntry e; + while ((e = readNextEntry()) != null && e.getTimestamp() < endtime) { + if (e.getTimestamp() >= starttime && (filter == null || filter.matches(e))) { + next = e; + return; + } + skippedAtStart++; + } + this.filter = filter; + } + + synchronized public long size() throws IOException { + if (LOG.isTraceEnabled()) { + LOG.trace("size() called"); + } + + if (this.endtime >= src.getEndTime()) { + return src.size() - skippedAtStart; + } + + long pos = in.getPosition(); + + if (LOG.isTraceEnabled()) { + LOG.trace("saved pos () = " + pos); + } + + LogEntry e; + + LogSkipList.Mark lastseg = src.getSkipList().findMarkBefore(this.endtime); + in.seek(lastseg.getBytes()); + buf = ""; // clear the buf so we don't get something we read before we sought + // number of entries skipped to get to the end of the iterator, less the number skipped to get to the start + long count = lastseg.getEntriesSkipped() - skippedAtStart; + + while ((e = readNextEntry()) != null) { + if (LOG.isTraceEnabled()) { + //LOG.trace(e); + } + if (e.getTimestamp() > this.endtime) { + break; + } + count++; + } + in.seek(pos); + buf = ""; + + if (LOG.isTraceEnabled()) { + LOG.trace("size() = " + count); + } + + return count; + } + + synchronized private LogEntry readNextEntry() { + try { + try { + while (true) { + String line = in.readLine(); + if (line == null) { + break; + } + + Matcher m = src.timep.matcher(line); + if (m.lookingAt()) { + if (buf.length() > 0) { + LogEntry e = new Log4JEntry(src.timestampFromText(dateformat, buf), src.getServerId(), buf); + buf = line; + return e; + } + buf = line; + } else if (buf.length() > 0) { + buf += line + "\n"; + } + } + } catch (EOFException eof) { + // ignore, we've simply come to the end of the file + } + if (buf.length() > 0) { + LogEntry e = new Log4JEntry(src.timestampFromText(dateformat, buf), src.getServerId(), buf); + buf = ""; + return e; + } + } catch (Exception e) { + LOG.error("Error reading next entry in file (" + src.file + "): " + e); + return null; + } + return null; + } + + public boolean hasNext() { + return next != null; + } + + public LogEntry next() throws NoSuchElementException { + LogEntry ret = next; + LogEntry e = readNextEntry(); + + if (filter != null) { + try { + while (e != null && !filter.matches(e)) { + e = readNextEntry(); + } + } catch (FilterException fe) { + throw new NoSuchElementException(e.toString()); + } + } + + if (e != null && e.getTimestamp() < endtime) { + next = e; + } else { + next = null; + } + return ret; + } + + public void remove() throws UnsupportedOperationException { + throw new UnsupportedOperationException("remove not supported for L4J logs"); + } + + public void close() throws IOException { + in.close(); + } + + public String toString() { + String size; + try { + size = new Long(size()).toString(); + } catch (IOException ioe) { + size = "Unable to read"; + } + return "Log4JSourceIterator(start=" + starttime + ", end=" + endtime + ", size=" + size + ")"; + } + } + + public LogIterator iterator(long starttime, long endtime) throws IllegalArgumentException { + try { + return iterator(starttime, endtime, null); + } catch (FilterException fe) { + assert(false); //"This should never happen, you can't have a filter exception without a filter"); + return null; + } + } + + public LogIterator iterator(long starttime, long endtime, FilterOp filter) throws IllegalArgumentException, FilterException{ + // sanitise start and end times + if (endtime < starttime) { + throw new IllegalArgumentException("End time (" + endtime + ") must be greater or equal to starttime (" + starttime + ")"); + } + + return new Log4JSourceIterator(this, starttime, endtime, filter); + } + + public LogIterator iterator() throws IllegalArgumentException { + return iterator(starttime, endtime+1); + } + + public Log4JSource(String file) throws IOException { + this.file=file; + + timep = Pattern.compile("^(\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2},\\d{3})"); + skiplist = new LogSkipList(); + init(); + } + + private static long timestampFromText(SimpleDateFormat format, String s) { + Date d = null; + try { + d = format.parse(s); + } catch (ParseException e) { + return 0; + } + Calendar c = new GregorianCalendar(); + c.setTime(d); + return c.getTimeInMillis(); + } + + private void init() throws IOException { + File f = new File(file); + RandomAccessFileReader in = new RandomAccessFileReader(f); + SimpleDateFormat dateformat = new SimpleDateFormat(DATE_FORMAT); + Pattern idp = Pattern.compile("\\[myid:(\\d+)\\]"); + + long lastFp = in.getPosition(); + String line = in.readLine(); + Matcher m = null; + + // if we have read data from the file, and it matchs the timep pattern + if ((line != null) && (m = timep.matcher(line)).lookingAt()) { + starttime = timestampFromText(dateformat, m.group(1)); + } else { + throw new IOException("Invalid log4j format. First line doesn't start with time"); + } + + /* + Count number of log entries. Any line starting with a timestamp counts as an entry + */ + String lastentry = line; + try { + while (line != null) { + m = timep.matcher(line); + if (m.lookingAt()) { + if (size % skipN == 0) { + long time = timestampFromText(dateformat, m.group(1)); + skiplist.addMark(time, lastFp, size); + } + size++; + lastentry = line; + } + if (serverid == 0 && (m = idp.matcher(line)).find()) { + serverid = Integer.valueOf(m.group(1)); + } + + lastFp = in.getPosition(); + line = in.readLine(); + } + } catch (EOFException eof) { + // ignore, simply end of file, though really (line!=null) should have caught this + } finally { + in.close(); + } + + m = timep.matcher(lastentry); + if (m.lookingAt()) { + endtime = timestampFromText(dateformat, m.group(1)); + } else { + throw new IOException("Invalid log4j format. Last line doesn't start with time"); + } + } + + public String toString() { + return "Log4JSource(file=" + file + ", size=" + size + ", start=" + starttime + ", end=" + endtime +", id=" + serverid +")"; + } + + public static void main(String[] args) throws IOException { + final Log4JSource s = new Log4JSource(args[0]); + System.out.println(s); + + LogIterator iter; + + if (args.length == 3) { + final long starttime = Long.valueOf(args[1]); + final long endtime = Long.valueOf(args[2]); + iter = s.iterator(starttime, endtime); + + Thread t1 = new Thread() { public void run () { + + LogIterator iter = s.iterator(starttime, endtime); + System.out.println(iter); + try { + iter.close(); + } catch (IOException ioe) { + System.out.println(ioe.getMessage()); + } + }; }; + Thread t2 = new Thread() { public void run () { + + LogIterator iter = s.iterator(starttime, endtime); + System.out.println(iter); + try { + iter.close(); + } catch (IOException ioe) { + System.out.println(ioe.getMessage()); + } + }; }; + Thread t3 = new Thread() { public void run () { + + LogIterator iter = s.iterator(starttime, endtime); + System.out.println(iter); + }; }; + t1.start(); + t2.start(); + // t3.start(); + } else { + iter = s.iterator(); + } + + /*while (iter.hasNext()) { + System.out.println(iter.next()); + }*/ + iter.close(); + } + + public int getServerId() { + return serverid; + } +}
http://git-wip-us.apache.org/repos/asf/zookeeper/blob/63aaf0a1/zookeeper-contrib/zookeeper-contrib-loggraph/src/main/java/org/apache/zookeeper/graph/LogEntry.java ---------------------------------------------------------------------- diff --git a/zookeeper-contrib/zookeeper-contrib-loggraph/src/main/java/org/apache/zookeeper/graph/LogEntry.java b/zookeeper-contrib/zookeeper-contrib-loggraph/src/main/java/org/apache/zookeeper/graph/LogEntry.java new file mode 100644 index 0000000..a8252eb --- /dev/null +++ b/zookeeper-contrib/zookeeper-contrib-loggraph/src/main/java/org/apache/zookeeper/graph/LogEntry.java @@ -0,0 +1,46 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zookeeper.graph; + +import java.io.Serializable; +import java.util.HashMap; + +public abstract class LogEntry implements Serializable { + private HashMap attributes; + + public enum Type { UNKNOWN, LOG4J, TXN }; + + public LogEntry(long timestamp) { + attributes = new HashMap(); + setAttribute("timestamp", new Long(timestamp)); + } + + public long getTimestamp() { + return (Long)getAttribute("timestamp"); + } + + public abstract Type getType(); + + public void setAttribute(String key, Object v) { + attributes.put(key, v); + } + + public Object getAttribute(String key) { + return attributes.get(key); + } +} http://git-wip-us.apache.org/repos/asf/zookeeper/blob/63aaf0a1/zookeeper-contrib/zookeeper-contrib-loggraph/src/main/java/org/apache/zookeeper/graph/LogIterator.java ---------------------------------------------------------------------- diff --git a/zookeeper-contrib/zookeeper-contrib-loggraph/src/main/java/org/apache/zookeeper/graph/LogIterator.java b/zookeeper-contrib/zookeeper-contrib-loggraph/src/main/java/org/apache/zookeeper/graph/LogIterator.java new file mode 100644 index 0000000..9af440b --- /dev/null +++ b/zookeeper-contrib/zookeeper-contrib-loggraph/src/main/java/org/apache/zookeeper/graph/LogIterator.java @@ -0,0 +1,26 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zookeeper.graph; + +import java.io.Closeable; +import java.util.Iterator; +import java.io.IOException; + +public interface LogIterator extends Iterator<LogEntry>, Closeable { + long size() throws IOException;; +}; http://git-wip-us.apache.org/repos/asf/zookeeper/blob/63aaf0a1/zookeeper-contrib/zookeeper-contrib-loggraph/src/main/java/org/apache/zookeeper/graph/LogServer.java ---------------------------------------------------------------------- diff --git a/zookeeper-contrib/zookeeper-contrib-loggraph/src/main/java/org/apache/zookeeper/graph/LogServer.java b/zookeeper-contrib/zookeeper-contrib-loggraph/src/main/java/org/apache/zookeeper/graph/LogServer.java new file mode 100644 index 0000000..5cffcdd --- /dev/null +++ b/zookeeper-contrib/zookeeper-contrib-loggraph/src/main/java/org/apache/zookeeper/graph/LogServer.java @@ -0,0 +1,66 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zookeeper.graph; + +import java.io.IOException; + +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import javax.servlet.ServletException; + +import java.io.IOException; + +import org.eclipse.jetty.server.Server; +import org.eclipse.jetty.server.Request; +import org.eclipse.jetty.server.handler.AbstractHandler; +import org.eclipse.jetty.servlet.ServletContextHandler; +import org.eclipse.jetty.servlet.ServletHolder; + +import org.apache.zookeeper.graph.servlets.*; + +public class LogServer extends ServletContextHandler { + public LogServer(MergedLogSource src) throws Exception { + super(ServletContextHandler.SESSIONS); + setContextPath("/"); + + addServlet(new ServletHolder(new StaticContent()),"/graph/*"); + + addServlet(new ServletHolder(new Fs()),"/fs"); + addServlet(new ServletHolder(new GraphData(src)), "/data"); + addServlet(new ServletHolder(new FileLoader(src)), "/loadfile"); + addServlet(new ServletHolder(new NumEvents(src)), "/info"); + addServlet(new ServletHolder(new Throughput(src)), "/throughput"); + } + + public static void main(String[] args) { + try { + MergedLogSource src = new MergedLogSource(args); + System.out.println(src); + + Server server = new Server(8182); + server.setHandler(new LogServer(src)); + + server.start(); + server.join(); + + } catch (Exception e) { + // Something is wrong. + e.printStackTrace(); + } + } +} http://git-wip-us.apache.org/repos/asf/zookeeper/blob/63aaf0a1/zookeeper-contrib/zookeeper-contrib-loggraph/src/main/java/org/apache/zookeeper/graph/LogSkipList.java ---------------------------------------------------------------------- diff --git a/zookeeper-contrib/zookeeper-contrib-loggraph/src/main/java/org/apache/zookeeper/graph/LogSkipList.java b/zookeeper-contrib/zookeeper-contrib-loggraph/src/main/java/org/apache/zookeeper/graph/LogSkipList.java new file mode 100644 index 0000000..e744442 --- /dev/null +++ b/zookeeper-contrib/zookeeper-contrib-loggraph/src/main/java/org/apache/zookeeper/graph/LogSkipList.java @@ -0,0 +1,95 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zookeeper.graph; + +import java.util.List; +import java.util.LinkedList; +import java.util.NoSuchElementException; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** +Generic skip list for holding a rough index of a log file. When the log file is loaded, this +index is built by adding a mark every n entries. Then when a specific time position is requested +from the file, a point at most n-1 entries before the time position can be jumped to. + +*/ +public class LogSkipList { + private static final Logger LOG = LoggerFactory.getLogger(LogSkipList.class); + + private LinkedList<Mark> marks; + + public class Mark { + private long time; + private long bytes; + private long skipped; + + public Mark(long time, long bytes, long skipped) { + this.time = time; + this.bytes = bytes; + this.skipped = skipped; + } + + public long getTime() { return this.time; } + public long getBytes() { return this.bytes; } + public long getEntriesSkipped() { return this.skipped; } + + public String toString() { + return "Mark(time=" + time + ", bytes=" + bytes + ", skipped=" + skipped + ")"; + } + }; + + public LogSkipList() { + if (LOG.isTraceEnabled()) { + LOG.trace("New skip list"); + } + marks = new LinkedList<Mark>(); + } + + public void addMark(long time, long bytes, long skipped) { + if (LOG.isTraceEnabled()) { + LOG.trace("addMark (time:" + time + ", bytes: " + bytes + ", skipped: " + skipped + ")"); + } + marks.add(new Mark(time, bytes, skipped)); + } + + /** + Find the last mark in the skip list before time. + */ + public Mark findMarkBefore(long time) throws NoSuchElementException { + if (LOG.isTraceEnabled()) { + LOG.trace("findMarkBefore(" + time + ")"); + } + + Mark last = marks.getFirst(); + for (Mark m: marks) { + if (m.getTime() > time) { + break; + } + last = m; + } + + if (LOG.isTraceEnabled()) { + LOG.trace("return " + last ); + } + + return last; + } + +}; http://git-wip-us.apache.org/repos/asf/zookeeper/blob/63aaf0a1/zookeeper-contrib/zookeeper-contrib-loggraph/src/main/java/org/apache/zookeeper/graph/LogSource.java ---------------------------------------------------------------------- diff --git a/zookeeper-contrib/zookeeper-contrib-loggraph/src/main/java/org/apache/zookeeper/graph/LogSource.java b/zookeeper-contrib/zookeeper-contrib-loggraph/src/main/java/org/apache/zookeeper/graph/LogSource.java new file mode 100644 index 0000000..9845c7f --- /dev/null +++ b/zookeeper-contrib/zookeeper-contrib-loggraph/src/main/java/org/apache/zookeeper/graph/LogSource.java @@ -0,0 +1,33 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zookeeper.graph; +import java.util.Iterator; + +public interface LogSource extends Iterable<LogEntry> { + public LogIterator iterator(long starttime, long endtime, FilterOp filter) throws IllegalArgumentException, FilterException; + + public LogIterator iterator(long starttime, long endtime) throws IllegalArgumentException; + + public LogIterator iterator() throws IllegalArgumentException; + + public boolean overlapsRange(long starttime, long endtime); + + public long size(); + public long getStartTime(); + public long getEndTime(); +} http://git-wip-us.apache.org/repos/asf/zookeeper/blob/63aaf0a1/zookeeper-contrib/zookeeper-contrib-loggraph/src/main/java/org/apache/zookeeper/graph/MeasureThroughput.java ---------------------------------------------------------------------- diff --git a/zookeeper-contrib/zookeeper-contrib-loggraph/src/main/java/org/apache/zookeeper/graph/MeasureThroughput.java b/zookeeper-contrib/zookeeper-contrib-loggraph/src/main/java/org/apache/zookeeper/graph/MeasureThroughput.java new file mode 100644 index 0000000..1c83da7 --- /dev/null +++ b/zookeeper-contrib/zookeeper-contrib-loggraph/src/main/java/org/apache/zookeeper/graph/MeasureThroughput.java @@ -0,0 +1,103 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zookeeper.graph; + +import java.io.IOException; +import java.io.BufferedOutputStream; +import java.io.FileOutputStream; +import java.io.DataOutputStream; +import java.io.PrintStream; + +import java.util.HashSet; + +public class MeasureThroughput { + private static final int MS_PER_SEC = 1000; + private static final int MS_PER_MIN = MS_PER_SEC*60; + private static final int MS_PER_HOUR = MS_PER_MIN*60; + + public static void main(String[] args) throws IOException { + MergedLogSource source = new MergedLogSource(args); + + PrintStream ps_ms = new PrintStream(new BufferedOutputStream(new FileOutputStream("throughput-ms.out"))); + PrintStream ps_sec = new PrintStream(new BufferedOutputStream(new FileOutputStream("throughput-sec.out"))); + PrintStream ps_min = new PrintStream(new BufferedOutputStream(new FileOutputStream("throughput-min.out"))); + PrintStream ps_hour = new PrintStream(new BufferedOutputStream(new FileOutputStream("throughput-hour.out"))); + LogIterator iter; + + System.out.println(source); + iter = source.iterator(); + long currentms = 0; + long currentsec = 0; + long currentmin = 0; + long currenthour = 0; + HashSet<Long> zxids_ms = new HashSet<Long>(); + long zxid_sec = 0; + long zxid_min = 0; + long zxid_hour = 0; + + while (iter.hasNext()) { + LogEntry e = iter.next(); + TransactionEntry cxn = (TransactionEntry)e; + + long ms = cxn.getTimestamp(); + long sec = ms/MS_PER_SEC; + long min = ms/MS_PER_MIN; + long hour = ms/MS_PER_HOUR; + + if (currentms != ms && currentms != 0) { + ps_ms.println("" + currentms + " " + zxids_ms.size()); + + zxid_sec += zxids_ms.size(); + zxid_min += zxids_ms.size(); + zxid_hour += zxids_ms.size(); + zxids_ms.clear(); + } + + if (currentsec != sec && currentsec != 0) { + ps_sec.println("" + currentsec*MS_PER_SEC + " " + zxid_sec); + + zxid_sec = 0; + } + + if (currentmin != min && currentmin != 0) { + ps_min.println("" + currentmin*MS_PER_MIN + " " + zxid_min); + + zxid_min = 0; + } + + if (currenthour != hour && currenthour != 0) { + ps_hour.println("" + currenthour*MS_PER_HOUR + " " + zxid_hour); + + zxid_hour = 0; + } + + currentms = ms; + currentsec = sec; + currentmin = min; + currenthour = hour; + + zxids_ms.add(cxn.getZxid()); + } + + iter.close(); + ps_ms.close(); + ps_sec.close(); + ps_min.close(); + ps_hour.close(); + } +}; http://git-wip-us.apache.org/repos/asf/zookeeper/blob/63aaf0a1/zookeeper-contrib/zookeeper-contrib-loggraph/src/main/java/org/apache/zookeeper/graph/MergedLogSource.java ---------------------------------------------------------------------- diff --git a/zookeeper-contrib/zookeeper-contrib-loggraph/src/main/java/org/apache/zookeeper/graph/MergedLogSource.java b/zookeeper-contrib/zookeeper-contrib-loggraph/src/main/java/org/apache/zookeeper/graph/MergedLogSource.java new file mode 100644 index 0000000..bb789d3 --- /dev/null +++ b/zookeeper-contrib/zookeeper-contrib-loggraph/src/main/java/org/apache/zookeeper/graph/MergedLogSource.java @@ -0,0 +1,219 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zookeeper.graph; + +import java.io.ByteArrayInputStream; +import java.io.EOFException; +import java.io.FileInputStream; +import java.io.IOException; +import java.text.DateFormat; +import java.util.Date; +import java.util.zip.Adler32; +import java.util.zip.Checksum; +import java.util.HashMap; + +import org.apache.jute.BinaryInputArchive; +import org.apache.jute.InputArchive; +import org.apache.jute.Record; +import org.apache.zookeeper.server.TraceFormatter; +import org.apache.zookeeper.server.persistence.FileHeader; +import org.apache.zookeeper.server.persistence.FileTxnLog; +import org.apache.zookeeper.server.util.SerializeUtils; +import org.apache.zookeeper.txn.TxnHeader; + +import org.apache.zookeeper.ZooDefs.OpCode; + +import org.apache.zookeeper.txn.CreateSessionTxn; +import org.apache.zookeeper.txn.CreateTxn; +import org.apache.zookeeper.txn.DeleteTxn; +import org.apache.zookeeper.txn.ErrorTxn; +import org.apache.zookeeper.txn.SetACLTxn; +import org.apache.zookeeper.txn.SetDataTxn; +import org.apache.zookeeper.txn.TxnHeader; + +import java.io.Closeable; +import java.io.FileNotFoundException; +import java.util.Vector; +import java.util.Iterator; +import java.util.Collections; +import java.util.NoSuchElementException; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class MergedLogSource implements LogSource { + private static final Logger LOG = LoggerFactory.getLogger(MergedLogSource.class); + private Vector<LogSource> sources = null; + private long starttime = 0; + private long endtime = 0; + private long size = 0; + + public boolean overlapsRange(long starttime, long endtime) { + return (starttime <= this.endtime && endtime >= this.starttime); + } + + public long size() { return size; } + public long getStartTime() { return starttime; } + public long getEndTime() { return endtime; } + + private class MergedLogSourceIterator implements LogIterator { + private LogEntry next = null; + private long start = 0; + private long end = 0; + private MergedLogSource src = null; + private LogIterator[] sources = null; + private LogEntry[] nexts = null; + private FilterOp filter = null; + + public MergedLogSourceIterator(MergedLogSource src, long starttime, long endtime, FilterOp filter) throws IllegalArgumentException, FilterException { + Vector<LogIterator> iters = new Vector<LogIterator>(); + for (LogSource s : src.sources) { + if (s.overlapsRange(starttime, endtime)) { + iters.add(s.iterator(starttime, endtime, filter)); + } + } + + sources = new LogIterator[iters.size()]; + sources = iters.toArray(sources); + nexts = new LogEntry[iters.size()]; + for (int i = 0; i < sources.length; i++) { + if (sources[i].hasNext()) + nexts[i] = sources[i].next(); + } + this.filter = filter; + } + + public MergedLogSourceIterator(MergedLogSource src, long starttime, long endtime) throws IllegalArgumentException, FilterException { + this(src, starttime, endtime, null); + } + + public long size() throws IOException { + long size = 0; + for (LogIterator i : sources) { + size += i.size(); + } + return size; + } + + public boolean hasNext() { + for (LogEntry n : nexts) { + if (n != null) return true; + } + return false; + } + + public LogEntry next() { + int min = -1; + for (int i = 0; i < nexts.length; i++) { + if (nexts[i] != null) { + if (min == -1) { + min = i; + } else if (nexts[i].getTimestamp() < nexts[min].getTimestamp()) { + min = i; + } + } + } + if (min == -1) { + return null; + } else { + LogEntry e = nexts[min]; + nexts[min] = sources[min].next(); + return e; + } + } + + public void remove() throws UnsupportedOperationException { + throw new UnsupportedOperationException("remove not supported for Merged logs"); + } + + public void close() throws IOException { + for (LogIterator i : sources) { + i.close(); + } + } + } + + public LogIterator iterator(long starttime, long endtime) throws IllegalArgumentException { + try { + return iterator(starttime, endtime, null); + } catch (FilterException fe) { + assert(false); // shouldn't happen without filter + return null; + } + } + + public LogIterator iterator(long starttime, long endtime, FilterOp filter) throws IllegalArgumentException, FilterException { + // sanitise start and end times + if (endtime < starttime) { + throw new IllegalArgumentException("End time (" + endtime + ") must be greater or equal to starttime (" + starttime + ")"); + } + + return new MergedLogSourceIterator(this, starttime, endtime, filter); + } + + public LogIterator iterator() throws IllegalArgumentException { + return iterator(starttime, endtime+1); + } + + public MergedLogSource(String[] files) throws IOException { + sources = new Vector<LogSource>(); + for (String f : files) { + addSource(f); + } + } + + public void addSource(String f) throws IOException { + LogSource s = null; + if (TxnLogSource.isTransactionFile(f)) { + s = new TxnLogSource(f); + } else { + s = new Log4JSource(f); + } + + size += s.size(); + endtime = s.getEndTime() > endtime ? s.getEndTime() : endtime; + starttime = s.getStartTime() < starttime || starttime == 0 ? s.getStartTime() : starttime; + sources.add(s); + } + + public String toString() { + String s = "MergedLogSource(size=" + size + ", start=" + starttime + ", end=" + endtime +")"; + for (LogSource src : sources) { + s += "\n\t- " +src; + } + return s; + } + + public static void main(String[] args) throws IOException { + System.out.println("Time: " + System.currentTimeMillis()); + MergedLogSource s = new MergedLogSource(args); + System.out.println(s); + + LogIterator iter; + + iter = s.iterator(); + System.out.println("Time: " + System.currentTimeMillis()); + System.out.println("Iterator Size: " + iter.size()); + System.out.println("Time: " + System.currentTimeMillis()); + /* while (iter.hasNext()) { + System.out.println(iter.next()); + }*/ + iter.close(); + System.out.println("Time: " + System.currentTimeMillis()); + } +} http://git-wip-us.apache.org/repos/asf/zookeeper/blob/63aaf0a1/zookeeper-contrib/zookeeper-contrib-loggraph/src/main/java/org/apache/zookeeper/graph/RandomAccessFileReader.java ---------------------------------------------------------------------- diff --git a/zookeeper-contrib/zookeeper-contrib-loggraph/src/main/java/org/apache/zookeeper/graph/RandomAccessFileReader.java b/zookeeper-contrib/zookeeper-contrib-loggraph/src/main/java/org/apache/zookeeper/graph/RandomAccessFileReader.java new file mode 100644 index 0000000..13a41a5 --- /dev/null +++ b/zookeeper-contrib/zookeeper-contrib-loggraph/src/main/java/org/apache/zookeeper/graph/RandomAccessFileReader.java @@ -0,0 +1,329 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zookeeper.graph; + +import java.io.File; +import java.io.Reader; +import java.io.IOException; +import java.io.EOFException; +import java.io.RandomAccessFile; +import java.io.FileNotFoundException; + +import java.io.DataInputStream; +import java.io.ByteArrayInputStream; +import java.io.DataInput; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class RandomAccessFileReader extends Reader implements DataInput { + private static final Logger LOG = LoggerFactory.getLogger(RandomAccessFileReader.class); + private RandomAccessFile file; + private byte[] buffer; + private int buffersize; + private int bufferoffset; + private long fileoffset; + private long fp; + + private static final int DEFAULT_BUFFER_SIZE = 512*1024; // 512k + private int point = 0; + + public RandomAccessFileReader(File f) throws FileNotFoundException { + file = new RandomAccessFile(f, "r"); + if (LOG.isDebugEnabled()) { + try { + LOG.debug("Opened file(" + f + ") with FD (" + file.getFD() + ")"); + } catch (IOException ioe) { + LOG.debug("Opened file(" + f + ") coulds get FD"); + } + } + + buffer = new byte[DEFAULT_BUFFER_SIZE]; + buffersize = 0; + bufferoffset = 0; + fileoffset = 0; + fp = 0; + } + + /** + fill the buffer from the file. + fp keeps track of the file pointer. + fileoffset is the offset into the file to where the buffer came from. + */ + private int fill() throws IOException { + fileoffset = fp; + int read = file.read(buffer, 0, buffer.length); + + if (LOG.isDebugEnabled()) { + String buf = new String(buffer, 0, 40, "UTF-8"); + LOG.debug("fill(buffer=" + buf + ")"); + } + + if (read == -1) { // eof reached + buffersize = 0; + } else { + buffersize = read; + } + fp += buffersize; + bufferoffset = 0; + + return buffersize; + } + + /** + * Reader interface + */ + public boolean markSupported() { return false; } + + /** + copy what we can from buffer. if it's not enough, fill buffer again and copy again + */ + synchronized public int read(char[] cbuf, int off, int len) throws IOException { + // This could be faster, but probably wont be used + byte[] b = new byte[2]; + int bytesread = 0; + while (len > 0) { + int read = read(b, 0, 2); + bytesread += read; + if (read < 2) { + return bytesread; + } + cbuf[off] = (char)((b[0] << 8) | (b[1] & 0xff)); + off += read; + len -= read; + } + + return bytesread; + } + + synchronized public int read(byte[] buf, int off, int len) throws IOException { + if (LOG.isTraceEnabled()) { + LOG.trace("read(buf, off=" + off + ", len=" + len); + } + + int read = 0; + while (len > 0) { + if (buffersize == 0) { + fill(); + if (buffersize == 0) { + break; + } + } + + int tocopy = Math.min(len, buffersize); + if (LOG.isTraceEnabled()) { + LOG.trace("tocopy=" + tocopy); + } + + System.arraycopy(buffer, bufferoffset, buf, off, tocopy); + buffersize -= tocopy; + bufferoffset += tocopy; + + len -= tocopy; + read += tocopy; + off += tocopy; + } + if (LOG.isTraceEnabled()) { + LOG.trace("read=" + read); + } + + return read; + } + + public void close() throws IOException { + file.close(); + } + + /** + * Seek interface + */ + public long getPosition() { + return bufferoffset + fileoffset; + } + + synchronized public void seek(long pos) throws IOException { + if (LOG.isDebugEnabled()) { + LOG.debug("seek(" + pos + ")"); + } + file.seek(pos); + fp = pos; + buffersize = 0; // force a buffer fill on next read + } + + /** + works like the usual readLine but disregards \r to make things easier + */ + synchronized public String readLine() throws IOException { + StringBuffer s = null; + + // go through buffer until i find a \n, if i reach end of buffer first, put whats in buffer into string buffer, + // repeat + buffering: + for (;;) { + if (buffersize == 0) { + fill(); + if (buffersize == 0) { + break; + } + } + + for (int i = 0; i < buffersize; i++) { + if (buffer[bufferoffset + i] == '\n') { + if (i > 0) { // if \n is first char in buffer, leave the string buffer empty + if (s == null) { s = new StringBuffer(); } + s.append(new String(buffer, bufferoffset, i, "UTF-8")); + } + bufferoffset += i+1; + buffersize -= i+1; + break buffering; + } + } + + // We didn't find \n, read the whole buffer into string buffer + if (s == null) { s = new StringBuffer(); } + s.append(new String(buffer, bufferoffset, buffersize, "UTF-8")); + buffersize = 0; + } + + if (s == null) { + return null; + } else { + return s.toString(); + } + } + + /** + DataInput interface + */ + public void readFully(byte[] b) throws IOException { + readFully(b, 0, b.length); + } + + public void readFully(byte[] b, int off, int len) throws IOException + { + while (len > 0) { + int read = read(b, off, len); + len -= read; + off += read; + + if (read == 0) { + throw new EOFException("End of file reached"); + } + } + } + + public int skipBytes(int n) throws IOException { + seek(getPosition() + n); + return n; + } + + public boolean readBoolean() throws IOException { + return (readByte() != 0); + } + + public byte readByte() throws IOException { + byte[] b = new byte[1]; + readFully(b, 0, 1); + return b[0]; + } + + public int readUnsignedByte() throws IOException { + return (int)readByte(); + } + + public short readShort() throws IOException { + byte[] b = new byte[2]; + readFully(b, 0, 2); + return (short)((b[0] << 8) | (b[1] & 0xff)); + } + + public int readUnsignedShort() throws IOException { + byte[] b = new byte[2]; + readFully(b, 0, 2); + return (((b[0] & 0xff) << 8) | (b[1] & 0xff)); + } + + public char readChar() throws IOException { + return (char)readShort(); + } + + public int readInt() throws IOException { + byte[] b = new byte[4]; + readFully(b, 0, 4); + return (((b[0] & 0xff) << 24) | ((b[1] & 0xff) << 16) | ((b[2] & 0xff) << 8) | (b[3] & 0xff)); + } + + public long readLong() throws IOException { + byte[] b = new byte[8]; + readFully(b, 0, 8); + + return (((long)(b[0] & 0xff) << 56) | ((long)(b[1] & 0xff) << 48) | + ((long)(b[2] & 0xff) << 40) | ((long)(b[3] & 0xff) << 32) | + ((long)(b[4] & 0xff) << 24) | ((long)(b[5] & 0xff) << 16) | + ((long)(b[6] & 0xff) << 8) | ((long)(b[7] & 0xff))); + } + + public float readFloat() throws IOException { + return Float.intBitsToFloat(readInt()); + } + + public double readDouble() throws IOException { + return Double.longBitsToDouble(readLong()); + } + + public String readUTF() throws IOException { + int len = readUnsignedShort(); + byte[] bytes = new byte[len+2]; + bytes[0] = (byte)((len >> 8) & 0xFF); + bytes[1] = (byte)(len & 0xFF); + readFully(bytes, 2, len); + DataInputStream dis = new DataInputStream(new ByteArrayInputStream(bytes)); + return dis.readUTF(); + } + + public static void main(String[] args) throws IOException { + RandomAccessFileReader f = new RandomAccessFileReader(new File(args[0])); + + long pos0 = f.getPosition(); + for (int i = 0; i < 5; i++) { + System.out.println(f.readLine()); + } + System.out.println("============="); + long pos1 = f.getPosition(); + System.out.println("pos: " + pos1); + for (int i = 0; i < 5; i++) { + System.out.println(f.readLine()); + } + System.out.println("============="); + f.seek(pos1); + for (int i = 0; i < 5; i++) { + System.out.println(f.readLine()); + } + System.out.println("============="); + f.seek(pos0); + for (int i = 0; i < 5; i++) { + System.out.println(f.readLine()); + } + long pos2 = f.getPosition(); + System.out.println("============="); + System.out.println(f.readLine()); + f.seek(pos2); + System.out.println(f.readLine()); + f.close(); + } +}; http://git-wip-us.apache.org/repos/asf/zookeeper/blob/63aaf0a1/zookeeper-contrib/zookeeper-contrib-loggraph/src/main/java/org/apache/zookeeper/graph/TransactionEntry.java ---------------------------------------------------------------------- diff --git a/zookeeper-contrib/zookeeper-contrib-loggraph/src/main/java/org/apache/zookeeper/graph/TransactionEntry.java b/zookeeper-contrib/zookeeper-contrib-loggraph/src/main/java/org/apache/zookeeper/graph/TransactionEntry.java new file mode 100644 index 0000000..33c7189 --- /dev/null +++ b/zookeeper-contrib/zookeeper-contrib-loggraph/src/main/java/org/apache/zookeeper/graph/TransactionEntry.java @@ -0,0 +1,59 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zookeeper.graph; + +public class TransactionEntry extends LogEntry { + public TransactionEntry(long timestamp, long clientId, long Cxid, long Zxid, String op) { + this(timestamp, clientId, Cxid, Zxid, op, ""); + } + + public TransactionEntry(long timestamp, long clientId, long Cxid, long Zxid, String op, String extra) { + super(timestamp); + setAttribute("client-id", new Long(clientId)); + setAttribute("cxid", new Long(Cxid)); + setAttribute("zxid", new Long(Zxid)); + setAttribute("operation", op); + setAttribute("extra", extra); + } + + public long getClientId() { + return (Long)getAttribute("client-id"); + } + + public long getCxid() { + return (Long)getAttribute("cxid"); + } + + public long getZxid() { + return (Long)getAttribute("zxid"); + } + + public String getOp() { + return (String)getAttribute("operation"); + } + + public String getExtra() { + return (String)getAttribute("extra"); + } + + public String toString() { + return getTimestamp() + ":::session(0x" + Long.toHexString(getClientId()) + ") cxid(0x" + Long.toHexString(getCxid()) + ") zxid(0x" + Long.toHexString(getZxid()) + ") op(" + getOp() + ") extra(" + getExtra() +")"; + } + + public Type getType() { return LogEntry.Type.TXN; } +} http://git-wip-us.apache.org/repos/asf/zookeeper/blob/63aaf0a1/zookeeper-contrib/zookeeper-contrib-loggraph/src/main/java/org/apache/zookeeper/graph/TxnLogSource.java ---------------------------------------------------------------------- diff --git a/zookeeper-contrib/zookeeper-contrib-loggraph/src/main/java/org/apache/zookeeper/graph/TxnLogSource.java b/zookeeper-contrib/zookeeper-contrib-loggraph/src/main/java/org/apache/zookeeper/graph/TxnLogSource.java new file mode 100644 index 0000000..809c455 --- /dev/null +++ b/zookeeper-contrib/zookeeper-contrib-loggraph/src/main/java/org/apache/zookeeper/graph/TxnLogSource.java @@ -0,0 +1,376 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zookeeper.graph; + +import java.io.ByteArrayInputStream; +import java.io.EOFException; +import java.io.FileInputStream; +import java.io.IOException; +import java.text.DateFormat; +import java.util.Date; +import java.util.zip.Adler32; +import java.util.zip.Checksum; +import java.util.HashMap; + +import org.apache.jute.BinaryInputArchive; +import org.apache.jute.InputArchive; +import org.apache.jute.Record; +import org.apache.zookeeper.server.TraceFormatter; +import org.apache.zookeeper.server.persistence.FileHeader; +import org.apache.zookeeper.server.persistence.FileTxnLog; +import org.apache.zookeeper.server.util.SerializeUtils; +import org.apache.zookeeper.txn.TxnHeader; + +import org.apache.zookeeper.ZooDefs.OpCode; + +import org.apache.zookeeper.txn.CreateSessionTxn; +import org.apache.zookeeper.txn.CreateTxn; +import org.apache.zookeeper.txn.DeleteTxn; +import org.apache.zookeeper.txn.ErrorTxn; +import org.apache.zookeeper.txn.SetACLTxn; +import org.apache.zookeeper.txn.SetDataTxn; +import org.apache.zookeeper.txn.TxnHeader; + +import java.io.File; +import java.io.Closeable; +import java.io.FileNotFoundException; +import java.util.Iterator; +import java.util.NoSuchElementException; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class TxnLogSource implements LogSource { + private static final Logger LOG = LoggerFactory.getLogger(TxnLogSource.class); + + private LogSkipList skiplist = null; + private static final int skipN = 10000; + + private String file = null; + private long starttime = 0; + private long endtime = 0; + private long size = 0; + + public boolean overlapsRange(long starttime, long endtime) { + return (starttime <= this.endtime && endtime >= this.starttime); + } + + public long size() { return size; } + public long getStartTime() { return starttime; } + public long getEndTime() { return endtime; } + public LogSkipList getSkipList() { return skiplist; } + + public static boolean isTransactionFile(String file) throws IOException { + RandomAccessFileReader reader = new RandomAccessFileReader(new File(file)); + BinaryInputArchive logStream = new BinaryInputArchive(reader); + FileHeader fhdr = new FileHeader(); + fhdr.deserialize(logStream, "fileheader"); + reader.close(); + + return fhdr.getMagic() == FileTxnLog.TXNLOG_MAGIC; + } + + private class TxnLogSourceIterator implements LogIterator { + private LogEntry next = null; + private long starttime = 0; + private long endtime = 0; + private TxnLogSource src = null; + private RandomAccessFileReader reader = null; + private BinaryInputArchive logStream = null; + private long skippedAtStart = 0; + private FilterOp filter = null; + + public TxnLogSourceIterator(TxnLogSource src, long starttime, long endtime) throws IllegalArgumentException, FilterException { + this(src,starttime,endtime,null); + } + + public TxnLogSourceIterator(TxnLogSource src, long starttime, long endtime, FilterOp filter) throws IllegalArgumentException, FilterException { + try { + this.src = src; + this.starttime = starttime; + this.endtime = endtime; + reader = new RandomAccessFileReader(new File(src.file)); + logStream = new BinaryInputArchive(reader); + FileHeader fhdr = new FileHeader(); + fhdr.deserialize(logStream, "fileheader"); + } catch (Exception e) { + throw new IllegalArgumentException("Cannot open transaction log ("+src.file+") :" + e); + } + + LogSkipList.Mark start = src.getSkipList().findMarkBefore(starttime); + try { + reader.seek(start.getBytes()); + skippedAtStart = start.getEntriesSkipped(); + } catch (IOException ioe) { + // if we can't skip, we should just read from the start + } + + this.filter = filter; + + LogEntry e; + while ((e = readNextEntry()) != null && e.getTimestamp() < endtime) { + if (e.getTimestamp() >= starttime && (filter == null || filter.matches(e)) ) { + next = e; + return; + } + skippedAtStart++; + } + + + } + + public long size() throws IOException { + if (this.endtime >= src.getEndTime()) { + return src.size() - skippedAtStart; + } + + long pos = reader.getPosition(); + LogEntry e; + + LogSkipList.Mark lastseg = src.getSkipList().findMarkBefore(this.endtime); + reader.seek(lastseg.getBytes()); + // number of entries skipped to get to the end of the iterator, less the number skipped to get to the start + long count = lastseg.getEntriesSkipped() - skippedAtStart; + + while ((e = readNextEntry()) != null) { + if (e.getTimestamp() > this.endtime) { + break; + } + count++; + } + reader.seek(pos);; + + return count; + } + + private LogEntry readNextEntry() { + LogEntry e = null; + try { + long crcValue; + byte[] bytes; + try { + crcValue = logStream.readLong("crcvalue"); + + bytes = logStream.readBuffer("txnEntry"); + } catch (EOFException ex) { + return null; + } + + if (bytes.length == 0) { + return null; + } + Checksum crc = new Adler32(); + crc.update(bytes, 0, bytes.length); + if (crcValue != crc.getValue()) { + throw new IOException("CRC doesn't match " + crcValue + + " vs " + crc.getValue()); + } + TxnHeader hdr = new TxnHeader(); + Record r = SerializeUtils.deserializeTxn(bytes, hdr); + + switch (hdr.getType()) { + case OpCode.createSession: { + e = new TransactionEntry(hdr.getTime(), hdr.getClientId(), hdr.getCxid(), hdr.getZxid(), "createSession"); + } + break; + case OpCode.closeSession: { + e = new TransactionEntry(hdr.getTime(), hdr.getClientId(), hdr.getCxid(), hdr.getZxid(), "closeSession"); + } + break; + case OpCode.create: + if (r != null) { + CreateTxn create = (CreateTxn)r; + String path = create.getPath(); + e = new TransactionEntry(hdr.getTime(), hdr.getClientId(), hdr.getCxid(), hdr.getZxid(), "create", path); + } + break; + case OpCode.setData: + if (r != null) { + SetDataTxn set = (SetDataTxn)r; + String path = set.getPath(); + e = new TransactionEntry(hdr.getTime(), hdr.getClientId(), hdr.getCxid(), hdr.getZxid(), "setData", path); + } + break; + case OpCode.setACL: + if (r != null) { + SetACLTxn setacl = (SetACLTxn)r; + String path = setacl.getPath(); + e = new TransactionEntry(hdr.getTime(), hdr.getClientId(), hdr.getCxid(), hdr.getZxid(), "setACL", path); + } + break; + case OpCode.error: + if (r != null) { + ErrorTxn error = (ErrorTxn)r; + + e = new TransactionEntry(hdr.getTime(), hdr.getClientId(), hdr.getCxid(), hdr.getZxid(), "error", "Error: " + error.getErr()); + } + break; + default: + LOG.info("Unknown op: " + hdr.getType()); + break; + } + + if (logStream.readByte("EOR") != 'B') { + throw new EOFException("Last transaction was partial."); + } + } catch (Exception ex) { + LOG.error("Error reading transaction from (" + src.file + ") :" + e); + return null; + } + return e; + } + + public boolean hasNext() { + return next != null; + } + + public LogEntry next() throws NoSuchElementException { + LogEntry ret = next; + LogEntry e = readNextEntry(); + + if (filter != null) { + try { + while (e != null && !filter.matches(e)) { + e = readNextEntry(); + } + } catch (FilterException fe) { + throw new NoSuchElementException(fe.toString()); + } + } + if (e != null && e.getTimestamp() < endtime) { + next = e; + } else { + next = null; + } + return ret; + } + + public void remove() throws UnsupportedOperationException { + throw new UnsupportedOperationException("remove not supported for Txn logs"); + } + + public void close() throws IOException { + reader.close(); + } + } + + public LogIterator iterator(long starttime, long endtime) throws IllegalArgumentException { + try { + return iterator(starttime, endtime, null); + } catch (FilterException fe) { + assert(false); // should never ever happen + return null; + } + } + + public LogIterator iterator(long starttime, long endtime, FilterOp filter) throws IllegalArgumentException, FilterException { + // sanitise start and end times + if (endtime < starttime) { + throw new IllegalArgumentException("End time (" + endtime + ") must be greater or equal to starttime (" + starttime + ")"); + } + + return new TxnLogSourceIterator(this, starttime, endtime, filter); + } + + public LogIterator iterator() throws IllegalArgumentException { + return iterator(starttime, endtime+1); + } + + public TxnLogSource(String file) throws IOException { + this.file = file; + + skiplist = new LogSkipList(); + + RandomAccessFileReader reader = new RandomAccessFileReader(new File(file)); + try { + BinaryInputArchive logStream = new BinaryInputArchive(reader); + FileHeader fhdr = new FileHeader(); + fhdr.deserialize(logStream, "fileheader"); + + byte[] bytes = null; + while (true) { + long lastFp = reader.getPosition(); + + long crcValue; + + try { + crcValue = logStream.readLong("crcvalue"); + bytes = logStream.readBuffer("txnEntry"); + } catch (EOFException e) { + break; + } + + if (bytes.length == 0) { + break; + } + Checksum crc = new Adler32(); + crc.update(bytes, 0, bytes.length); + if (crcValue != crc.getValue()) { + throw new IOException("CRC doesn't match " + crcValue + + " vs " + crc.getValue()); + } + if (logStream.readByte("EOR") != 'B') { + throw new EOFException("Last transaction was partial."); + } + TxnHeader hdr = new TxnHeader(); + Record r = SerializeUtils.deserializeTxn(bytes, hdr); + + if (starttime == 0) { + starttime = hdr.getTime(); + } + endtime = hdr.getTime(); + + if (size % skipN == 0) { + skiplist.addMark(hdr.getTime(), lastFp, size); + } + size++; + } + if (bytes == null) { + throw new IOException("Nothing read from ("+file+")"); + } + } finally { + reader.close(); + } + } + + public String toString() { + return "TxnLogSource(file=" + file + ", size=" + size + ", start=" + starttime + ", end=" + endtime +")"; + } + + public static void main(String[] args) throws IOException, FilterException { + TxnLogSource s = new TxnLogSource(args[0]); + System.out.println(s); + + LogIterator iter; + + if (args.length == 3) { + long starttime = Long.valueOf(args[1]); + long endtime = Long.valueOf(args[2]); + FilterOp fo = new FilterParser("(or (and (> zxid 0x2f0bd6f5e0) (< zxid 0x2f0bd6f5e9)) (= operation \"error\"))").parse(); + System.out.println("fo: " + fo); + iter = s.iterator(starttime, endtime, fo); + } else { + iter = s.iterator(); + } + System.out.println(iter); + while (iter.hasNext()) { + System.out.println(iter.next()); + } + iter.close(); + } +} http://git-wip-us.apache.org/repos/asf/zookeeper/blob/63aaf0a1/zookeeper-contrib/zookeeper-contrib-loggraph/src/main/java/org/apache/zookeeper/graph/filterops/AndOp.java ---------------------------------------------------------------------- diff --git a/zookeeper-contrib/zookeeper-contrib-loggraph/src/main/java/org/apache/zookeeper/graph/filterops/AndOp.java b/zookeeper-contrib/zookeeper-contrib-loggraph/src/main/java/org/apache/zookeeper/graph/filterops/AndOp.java new file mode 100644 index 0000000..581bdaa --- /dev/null +++ b/zookeeper-contrib/zookeeper-contrib-loggraph/src/main/java/org/apache/zookeeper/graph/filterops/AndOp.java @@ -0,0 +1,33 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zookeeper.graph.filterops; + +import org.apache.zookeeper.graph.LogEntry; +import org.apache.zookeeper.graph.FilterOp; +import org.apache.zookeeper.graph.FilterException; + +public class AndOp extends FilterOp { + public boolean matches(LogEntry entry) throws FilterException { + for (FilterOp f : subOps) { + if (!f.matches(entry)) { + return false; + } + } + return true; + } +} http://git-wip-us.apache.org/repos/asf/zookeeper/blob/63aaf0a1/zookeeper-contrib/zookeeper-contrib-loggraph/src/main/java/org/apache/zookeeper/graph/filterops/Arg.java ---------------------------------------------------------------------- diff --git a/zookeeper-contrib/zookeeper-contrib-loggraph/src/main/java/org/apache/zookeeper/graph/filterops/Arg.java b/zookeeper-contrib/zookeeper-contrib-loggraph/src/main/java/org/apache/zookeeper/graph/filterops/Arg.java new file mode 100644 index 0000000..4fda3cf --- /dev/null +++ b/zookeeper-contrib/zookeeper-contrib-loggraph/src/main/java/org/apache/zookeeper/graph/filterops/Arg.java @@ -0,0 +1,36 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zookeeper.graph.filterops; + +import org.apache.zookeeper.graph.FilterOp.*; + +public class Arg<T> { + private ArgType type; + protected T value; + + protected Arg(ArgType type) { + this.type = type; + } + + public ArgType getType() { return type; } + public T getValue() { return value; } + + public String toString() { + return "[" + type + ":" + value + "]"; + } +} http://git-wip-us.apache.org/repos/asf/zookeeper/blob/63aaf0a1/zookeeper-contrib/zookeeper-contrib-loggraph/src/main/java/org/apache/zookeeper/graph/filterops/EqualsOp.java ---------------------------------------------------------------------- diff --git a/zookeeper-contrib/zookeeper-contrib-loggraph/src/main/java/org/apache/zookeeper/graph/filterops/EqualsOp.java b/zookeeper-contrib/zookeeper-contrib-loggraph/src/main/java/org/apache/zookeeper/graph/filterops/EqualsOp.java new file mode 100644 index 0000000..409815a --- /dev/null +++ b/zookeeper-contrib/zookeeper-contrib-loggraph/src/main/java/org/apache/zookeeper/graph/filterops/EqualsOp.java @@ -0,0 +1,44 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zookeeper.graph.filterops; + +import org.apache.zookeeper.graph.LogEntry; +import org.apache.zookeeper.graph.FilterOp; +import org.apache.zookeeper.graph.FilterException; + +public class EqualsOp extends FilterOp { + public boolean matches(LogEntry entry) throws FilterException { + + Object last = null; + for (Arg a : args) { + Object v = a.getValue(); + if (a.getType() == FilterOp.ArgType.SYMBOL) { + String key = (String)a.getValue(); + v = entry.getAttribute(key); + } + + if (last != null + && !last.equals(v)) { + return false; + } + last = v; + } + + return true; + } +} http://git-wip-us.apache.org/repos/asf/zookeeper/blob/63aaf0a1/zookeeper-contrib/zookeeper-contrib-loggraph/src/main/java/org/apache/zookeeper/graph/filterops/GreaterThanOp.java ---------------------------------------------------------------------- diff --git a/zookeeper-contrib/zookeeper-contrib-loggraph/src/main/java/org/apache/zookeeper/graph/filterops/GreaterThanOp.java b/zookeeper-contrib/zookeeper-contrib-loggraph/src/main/java/org/apache/zookeeper/graph/filterops/GreaterThanOp.java new file mode 100644 index 0000000..244dd3d --- /dev/null +++ b/zookeeper-contrib/zookeeper-contrib-loggraph/src/main/java/org/apache/zookeeper/graph/filterops/GreaterThanOp.java @@ -0,0 +1,70 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zookeeper.graph.filterops; + +import org.apache.zookeeper.graph.LogEntry; +import org.apache.zookeeper.graph.FilterOp; +import org.apache.zookeeper.graph.FilterException; + +public class GreaterThanOp extends FilterOp { + public boolean matches(LogEntry entry) throws FilterException { + Arg first = args.get(0); + + if (first != null) { + FilterOp.ArgType type = first.getType(); + if (type == FilterOp.ArgType.SYMBOL) { + String key = (String)first.getValue(); + Object v = entry.getAttribute(key); + if (v instanceof String) { + type = FilterOp.ArgType.STRING; + } else if (v instanceof Double || v instanceof Long || v instanceof Integer || v instanceof Short) { + type = FilterOp.ArgType.NUMBER; + } else { + throw new FilterException("LessThanOp: Invalid argument, first argument resolves to neither a String nor a Number"); + } + } + + Object last = null; + for (Arg a : args) { + Object v = a.getValue(); + if (a.getType() == FilterOp.ArgType.SYMBOL) { + String key = (String)a.getValue(); + v = entry.getAttribute(key); + } + + if (last != null) { + if (type == FilterOp.ArgType.STRING) { + if (((String)last).compareTo((String)v) <= 0) { + return false; + } + } else if (type == FilterOp.ArgType.NUMBER) { + // System.out.println("last[" + ((Number)last).longValue() + "] v["+ ((Number)v).longValue() + "]"); + if (((Number)last).longValue() <= ((Number)v).longValue()) { + return false; + } + } + } + last = v; + } + return true; + } else { + return true; + } + } + +} http://git-wip-us.apache.org/repos/asf/zookeeper/blob/63aaf0a1/zookeeper-contrib/zookeeper-contrib-loggraph/src/main/java/org/apache/zookeeper/graph/filterops/LessThanOp.java ---------------------------------------------------------------------- diff --git a/zookeeper-contrib/zookeeper-contrib-loggraph/src/main/java/org/apache/zookeeper/graph/filterops/LessThanOp.java b/zookeeper-contrib/zookeeper-contrib-loggraph/src/main/java/org/apache/zookeeper/graph/filterops/LessThanOp.java new file mode 100644 index 0000000..b7d9e09 --- /dev/null +++ b/zookeeper-contrib/zookeeper-contrib-loggraph/src/main/java/org/apache/zookeeper/graph/filterops/LessThanOp.java @@ -0,0 +1,69 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zookeeper.graph.filterops; + +import org.apache.zookeeper.graph.LogEntry; +import org.apache.zookeeper.graph.FilterOp; +import org.apache.zookeeper.graph.FilterException; + +public class LessThanOp extends FilterOp { + public boolean matches(LogEntry entry) throws FilterException { + Arg first = args.get(0); + + if (first != null) { + FilterOp.ArgType type = first.getType(); + if (type == FilterOp.ArgType.SYMBOL) { + String key = (String)first.getValue(); + Object v = entry.getAttribute(key); + if (v instanceof String) { + type = FilterOp.ArgType.STRING; + } else if (v instanceof Double || v instanceof Long || v instanceof Integer || v instanceof Short) { + type = FilterOp.ArgType.NUMBER; + } else { + throw new FilterException("LessThanOp: Invalid argument, first argument resolves to neither a String nor a Number"); + } + } + + Object last = null; + for (Arg a : args) { + Object v = a.getValue(); + if (a.getType() == FilterOp.ArgType.SYMBOL) { + String key = (String)a.getValue(); + v = entry.getAttribute(key); + } + + if (last != null) { + if (type == FilterOp.ArgType.STRING) { + if (((String)last).compareTo((String)v) >= 0) { + return false; + } + } else if (type == FilterOp.ArgType.NUMBER) { + if (((Number)last).doubleValue() >= ((Number)v).doubleValue()) { + return false; + } + } + } + last = v; + } + return true; + } else { + return true; + } + } + +} http://git-wip-us.apache.org/repos/asf/zookeeper/blob/63aaf0a1/zookeeper-contrib/zookeeper-contrib-loggraph/src/main/java/org/apache/zookeeper/graph/filterops/NotOp.java ---------------------------------------------------------------------- diff --git a/zookeeper-contrib/zookeeper-contrib-loggraph/src/main/java/org/apache/zookeeper/graph/filterops/NotOp.java b/zookeeper-contrib/zookeeper-contrib-loggraph/src/main/java/org/apache/zookeeper/graph/filterops/NotOp.java new file mode 100644 index 0000000..d8ed757 --- /dev/null +++ b/zookeeper-contrib/zookeeper-contrib-loggraph/src/main/java/org/apache/zookeeper/graph/filterops/NotOp.java @@ -0,0 +1,31 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zookeeper.graph.filterops; + +import org.apache.zookeeper.graph.LogEntry; +import org.apache.zookeeper.graph.FilterOp; +import org.apache.zookeeper.graph.FilterException; + +public class NotOp extends FilterOp { + public boolean matches(LogEntry entry) throws FilterException { + if (subOps.size() != 1) { + throw new FilterException("Not operation can only take one argument"); + } + return !subOps.get(0).matches(entry); + } +} http://git-wip-us.apache.org/repos/asf/zookeeper/blob/63aaf0a1/zookeeper-contrib/zookeeper-contrib-loggraph/src/main/java/org/apache/zookeeper/graph/filterops/NumberArg.java ---------------------------------------------------------------------- diff --git a/zookeeper-contrib/zookeeper-contrib-loggraph/src/main/java/org/apache/zookeeper/graph/filterops/NumberArg.java b/zookeeper-contrib/zookeeper-contrib-loggraph/src/main/java/org/apache/zookeeper/graph/filterops/NumberArg.java new file mode 100644 index 0000000..d6b584d --- /dev/null +++ b/zookeeper-contrib/zookeeper-contrib-loggraph/src/main/java/org/apache/zookeeper/graph/filterops/NumberArg.java @@ -0,0 +1,28 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zookeeper.graph.filterops; + +import org.apache.zookeeper.graph.FilterOp.*; + +public class NumberArg extends Arg<Long> { + public NumberArg(Long value) { + super(ArgType.NUMBER); + this.value = value; + } +}; + http://git-wip-us.apache.org/repos/asf/zookeeper/blob/63aaf0a1/zookeeper-contrib/zookeeper-contrib-loggraph/src/main/java/org/apache/zookeeper/graph/filterops/OrOp.java ---------------------------------------------------------------------- diff --git a/zookeeper-contrib/zookeeper-contrib-loggraph/src/main/java/org/apache/zookeeper/graph/filterops/OrOp.java b/zookeeper-contrib/zookeeper-contrib-loggraph/src/main/java/org/apache/zookeeper/graph/filterops/OrOp.java new file mode 100644 index 0000000..d681589 --- /dev/null +++ b/zookeeper-contrib/zookeeper-contrib-loggraph/src/main/java/org/apache/zookeeper/graph/filterops/OrOp.java @@ -0,0 +1,33 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zookeeper.graph.filterops; + +import org.apache.zookeeper.graph.LogEntry; +import org.apache.zookeeper.graph.FilterOp; +import org.apache.zookeeper.graph.FilterException; + +public class OrOp extends FilterOp { + public boolean matches(LogEntry entry) throws FilterException { + for (FilterOp f : subOps) { + if (f.matches(entry)) { + return true; + } + } + return false; + } +} http://git-wip-us.apache.org/repos/asf/zookeeper/blob/63aaf0a1/zookeeper-contrib/zookeeper-contrib-loggraph/src/main/java/org/apache/zookeeper/graph/filterops/StringArg.java ---------------------------------------------------------------------- diff --git a/zookeeper-contrib/zookeeper-contrib-loggraph/src/main/java/org/apache/zookeeper/graph/filterops/StringArg.java b/zookeeper-contrib/zookeeper-contrib-loggraph/src/main/java/org/apache/zookeeper/graph/filterops/StringArg.java new file mode 100644 index 0000000..7345d3c --- /dev/null +++ b/zookeeper-contrib/zookeeper-contrib-loggraph/src/main/java/org/apache/zookeeper/graph/filterops/StringArg.java @@ -0,0 +1,28 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zookeeper.graph.filterops; + +import org.apache.zookeeper.graph.FilterOp.*; + +public class StringArg extends Arg<String> { + public StringArg(String value) { + super(ArgType.STRING); + this.value = value; + } +}; + http://git-wip-us.apache.org/repos/asf/zookeeper/blob/63aaf0a1/zookeeper-contrib/zookeeper-contrib-loggraph/src/main/java/org/apache/zookeeper/graph/filterops/SymbolArg.java ---------------------------------------------------------------------- diff --git a/zookeeper-contrib/zookeeper-contrib-loggraph/src/main/java/org/apache/zookeeper/graph/filterops/SymbolArg.java b/zookeeper-contrib/zookeeper-contrib-loggraph/src/main/java/org/apache/zookeeper/graph/filterops/SymbolArg.java new file mode 100644 index 0000000..077553b --- /dev/null +++ b/zookeeper-contrib/zookeeper-contrib-loggraph/src/main/java/org/apache/zookeeper/graph/filterops/SymbolArg.java @@ -0,0 +1,27 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zookeeper.graph.filterops; + +import org.apache.zookeeper.graph.FilterOp.*; + +public class SymbolArg extends Arg<String> { + public SymbolArg(String value) { + super(ArgType.SYMBOL); + this.value = value; + } +}; http://git-wip-us.apache.org/repos/asf/zookeeper/blob/63aaf0a1/zookeeper-contrib/zookeeper-contrib-loggraph/src/main/java/org/apache/zookeeper/graph/filterops/XorOp.java ---------------------------------------------------------------------- diff --git a/zookeeper-contrib/zookeeper-contrib-loggraph/src/main/java/org/apache/zookeeper/graph/filterops/XorOp.java b/zookeeper-contrib/zookeeper-contrib-loggraph/src/main/java/org/apache/zookeeper/graph/filterops/XorOp.java new file mode 100644 index 0000000..9e778b1 --- /dev/null +++ b/zookeeper-contrib/zookeeper-contrib-loggraph/src/main/java/org/apache/zookeeper/graph/filterops/XorOp.java @@ -0,0 +1,40 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zookeeper.graph.filterops; + +import org.apache.zookeeper.graph.LogEntry; +import org.apache.zookeeper.graph.FilterOp; +import org.apache.zookeeper.graph.FilterException; + +public class XorOp extends FilterOp { + public boolean matches(LogEntry entry) throws FilterException { + int count = 0; + for (FilterOp f : subOps) { + if (f.matches(entry)) { + count++; + if (count > 1) { + return false; + } + } + } + if (count == 1) { + return true; + } + return false; + } +} http://git-wip-us.apache.org/repos/asf/zookeeper/blob/63aaf0a1/zookeeper-contrib/zookeeper-contrib-loggraph/src/main/java/org/apache/zookeeper/graph/servlets/FileLoader.java ---------------------------------------------------------------------- diff --git a/zookeeper-contrib/zookeeper-contrib-loggraph/src/main/java/org/apache/zookeeper/graph/servlets/FileLoader.java b/zookeeper-contrib/zookeeper-contrib-loggraph/src/main/java/org/apache/zookeeper/graph/servlets/FileLoader.java new file mode 100644 index 0000000..67e8945 --- /dev/null +++ b/zookeeper-contrib/zookeeper-contrib-loggraph/src/main/java/org/apache/zookeeper/graph/servlets/FileLoader.java @@ -0,0 +1,60 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zookeeper.graph.servlets; + +import java.io.File; +import java.io.IOException; +import java.io.FileNotFoundException; + +import javax.servlet.ServletException; +import javax.servlet.http.HttpServlet; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import org.json.simple.JSONArray; +import org.json.simple.JSONObject; +import org.json.simple.JSONValue; + +import org.apache.zookeeper.graph.*; + +public class FileLoader extends JsonServlet +{ + private MergedLogSource source = null; + + public FileLoader(MergedLogSource src) throws Exception { + source = src; + } + + String handleRequest(JsonRequest request) throws Exception + { + String output = ""; + + String file = request.getString("path", "/"); + JSONObject o = new JSONObject(); + try { + this.source.addSource(file); + o.put("status", "OK"); + + } catch (Exception e) { + o.put("status", "ERR"); + o.put("error", e.toString()); + } + + return JSONValue.toJSONString(o); + } +} http://git-wip-us.apache.org/repos/asf/zookeeper/blob/63aaf0a1/zookeeper-contrib/zookeeper-contrib-loggraph/src/main/java/org/apache/zookeeper/graph/servlets/Fs.java ---------------------------------------------------------------------- diff --git a/zookeeper-contrib/zookeeper-contrib-loggraph/src/main/java/org/apache/zookeeper/graph/servlets/Fs.java b/zookeeper-contrib/zookeeper-contrib-loggraph/src/main/java/org/apache/zookeeper/graph/servlets/Fs.java new file mode 100644 index 0000000..e5b1a01 --- /dev/null +++ b/zookeeper-contrib/zookeeper-contrib-loggraph/src/main/java/org/apache/zookeeper/graph/servlets/Fs.java @@ -0,0 +1,69 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zookeeper.graph.servlets; + +import java.io.File; +import java.io.IOException; +import java.io.FileNotFoundException; + +import javax.servlet.ServletException; +import javax.servlet.http.HttpServlet; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import org.json.simple.JSONArray; +import org.json.simple.JSONObject; +import org.json.simple.JSONValue; +import java.util.Arrays; +import java.util.Comparator; + +public class Fs extends JsonServlet +{ + String handleRequest(JsonRequest request) throws Exception + { + String output = ""; + JSONArray filelist = new JSONArray(); + + File base = new File(request.getString("path", "/")); + if (!base.exists() || !base.isDirectory()) { + throw new FileNotFoundException("Couldn't find [" + request + "]"); + } + File[] files = base.listFiles(); + Arrays.sort(files, new Comparator<File>() { + public int compare(File o1, File o2) { + if (o1.isDirectory() != o2.isDirectory()) { + if (o1.isDirectory()) { + return -1; + } else { + return 1; + } + } + return o1.getName().compareToIgnoreCase(o2.getName()); + } + }); + + for (File f : files) { + JSONObject o = new JSONObject(); + o.put("file", f.getName()); + o.put("type", f.isDirectory() ? "D" : "F"); + o.put("path", f.getCanonicalPath()); + filelist.add(o); + } + return JSONValue.toJSONString(filelist); + } +}
