Repository: zookeeper Updated Branches: refs/heads/master b58791016 -> 657e2e832
ZOOKEEPER-3142: Extend SnapshotFormatter to dump data in json format Author: Brian Nixon <ni...@fb.com> Reviewers: Michael Han <h...@apache.org> Closes #619 from enixon/extend-sf Project: http://git-wip-us.apache.org/repos/asf/zookeeper/repo Commit: http://git-wip-us.apache.org/repos/asf/zookeeper/commit/657e2e83 Tree: http://git-wip-us.apache.org/repos/asf/zookeeper/tree/657e2e83 Diff: http://git-wip-us.apache.org/repos/asf/zookeeper/diff/657e2e83 Branch: refs/heads/master Commit: 657e2e832ee31738d70d54a046d4254f3ed8267a Parents: b587910 Author: Brian Nixon <ni...@fb.com> Authored: Fri Sep 14 16:09:36 2018 -0700 Committer: Michael Han <h...@apache.org> Committed: Fri Sep 14 16:09:36 2018 -0700 ---------------------------------------------------------------------- build.xml | 2 +- ivy.xml | 3 + .../zookeeper/server/SnapshotFormatter.java | 179 +++++++++++++++---- 3 files changed, 150 insertions(+), 34 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/zookeeper/blob/657e2e83/build.xml ---------------------------------------------------------------------- diff --git a/build.xml b/build.xml index 18eb104..2043e68 100644 --- a/build.xml +++ b/build.xml @@ -59,7 +59,7 @@ xmlns:cs="antlib:com.puppycrawl.tools.checkstyle.ant"> <property name="kerby.version" value="1.1.0"/> <property name="clover.version" value="4.2.1" /> - + <property name="json.version" value="1.1.1"/> <!-- ====================================================== --> <!-- Project properties --> http://git-wip-us.apache.org/repos/asf/zookeeper/blob/657e2e83/ivy.xml ---------------------------------------------------------------------- diff --git a/ivy.xml b/ivy.xml index 3387fca..663216e 100644 --- a/ivy.xml +++ b/ivy.xml @@ -63,6 +63,9 @@ <artifact name="netty" type="jar" conf="default"/> </dependency> + <dependency org="com.googlecode.json-simple" name="json-simple" rev="${json.version}" > + <exclude org="junit" module="junit"/> + </dependency> <dependency org="junit" name="junit" rev="${junit.version}" conf="test->default"/> <dependency org="org.mockito" name="mockito-all" rev="${mockito.version}" conf="test->default"/> http://git-wip-us.apache.org/repos/asf/zookeeper/blob/657e2e83/src/java/main/org/apache/zookeeper/server/SnapshotFormatter.java ---------------------------------------------------------------------- diff --git a/src/java/main/org/apache/zookeeper/server/SnapshotFormatter.java b/src/java/main/org/apache/zookeeper/server/SnapshotFormatter.java index 360ad8d..2a80d89 100644 --- a/src/java/main/org/apache/zookeeper/server/SnapshotFormatter.java +++ b/src/java/main/org/apache/zookeeper/server/SnapshotFormatter.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + * <p> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p> * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -20,8 +20,10 @@ package org.apache.zookeeper.server; import java.io.BufferedInputStream; import java.io.FileInputStream; +import java.io.File; import java.io.IOException; import java.io.InputStream; +import java.util.Base64; import java.util.Date; import java.util.HashMap; import java.util.Map; @@ -34,71 +36,130 @@ import org.apache.jute.InputArchive; import org.apache.yetus.audience.InterfaceAudience; import org.apache.zookeeper.data.StatPersisted; import org.apache.zookeeper.server.persistence.FileSnap; +import org.apache.zookeeper.server.persistence.Util; +import org.json.simple.JSONValue; + +import static org.apache.zookeeper.server.persistence.FileSnap.SNAPSHOT_FILE_PREFIX; /** * Dump a snapshot file to stdout. + * + * For JSON format, followed https://dev.yorhel.nl/ncdu/jsonfmt */ @InterfaceAudience.Public public class SnapshotFormatter { + // per-znode counter so ncdu treats each as a unique object + private static Integer INODE_IDX = 1000; + /** * USAGE: SnapshotFormatter snapshot_file */ public static void main(String[] args) throws Exception { - if (args.length != 1) { - System.err.println("USAGE: SnapshotFormatter snapshot_file"); + String snapshotFile = null; + boolean dumpData = false; + boolean dumpJson = false; + + int i; + for (i = 0; i < args.length; i++) { + if (args[i].equals("-d")) { + dumpData = true; + } else if (args[i].equals("-json")) { + dumpJson = true; + } else { + snapshotFile = args[i]; + i++; + break; + } + } + if (args.length != i || snapshotFile == null) { + System.err.println("USAGE: SnapshotFormatter [-d|-json] snapshot_file"); + System.err.println(" -d dump the data for each znode"); + System.err.println(" -json dump znode info in json format"); System.exit(ExitCode.INVALID_INVOCATION.getValue()); } - new SnapshotFormatter().run(args[0]); + if (dumpData && dumpJson) { + System.err.println("Cannot specify both data dump (-d) and json mode (-json) in same call"); + System.exit(ExitCode.INVALID_INVOCATION.getValue()); + } + + new SnapshotFormatter().run(snapshotFile, dumpData, dumpJson); } - - public void run(String snapshotFileName) throws IOException { - InputStream is = new CheckedInputStream( + + public void run(String snapshotFileName, boolean dumpData, boolean dumpJson) + throws IOException { + File snapshotFile = new File(snapshotFileName); + try (InputStream is = new CheckedInputStream( new BufferedInputStream(new FileInputStream(snapshotFileName)), - new Adler32()); - InputArchive ia = BinaryInputArchive.getArchive(is); - - FileSnap fileSnap = new FileSnap(null); + new Adler32())) { + InputArchive ia = BinaryInputArchive.getArchive(is); + + FileSnap fileSnap = new FileSnap(null); + + DataTree dataTree = new DataTree(); + Map<Long, Integer> sessions = new HashMap<Long, Integer>(); - DataTree dataTree = new DataTree(); - Map<Long, Integer> sessions = new HashMap<Long, Integer>(); - - fileSnap.deserialize(dataTree, sessions, ia); + fileSnap.deserialize(dataTree, sessions, ia); + long fileNameZxid = Util.getZxidFromName(snapshotFile.getName(), SNAPSHOT_FILE_PREFIX); - printDetails(dataTree, sessions); + if (dumpJson) { + printSnapshotJson(dataTree); + } else { + printDetails(dataTree, sessions, dumpData, fileNameZxid); + } + } } - private void printDetails(DataTree dataTree, Map<Long, Integer> sessions) { - printZnodeDetails(dataTree); + private void printDetails( + DataTree dataTree, Map<Long, Integer> sessions, + boolean dumpData, long fileNameZxid + ) { + long dtZxid = printZnodeDetails(dataTree, dumpData); printSessionDetails(dataTree, sessions); + System.out.println(String.format("----%nLast zxid: 0x%s", Long.toHexString(Math.max(fileNameZxid, dtZxid)))); } - private void printZnodeDetails(DataTree dataTree) { - System.out.println(String.format("ZNode Details (count=%d):", - dataTree.getNodeCount())); - - printZnode(dataTree, "/"); + private long printZnodeDetails(DataTree dataTree, boolean dumpData) { + System.out.println(String.format( + "ZNode Details (count=%d):", + dataTree.getNodeCount() + )); + + final long zxid = printZnode(dataTree, "/", dumpData); System.out.println("----"); + return zxid; } - private void printZnode(DataTree dataTree, String name) { + private long printZnode(DataTree dataTree, String name, boolean dumpData) { System.out.println("----"); DataNode n = dataTree.getNode(name); Set<String> children; - synchronized(n) { // keep findbugs happy + long zxid; + synchronized (n) { // keep findbugs happy System.out.println(name); printStat(n.stat); - if (n.data != null) { - System.out.println(" dataLength = " + n.data.length); + zxid = Math.max(n.stat.getMzxid(), n.stat.getPzxid()); + if (dumpData) { + System.out.println(" data = " + (n.data == null ? "" : + Base64.getEncoder().encodeToString(n.data))); } else { - System.out.println(" no data"); + System.out.println(" dataLength = " + + (n.data == null ? 0 : n.data.length)); } children = n.getChildren(); } - for (String child : children) { - printZnode(dataTree, name + (name.equals("/") ? "" : "/") + child); + if (children != null) { + for (String child : children) { + long cxid = printZnode( + dataTree, + name + (name.equals("/") ? "" : "/") + child, + dumpData + ); + zxid = Math.max(zxid, cxid); + } } + return zxid; } private void printSessionDetails(DataTree dataTree, Map<Long, Integer> sessions) { @@ -106,7 +167,8 @@ public class SnapshotFormatter { for (Map.Entry<Long, Integer> e : sessions.entrySet()) { long sid = e.getKey(); System.out.println(String.format("%#016x, %d, %d", - sid, e.getValue(), dataTree.getEphemerals(sid).size())); + sid, e.getValue(), dataTree.getEphemerals(sid).size() + )); } } @@ -125,4 +187,55 @@ public class SnapshotFormatter { private void printHex(String prefix, long value) { System.out.println(String.format(" %s = %#016x", prefix, value)); } + + private void printSnapshotJson(final DataTree dataTree) { + System.out.printf("[1,0,{\"progname\":\"SnapshotFormatter.java\",\"progver\":\"0.01\",\"timestamp\":%d}", System.currentTimeMillis()); + printZnodeJson(dataTree, "/"); + System.out.print("]"); + } + + private void printZnodeJson(final DataTree dataTree, final String fullPath) { + + final DataNode n = dataTree.getNode(fullPath); + + if (null == n) { + System.err.println("DataTree Node for " + fullPath + " doesn't exist"); + return; + } + + final String name = fullPath.equals("/") ? fullPath : fullPath.substring(fullPath.lastIndexOf( + "/") + 1); + + System.out.print(","); + + int dataLen; + synchronized (n) { // keep findbugs happy + dataLen = (n.data == null) ? 0 : n.data.length; + } + StringBuilder nodeSB = new StringBuilder(); + nodeSB.append("{"); + nodeSB.append("\"name\":\"").append(JSONValue.escape(name)).append("\"").append(","); + nodeSB.append("\"asize\":").append(dataLen).append(","); + nodeSB.append("\"dsize\":").append(dataLen).append(","); + nodeSB.append("\"dev\":").append(0).append(","); + nodeSB.append("\"ino\":").append(++INODE_IDX); + nodeSB.append("}"); + + Set<String> children; + synchronized (n) { // keep findbugs happy + children = n.getChildren(); + } + if (children != null && children.size() > 0) { + System.out.print("[" + nodeSB); + for (String child : children) { + printZnodeJson( + dataTree, + fullPath + (fullPath.equals("/") ? "" : "/") + child + ); + } + System.out.print("]"); + } else { + System.out.print(nodeSB); + } + } }
