add yaml support for extracting yaml items with comments and replacing extracts
Project: http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/commit/456049b3 Tree: http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/tree/456049b3 Diff: http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/diff/456049b3 Branch: refs/heads/master Commit: 456049b32a83030f3a55eede27e18fa16bf7cb43 Parents: 4338a8f Author: Alex Heneveld <[email protected]> Authored: Sun Mar 29 17:57:40 2015 -0500 Committer: Alex Heneveld <[email protected]> Committed: Wed Apr 15 20:05:19 2015 -0500 ---------------------------------------------------------------------- .../src/main/java/brooklyn/util/yaml/Yamls.java | 342 ++++++++++++++++++- .../test/java/brooklyn/util/yaml/YamlsTest.java | 127 +++++++ 2 files changed, 461 insertions(+), 8 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/456049b3/utils/common/src/main/java/brooklyn/util/yaml/Yamls.java ---------------------------------------------------------------------- diff --git a/utils/common/src/main/java/brooklyn/util/yaml/Yamls.java b/utils/common/src/main/java/brooklyn/util/yaml/Yamls.java index ad56bfe..2bcbe87 100644 --- a/utils/common/src/main/java/brooklyn/util/yaml/Yamls.java +++ b/utils/common/src/main/java/brooklyn/util/yaml/Yamls.java @@ -19,6 +19,7 @@ package brooklyn.util.yaml; import java.io.Reader; +import java.io.StringReader; import java.util.ArrayList; import java.util.Arrays; import java.util.Iterator; @@ -26,10 +27,23 @@ import java.util.List; import java.util.Map; import java.util.Map.Entry; +import javax.annotation.Nullable; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.yaml.snakeyaml.Yaml; +import org.yaml.snakeyaml.error.Mark; +import org.yaml.snakeyaml.nodes.MappingNode; +import org.yaml.snakeyaml.nodes.Node; +import org.yaml.snakeyaml.nodes.NodeId; +import org.yaml.snakeyaml.nodes.NodeTuple; +import org.yaml.snakeyaml.nodes.ScalarNode; +import org.yaml.snakeyaml.nodes.SequenceNode; import brooklyn.util.collections.Jsonya; +import brooklyn.util.collections.MutableList; +import brooklyn.util.exceptions.Exceptions; +import brooklyn.util.text.Strings; import com.google.common.annotations.Beta; import com.google.common.collect.Iterables; @@ -38,12 +52,13 @@ public class Yamls { private static final Logger log = LoggerFactory.getLogger(Yamls.class); - /** returns the given yaml object (map or list or primitive) as the given yaml-supperted type - * (map or list or primitive e.g. string, number, boolean). + /** returns the given (yaml-parsed) object as the given yaml type. + * <p> + * if the object is an iterable or iterator this method will fully expand it as a list. + * if the requested type is not an iterable or iterator, and the list contains a single item, this will take that single item. + * <p> + * in other cases this method simply does a type-check and cast (no other type coercion). * <p> - * if the object is an iterable containing a single element, and the type is not an iterable, - * this will attempt to unwrap it. - * * @throws IllegalArgumentException if the input is an iterable not containing a single element, * and the cast is requested to a non-iterable type * @throws ClassCastException if cannot be casted */ @@ -61,12 +76,12 @@ public class Yamls { while (xi.hasNext()) { result.add( xi.next() ); } - if (type.isAssignableFrom(Iterable.class)) return (T)result; - if (type.isAssignableFrom(Iterator.class)) return (T)result.iterator(); if (type.isAssignableFrom(List.class)) return (T)result; + if (type.isAssignableFrom(Iterator.class)) return (T)result.iterator(); x = Iterables.getOnlyElement(result); } - return (T)x; + if (type.isInstance(x)) return (T)x; + throw new ClassCastException("Cannot convert "+x+" to "+type); } /** @@ -175,4 +190,315 @@ public class Yamls { } return result; } + + @Beta + public static class YamlExtract { + String yaml; + NodeTuple focusTuple; + Node prev, key, focus, next; + Exception error; + boolean includeKey = false, includePrecedingComments = true, includeOriginalIndentation = false; + + private int indexStart(Node node, boolean defaultIsYamlEnd) { + if (node==null) return defaultIsYamlEnd ? yaml.length() : 0; + return index(node.getStartMark()); + } + private int indexEnd(Node node, boolean defaultIsYamlEnd) { + if (!found() || node==null) return defaultIsYamlEnd ? yaml.length() : 0; + return index(node.getEndMark()); + } + private int index(Mark mark) { + try { + return mark.getIndex(); + } catch (NoSuchMethodError e) { + throw new IllegalStateException("Class version error. This can happen if using a TestNG plugin in your IDE " + + "which is an older version, dragging in an older version of SnakeYAML which does not support Mark.getIndex.", e); + } + } + + public int getEndOfPrevious() { + return indexEnd(prev, false); + } + @Nullable public Node getKey() { + return key; + } + public Node getResult() { + return focus; + } + public int getStartOfThis() { + if (includeKey && focusTuple!=null) return indexStart(focusTuple.getKeyNode(), false); + return indexStart(focus, false); + } + private int getStartColumnOfThis() { + if (includeKey && focusTuple!=null) return focusTuple.getKeyNode().getStartMark().getColumn(); + return focus.getStartMark().getColumn(); + } + public int getEndOfThis() { + return getEndOfThis(false); + } + private int getEndOfThis(boolean goToEndIfNoNext) { + if (next==null && goToEndIfNoNext) return yaml.length(); + return indexEnd(focus, false); + } + public int getStartOfNext() { + return indexStart(next, true); + } + + private static int initialWhitespaceLength(String x) { + int i=0; + while (i < x.length() && x.charAt(i)==' ') i++; + return i; + } + + public String getFullYamlTextOriginal() { + return yaml; + } + + /** Returns the original YAML with the found item replaced by the given replacement YAML. + * @param replacement YAML to put in for the found item; + * this YAML typically should not have any special indentation -- if required when replacing it will be inserted. + * <p> + * if replacing an inline map entry, the supplied entry must follow the structure being replaced; + * for example, if replacing the value in <code>key: value</code> with a map, + * supplying a replacement <code>subkey: value</code> would result in invalid yaml; + * the replacement must be supplied with a newline, either before the subkey or after. + * (if unsure we believe it is always valid to include an initial newline or comment with newline.) + */ + public String getFullYamlTextWithExtractReplaced(String replacement) { + if (!found()) throw new IllegalStateException("Cannot perform replacement when item was not matched."); + String result = yaml.substring(0, getStartOfThis()); + + String[] newLines = replacement.split("\n"); + for (int i=1; i<newLines.length; i++) + newLines[i] = Strings.makePaddedString("", getStartColumnOfThis(), "", " ") + newLines[i]; + result += Strings.lines(newLines); + if (replacement.endsWith("\n")) result += "\n"; + + int end = getEndOfThis(); + result += yaml.substring(end); + + return result; + } + + /** Specifies whether the key should be included in the found text, + * when calling {@link #getMatchedYamlText()} or {@link #getFullYamlTextWithExtractReplaced(String)}, + * if the found item is a map entry. + * Defaults to false. + * @return this object, for use in a fluent constructions + */ + public YamlExtract withKeyIncluded(boolean includeKey) { + this.includeKey = includeKey; + return this; + } + + /** Specifies whether comments preceding the found item should be included, + * when calling {@link #getMatchedYamlText()} or {@link #getFullYamlTextWithExtractReplaced(String)}. + * This will not include comments which are indented further than the item, + * as those will typically be aligned with the previous item + * (whereas comments whose indentation is the same or less than the found item + * will typically be aligned with this item). + * Defaults to true. + * @return this object, for use in a fluent constructions + */ + public YamlExtract withPrecedingCommentsIncluded(boolean includePrecedingComments) { + this.includePrecedingComments = includePrecedingComments; + return this; + } + + /** Specifies whether the original indentation should be preserved + * (and in the case of the first line, whether whitespace should be inserted so its start column is preserved), + * when calling {@link #getMatchedYamlText()}. + * Defaults to false, the returned text will be outdented as far as possible. + * @return this object, for use in a fluent constructions + */ + public YamlExtract withOriginalIndentation(boolean includeOriginalIndentation) { + this.includeOriginalIndentation = includeOriginalIndentation; + return this; + } + + @Beta + public String getMatchedYamlText() { + if (!found()) return null; + + String[] body = yaml.substring(getStartOfThis(), getEndOfThis(true)).split("\n", -1); + + int firstLineIndentationOfFirstThing; + if (focusTuple!=null) { + firstLineIndentationOfFirstThing = focusTuple.getKeyNode().getStartMark().getColumn(); + } else { + firstLineIndentationOfFirstThing = focus.getStartMark().getColumn(); + } + int firstLineIndentationToAdd; + if (focusTuple!=null && (includeKey || body.length==1)) { + firstLineIndentationToAdd = focusTuple.getKeyNode().getStartMark().getColumn(); + } else { + firstLineIndentationToAdd = focus.getStartMark().getColumn(); + } + + + String firstLineIndentationToAddS = Strings.makePaddedString("", firstLineIndentationToAdd, "", " "); + String subsequentLineIndentationToRemoveS = firstLineIndentationToAddS; + +/* complexities of indentation: + +x: a + bc + +should become + +a + bc + +whereas + +- a: 0 + b: 1 + +selecting 0 should give + +a: 0 +b: 1 + + */ + List<String> result = MutableList.of(); + if (includePrecedingComments) { + String[] preceding = yaml.substring(getEndOfPrevious(), getStartOfThis()).split("\n"); + // suppress comments which are on the same line as the previous item or indented more than firstLineIndentation, + // ensuring appropriate whitespace is added to preceding[0] if it starts mid-line + if (preceding.length>0 && prev!=null) { + preceding[0] = Strings.makePaddedString("", prev.getEndMark().getColumn(), "", " ") + preceding[0]; + } + for (String p: preceding) { + int w = initialWhitespaceLength(p); + p = p.substring(w); + if (p.startsWith("#")) { + // only add if the hash is not indented further than the first line + if (w <= firstLineIndentationOfFirstThing) { + if (includeOriginalIndentation) p = firstLineIndentationToAddS + p; + result.add(p); + } + } + } + } + + boolean doneFirst = false; + for (String p: body) { + if (!doneFirst) { + if (includeOriginalIndentation) { + // have to insert the right amount of spacing + p = firstLineIndentationToAddS + p; + } + result.add(p); + doneFirst = true; + } else { + if (includeOriginalIndentation) { + result.add(p); + } else { + result.add(Strings.removeFromStart(p, subsequentLineIndentationToRemoveS)); + } + } + } + return Strings.join(result, "\n"); + } + + boolean found() { + return focus != null; + } + + public Exception getError() { + return error; + } + + @Override + public String toString() { + return "Extract["+focus+";prev="+prev+";key="+key+";next="+next+"]"; + } + } + + private static void findTextOfYamlAtPath(YamlExtract result, int pathIndex, Object ...path) { + if (pathIndex>=path.length) { + // we're done + return; + } + + Object pathItem = path[pathIndex]; + Node node = result.focus; + + if (node.getNodeId()==NodeId.mapping && pathItem instanceof String) { + // find key + Iterator<NodeTuple> ti = ((MappingNode)node).getValue().iterator(); + while (ti.hasNext()) { + NodeTuple t = ti.next(); + Node key = t.getKeyNode(); + if (key.getNodeId()==NodeId.scalar) { + if (pathItem.equals( ((ScalarNode)key).getValue() )) { + result.key = key; + result.focus = t.getValueNode(); + if (pathIndex+1<path.length) { + // there are more path items, so the key here is a previous node + result.prev = key; + } else { + result.focusTuple = t; + } + findTextOfYamlAtPath(result, pathIndex+1, path); + if (result.next==null) { + if (ti.hasNext()) result.next = ti.next().getKeyNode(); + } + return; + } else { + result.prev = t.getValueNode(); + } + } else { + throw new IllegalStateException("Key "+key+" is not a scalar, searching for "+pathItem+" at depth "+pathIndex+" of "+Arrays.asList(path)); + } + } + throw new IllegalStateException("Did not find "+pathItem+" in "+node+" at depth "+pathIndex+" of "+Arrays.asList(path)); + + } else if (node.getNodeId()==NodeId.sequence && pathItem instanceof Number) { + // find list item + List<Node> nl = ((SequenceNode)node).getValue(); + int i = ((Number)pathItem).intValue(); + if (i>=nl.size()) + throw new IllegalStateException("Index "+i+" is out of bounds in "+node+", searching for "+pathItem+" at depth "+pathIndex+" of "+Arrays.asList(path)); + if (i>0) result.prev = nl.get(i-1); + result.key = null; + result.focus = nl.get(i); + findTextOfYamlAtPath(result, pathIndex+1, path); + if (result.next==null) { + if (nl.size()>i+1) result.next = nl.get(i+1); + } + return; + + } else { + throw new IllegalStateException("Node "+node+" does not match selector "+pathItem+" at depth "+pathIndex+" of "+Arrays.asList(path)); + } + + // unreachable + } + + + /** Given a path, where each segment consists of a string (key) or number (element in list), + * this will find the YAML text for that element + * <p> + * If not found this will return a {@link YamlExtract} + * where {@link YamlExtract#isMatch()} is false and {@link YamlExtract#getError()} is set. */ + public static YamlExtract getTextOfYamlAtPath(String yaml, Object ...path) { + YamlExtract result = new YamlExtract(); + try { + int pathIndex = 0; + result.yaml = yaml; + result.focus = new Yaml().compose(new StringReader(yaml)); + + findTextOfYamlAtPath(result, pathIndex, path); + return result; + } catch (NoSuchMethodError e) { + throw new IllegalStateException("Class version error. This can happen if using a TestNG plugin in your IDE " + + "which is an older version, dragging in an older version of SnakeYAML which does not support Mark.getIndex.", e); + } catch (Exception e) { + Exceptions.propagateIfFatal(e); + log.debug("Unable to find element in yaml (setting in result): "+e); + result.error = e; + return result; + } + } } http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/456049b3/utils/common/src/test/java/brooklyn/util/yaml/YamlsTest.java ---------------------------------------------------------------------- diff --git a/utils/common/src/test/java/brooklyn/util/yaml/YamlsTest.java b/utils/common/src/test/java/brooklyn/util/yaml/YamlsTest.java index 27437d9..36c146b 100644 --- a/utils/common/src/test/java/brooklyn/util/yaml/YamlsTest.java +++ b/utils/common/src/test/java/brooklyn/util/yaml/YamlsTest.java @@ -20,14 +20,30 @@ package brooklyn.util.yaml; import static org.testng.Assert.assertEquals; +import java.util.Iterator; +import java.util.List; + +import org.testng.Assert; +import org.testng.TestNG; import org.testng.annotations.Test; +import brooklyn.util.collections.MutableList; + import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; public class YamlsTest { @Test + public void testGetAs() throws Exception { + MutableList<String> list = MutableList.of("x"); + assertEquals(Yamls.getAs(list.iterator(), List.class), list); + assertEquals(Yamls.getAs(list.iterator(), Iterable.class), list); + assertEquals(Yamls.getAs(list.iterator(), Iterator.class), list.iterator()); + assertEquals(Yamls.getAs(list.iterator(), String.class), "x"); + } + + @Test public void testGetAt() throws Exception { // leaf of map assertEquals(Yamls.getAt("k1: v", ImmutableList.of("k1")), "v"); @@ -44,4 +60,115 @@ public class YamlsTest { assertEquals(Yamls.getAt("k1: [v1, v2]", ImmutableList.<String>of("k1", "[0]")), "v1"); assertEquals(Yamls.getAt("k1: [v1, v2]", ImmutableList.<String>of("k1", "[1]")), "v2"); } + + + @Test + public void testExtractMap() { + String sample = "#before\nk1:\n- v1\nk2:\n # comment\n k21: v21\nk3: v3\n#after\n"; + + Assert.assertEquals(Yamls.getTextOfYamlAtPath(sample, "k1").withKeyIncluded(true).getMatchedYamlText(), + sample.substring(0, sample.indexOf("k2"))); + Assert.assertEquals(Yamls.getTextOfYamlAtPath(sample, "k3").withKeyIncluded(true).getMatchedYamlText(), + sample.substring(sample.indexOf("k3"))); + + // comments and no key, outdented - the default + Assert.assertEquals(Yamls.getTextOfYamlAtPath(sample, "k2", "k21").getMatchedYamlText(), + "# comment\nv21"); + Assert.assertEquals(Yamls.getTextOfYamlAtPath(sample, "k2", "k21").getMatchedYamlText(), + "# comment\nv21"); + // comments and key + Assert.assertEquals(Yamls.getTextOfYamlAtPath(sample, "k2", "k21").withKeyIncluded(true).getMatchedYamlText(), + "# comment\nk21: v21"); + // no comments + Assert.assertEquals(Yamls.getTextOfYamlAtPath(sample, "k2", "k21").withKeyIncluded(true).withPrecedingCommentsIncluded(false).getMatchedYamlText(), + "k21: v21"); + // no comments and no key + Assert.assertEquals(Yamls.getTextOfYamlAtPath(sample, "k2", "k21").withPrecedingCommentsIncluded(false).getMatchedYamlText(), + "v21"); + + // comments and no key, not outdented + Assert.assertEquals(Yamls.getTextOfYamlAtPath(sample, "k2", "k21").withOriginalIndentation(true).getMatchedYamlText(), + " # comment\n v21"); + // comments and key + Assert.assertEquals(Yamls.getTextOfYamlAtPath(sample, "k2", "k21").withKeyIncluded(true).withOriginalIndentation(true).getMatchedYamlText(), + " # comment\n k21: v21"); + // no comments + Assert.assertEquals(Yamls.getTextOfYamlAtPath(sample, "k2", "k21").withKeyIncluded(true).withPrecedingCommentsIncluded(false).withOriginalIndentation(true).getMatchedYamlText(), + " k21: v21"); + // no comments and no key + Assert.assertEquals(Yamls.getTextOfYamlAtPath(sample, "k2", "k21").withPrecedingCommentsIncluded(false).withOriginalIndentation(true).getMatchedYamlText(), + " v21"); + } + + @Test + public void testExtractInList() { + String sample = + "- a\n" + + "- b: 2\n" + + "- # c\n" + + " c1:\n" + + " 1\n" + + " c2:\n" + + " 2\n" + + "-\n" + + " - a # for a\n" + + " # for b\n" + + " - b\n"; + + Assert.assertEquals(Yamls.getTextOfYamlAtPath(sample, 0).getMatchedYamlText(), "a"); + Assert.assertEquals(Yamls.getTextOfYamlAtPath(sample, 1, "b").getMatchedYamlText(), "2"); + Assert.assertEquals(Yamls.getTextOfYamlAtPath(sample, 3, 0).getMatchedYamlText(), + "a" + // TODO comments after on same line not yet included - would be nice to add +// "a # for a" + ); + + // out-dent + Assert.assertEquals(Yamls.getTextOfYamlAtPath(sample, 2).getMatchedYamlText(), "c1:\n 1\nc2:\n 2\n"); + // don't outdent + Assert.assertEquals(Yamls.getTextOfYamlAtPath(sample, 2).withOriginalIndentation(true).getMatchedYamlText(), " c1:\n 1\n c2:\n 2\n"); + Assert.assertEquals(Yamls.getTextOfYamlAtPath(sample, 3, 0).withOriginalIndentation(true).getMatchedYamlText(), + " a" + // as above, comments after not included +// " a # for a" + ); + + // with preceding comments + // TODO final item includes newline (and comments) after - this behaviour might change, it's inconsistent, + // but it means the final comments aren't lost + Assert.assertEquals(Yamls.getTextOfYamlAtPath(sample, 3, 1).getMatchedYamlText(), "# for b\nb\n"); + + // exclude preceding comments + Assert.assertEquals(Yamls.getTextOfYamlAtPath(sample, 3, 1).withPrecedingCommentsIncluded(false).getMatchedYamlText(), "b\n"); + } + + @Test + public void testExtractMapIgnoringPreviousComments() { + String sample = "a: 1 # one\n" + + "b: 2 # two"; + Assert.assertEquals(Yamls.getTextOfYamlAtPath(sample, "b").getMatchedYamlText(), + "2 # two"); + } + + @Test + public void testExtractMapWithOddWhitespace() { + Assert.assertEquals(Yamls.getTextOfYamlAtPath("x: a\n bc", "x").getMatchedYamlText(), + "a\n bc"); + } + + @Test + public void testReplace() { + Assert.assertEquals(Yamls.getTextOfYamlAtPath("x: a\n bc", "x").getFullYamlTextWithExtractReplaced("\nc: 1\nd: 2"), + "x: \n c: 1\n d: 2"); + } + + // convenience, since running with older TestNG IDE plugin will fail (older snakeyaml dependency); + // if you run as a java app it doesn't bring in the IDE TestNG jar version, and it works + public static void main(String[] args) { + TestNG testng = new TestNG(); + testng.setTestClasses(new Class[] { YamlsTest.class }); +// testng.setVerbose(9); + testng.run(); + } + }
