Continued FM2 to FM3 converter...
Project: http://git-wip-us.apache.org/repos/asf/incubator-freemarker/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-freemarker/commit/92db5891 Tree: http://git-wip-us.apache.org/repos/asf/incubator-freemarker/tree/92db5891 Diff: http://git-wip-us.apache.org/repos/asf/incubator-freemarker/diff/92db5891 Branch: refs/heads/3 Commit: 92db58918b4eafaa184cb4fab87131019b419454 Parents: af3053f Author: ddekany <[email protected]> Authored: Fri Jul 7 02:15:22 2017 +0200 Committer: ddekany <[email protected]> Committed: Fri Jul 7 02:15:22 2017 +0200 ---------------------------------------------------------------------- .../core/FM2ASTToFM3SourceConverter.java | 306 +++++++++++++++---- .../freemarker/converter/ConversionMarkers.java | 49 ++- .../apache/freemarker/converter/Converter.java | 98 ++++-- .../freemarker/converter/FM2ToFM3Converter.java | 2 +- .../converter/FM2ToFM3ConverterTest.java | 40 +++ .../converter/GenericConverterTest.java | 51 +++- .../converter/test/ConverterTest.java | 23 ++ 7 files changed, 471 insertions(+), 98 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-freemarker/blob/92db5891/freemarker-converter/src/main/java/freemarker/core/FM2ASTToFM3SourceConverter.java ---------------------------------------------------------------------- diff --git a/freemarker-converter/src/main/java/freemarker/core/FM2ASTToFM3SourceConverter.java b/freemarker-converter/src/main/java/freemarker/core/FM2ASTToFM3SourceConverter.java index 4c2d19c..1d6f484 100644 --- a/freemarker-converter/src/main/java/freemarker/core/FM2ASTToFM3SourceConverter.java +++ b/freemarker-converter/src/main/java/freemarker/core/FM2ASTToFM3SourceConverter.java @@ -20,6 +20,7 @@ package freemarker.core; import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.Comparator; import java.util.HashMap; @@ -27,11 +28,17 @@ import java.util.List; import java.util.Map; import java.util.Set; -import org.apache.freemarker.converter.ConversionWarnReceiver; +import org.apache.freemarker.converter.ConversionMarkers; +import org.apache.freemarker.converter.ConversionMarkers.Type; import org.apache.freemarker.converter.ConverterException; import org.apache.freemarker.converter.ConverterUtils; import org.apache.freemarker.core.NamingConvention; -import org.apache.freemarker.core.util.*; +import org.apache.freemarker.core.util.FTLUtil; +import org.apache.freemarker.core.util._ClassUtil; +import org.apache.freemarker.core.util._NullArgumentException; +import org.apache.freemarker.core.util._StringUtil; + +import com.google.common.collect.ImmutableList; import freemarker.template.Configuration; import freemarker.template.Template; @@ -69,7 +76,7 @@ public class FM2ASTToFM3SourceConverter { private final Template template; private final String src; - private final ConversionWarnReceiver warnReceiver; + private final ConversionMarkers markers; private final StringBuilder out; private List<Integer> rowStartPositions; @@ -89,7 +96,7 @@ public class FM2ASTToFM3SourceConverter { * {@code false}. */ public static Result convert( - String templateName, String src, Configuration fm2Cfg, ConversionWarnReceiver warnReceiver) + String templateName, String src, Configuration fm2Cfg, ConversionMarkers warnReceiver) throws ConverterException { return new FM2ASTToFM3SourceConverter(templateName, src, fm2Cfg, warnReceiver).convert(); } @@ -110,7 +117,7 @@ public class FM2ASTToFM3SourceConverter { } private FM2ASTToFM3SourceConverter( - String templateName, String src, Configuration fm2Cfg, ConversionWarnReceiver warnReceiver) + String templateName, String src, Configuration fm2Cfg, ConversionMarkers warnReceiver) throws ConverterException { template = createTemplate(templateName, src, fm2Cfg); if (template.getParserConfiguration().getWhitespaceStripping()) { @@ -121,7 +128,7 @@ public class FM2ASTToFM3SourceConverter { this.src = src; - this.warnReceiver = warnReceiver; + this.markers = warnReceiver; this.out = new StringBuilder(); if (template.getActualTagSyntax() == Configuration.SQUARE_BRACKET_TAG_SYNTAX) { @@ -194,7 +201,7 @@ public class FM2ASTToFM3SourceConverter { FM2ASTToFM3SourceConverter customFtlDirSrcConverter = new FM2ASTToFM3SourceConverter( template.getName(), tagBeginChar + "@ftl" + src.substring(pos, tagEnd) + (hasSlash ? "" : "/") + tagEndChar, - template.getConfiguration(), warnReceiver + template.getConfiguration(), markers ); customFtlDirSrcConverter.printNextCustomDirAsFtlDir = true; String fm3Content = customFtlDirSrcConverter.convert().fm3Content; @@ -227,7 +234,7 @@ public class FM2ASTToFM3SourceConverter { private void printTemplateElement(TemplateElement node) throws ConverterException { if (node instanceof MixedContent) { - printChildrenElements(node); + printChildElements(node); } else if (node instanceof TextBlock) { print(getOnlyParam(node, ParameterRole.CONTENT, String.class)); } else if (node instanceof DollarVariable) { @@ -339,11 +346,153 @@ public class FM2ASTToFM3SourceConverter { printDirImport((LibraryLoad) node); } else if (node instanceof Include) { printDirInclude((Include) node); + } else if (node instanceof IteratorBlock) { + printDirListOrForeach((IteratorBlock) node, true); + } else if (node instanceof ListElseContainer) { + printDirListElseContainer((ListElseContainer) node); + } else if (node instanceof Sep) { + printDirSep((Sep) node); + } else if (node instanceof Items) { + printDirItems((Items) node); + } else if (node instanceof BreakInstruction) { + printDirBreak((BreakInstruction) node); } else { throw new ConverterException("Unhandled AST TemplateElement class: " + node.getClass().getName()); } } + private void printDirBreak(BreakInstruction node) throws ConverterException { + printCoreDirStartTagParameterless(node, "break"); + } + + private void printDirItems(Items node) throws ConverterException { + int pos = printCoreDirStartTagBeforeParams(node, "items"); + pos = printSeparatorAndWSAndExpComments(pos, "as"); + + int paramCnt = node.getParameterCount(); + assertNodeContent(paramCnt <= 2, node, "Expected at most 2 parameters"); + String loopVar1 = getParam(node, 0, ParameterRole.TARGET_LOOP_VARIABLE, String.class); + String loopVar2 = paramCnt >= 2 ? getParam(node, 1, ParameterRole.TARGET_LOOP_VARIABLE, String.class) : null; + + print(FTLUtil.escapeIdentifier(loopVar1)); + pos = getPositionAfterIdentifier(pos); + if (loopVar2 != null) { + pos = printSeparatorAndWSAndExpComments(pos, ","); + print(FTLUtil.escapeIdentifier(loopVar2)); + pos = getPositionAfterIdentifier(pos); + } + + printStartTagEnd(node, pos, false); + + printChildElements(node); + + printCoreDirEndTag(node, "items"); + } + + private void printDirListElseContainer(ListElseContainer node) throws ConverterException { + assertNodeContent(node.getChildCount() == 2, node, "Expected 2 child elements."); + + printDirListOrForeach((IteratorBlock) node.getChild(0), false); + printDirElseOfList((ElseOfList) node.getChild(1)); + printCoreDirEndTag(node, "list"); + } + + private void printDirElseOfList(ElseOfList node) throws ConverterException { + printCoreDirStartTagParameterless(node, "else"); + printChildElements(node); + } + + private void printDirSep(Sep node) throws ConverterException { + printCoreDirStartTagParameterless(node, "sep"); + printChildElements(node); + printCoreDirEndTag(node, Collections.singleton("sep"), "sep", true); + } + + private void printDirListOrForeach(IteratorBlock node, boolean printEndTag) throws ConverterException { + int paramCount = node.getParameterCount(); + assertNodeContent(paramCount <= 3, node, "ParameterCount <= 3 was expected"); + + int pos = printCoreDirStartTagBeforeParams(node, "list"); + + Expression listSource = getParam(node, 0, ParameterRole.LIST_SOURCE, Expression.class); + // To be future proof, we don't assume that the parameter count of list don't include the null parameters. + String loopVal1 = paramCount >= 2 ? getParam(node, 1, ParameterRole.TARGET_LOOP_VARIABLE, String.class) + : null; + String loopVal2 = paramCount >= 3 ? getParam(node, 2, ParameterRole.TARGET_LOOP_VARIABLE, String.class) + : null; + + String fm2TagName1; + String fm2TagName2; + if (node.getNodeTypeSymbol().equals("#list")) { + fm2TagName1 = "list"; + fm2TagName2 = null; + + printExp(listSource); + + if (loopVal1 != null) { // #list xs as <v1 | v1, v2> + pos = printSeparatorAndWSAndExpComments(getEndPositionExclusive(listSource), "as"); + + print(FTLUtil.escapeIdentifier(loopVal1)); + pos = getPositionAfterAssignmentTargetIdentifier(pos); + + if (loopVal2 != null) { // #list xs as <v1, v2> + pos = printSeparatorAndWSAndExpComments(pos, ","); + + print(FTLUtil.escapeIdentifier(loopVal2)); + pos = getPositionAfterAssignmentTargetIdentifier(pos); + } + + printWSAndExpComments(pos); + } + } else if (node.getNodeTypeSymbol().equals("#foreach")) { + fm2TagName1 = "foreach"; + fm2TagName2 = "forEach"; + + assertNodeContent(loopVal1 != null && loopVal2 == null, + node, "Unsupported #foreach parameter "); + + // We rewrite the #foreach to #list. We assume that comments after around the "in" belong to the loop + // variable, and comments after the list source belong to the list source. + + // #foreach <x> in xs: + pos = getPositionAfterIdentifier(pos); + + // #foreach x< >in xs: + int prevPos = pos; + pos = getPositionAfterWSAndExpComments(pos); + String postVar1WSAndComment = src.substring(prevPos, pos); + + // #foreach x <in> xs: + assertNodeContent(src.startsWith("in", pos), node, + "Keyword \"in\" expected at position {}.", pos); + pos += 2; // skip `in` + + // #foreach x in< >xs: + prevPos = pos; + pos = getPositionAfterWSAndExpComments(pos); + String postInWSAndComment = src.substring(prevPos, pos); + + // #foreach x in xs< >: + String postVar2WSAndComment = readWSAndExpComments(getEndPositionExclusive(listSource)); + + printExp(listSource); + printWithConvertedExpComments(rightTrim(postVar2WSAndComment)); + print(" as "); + print(FTLUtil.escapeIdentifier(loopVal1)); + printWithConvertedExpComments(rightTrim(postVar1WSAndComment)); + printWithConvertedExpComments(rightTrim(postInWSAndComment)); + } else { + throw new UnexpectedNodeContentException(node, "Expected #list or #foreach as node symbol", null); + } + print(tagEndChar); + + printChildElements(node); + + if (printEndTag) { + printCoreDirEndTag(node, ImmutableList.of("list", "foreach", "forEach"), "list", false); + } + } + private void printDirInclude(Include node) throws ConverterException { if (Configuration.getVersion().intValue() != Configuration.VERSION_2_3_26.intValue()) { throw new BugException("Fix things at [broken in 2.3.26] comments; version was: " @@ -358,7 +507,7 @@ public class FM2ASTToFM3SourceConverter { Expression parseParam = getParam(node, 1, ParameterRole.PARSE_PARAMETER, Expression.class); if (parseParam != null) { - warnReceiver.warn(parseParam.getBeginLine(), parseParam.getBeginColumn(), + markers.markInSource(parseParam.getBeginLine(), parseParam.getBeginColumn(), Type.WARN, "The \"parse\" parameter of #include was removed, as it's not supported anymore. Use the " + "templateConfigurations configuration setting to specify which files are not parsed."); @@ -366,7 +515,7 @@ public class FM2ASTToFM3SourceConverter { Expression encodingParam = getParam(node, 2, ParameterRole.ENCODING_PARAMETER, Expression.class); if (encodingParam != null) { - warnReceiver.warn(encodingParam.getBeginLine(), encodingParam.getBeginColumn(), + markers.markInSource(encodingParam.getBeginLine(), encodingParam.getBeginColumn(), Type.WARN, "The \"encoding\" parameter of #include was removed, as it's not supported anymore. Use the " + "templateConfigurations configuration setting to specify which files has a different " + "encoding than the configured default."); @@ -380,15 +529,15 @@ public class FM2ASTToFM3SourceConverter { sortExpressionsByPosition(templateName, parseParam, encodingParam, ignoreMissingParam); printExp(templateName); - String postNameWSOrComment = readAndWSAndExpComments(templateNameEndPos); + String postNameWSOrComment = readWSAndExpComments(templateNameEndPos); if (ignoreMissingParam != null || (parseParam == null && encodingParam == null)) { // This will separate us from ignoreMissing=exp, or from the tagEndChar - print(postNameWSOrComment); + printWithConvertedExpComments(postNameWSOrComment); } else { // We only have removed thing after in the src => no need for spacing after us int commentPos = postNameWSOrComment.indexOf("--") - 1; if (commentPos >= 0) { - print(rightTrim(postNameWSOrComment)); + printWithConvertedExpComments(rightTrim(postNameWSOrComment)); } } @@ -401,14 +550,14 @@ public class FM2ASTToFM3SourceConverter { printSeparatorAndWSAndExpComments(getPositionAfterIdentifier(identifierStartPos), "="); printExp(paramExp); - String postParamWSOrComment = readAndWSAndExpComments(getEndPositionExclusive(paramExp)); + String postParamWSOrComment = readWSAndExpComments(getEndPositionExclusive(paramExp)); if (i == sortedExps.size() - 1) { // We were the last int the source as well - print(postParamWSOrComment); + printWithConvertedExpComments(postParamWSOrComment); } else { int commentPos = postParamWSOrComment.indexOf("--") - 1; if (commentPos >= 0) { - print(rightTrim(postParamWSOrComment)); + printWithConvertedExpComments(rightTrim(postParamWSOrComment)); } } } @@ -479,7 +628,7 @@ public class FM2ASTToFM3SourceConverter { printExp(expTemplate); printStartTagEnd(node, expTemplate, false); - printChildrenElements(node); + printChildElements(node); printCoreDirEndTag(node, "escape"); } @@ -500,31 +649,32 @@ public class FM2ASTToFM3SourceConverter { throws ConverterException { assertParamCount(node, 0); - printCoreDirParameterlessStartTag(node, tagName); - printChildrenElements(node); + printCoreDirStartTagParameterless(node, tagName); + printChildElements(node); printCoreDirEndTag(node, tagName); } private void printDirGenericParameterlessWithoutNestedContent(TemplateElement node, String name) throws ConverterException { assertParamCount(node, 0); - printCoreDirParameterlessStartTag(node, name); + printCoreDirStartTagParameterless(node, name); } private void printDirAttemptRecover(AttemptBlock node) throws ConverterException { assertParamCount(node, 1); // 1: The recovery block - printCoreDirParameterlessStartTag(node, "attempt"); + printCoreDirStartTagParameterless(node, "attempt"); printNode(node.getChild(0)); assertNodeContent(node.getChild(1) instanceof RecoveryBlock, node, "child[1] should be #recover"); RecoveryBlock recoverDir = getOnlyParam(node, ParameterRole.ERROR_HANDLER, RecoveryBlock.class); - printCoreDirParameterlessStartTag(recoverDir, "recover"); + printCoreDirStartTagParameterless(recoverDir, "recover"); - printChildrenElements(recoverDir); + printChildElements(recoverDir); - printCoreDirEndTag(node, "attempt"); // in FM2 this could be /#recover, but we normalize it + // In FM2 this could be </#recover> as well, but we normalize it + printCoreDirEndTag(node, ImmutableList.of("attempt", "recover"), "attempt", false); } private void printDirAssignmentMultiple(AssignmentInstruction node) throws ConverterException { @@ -685,7 +835,7 @@ public class FM2ASTToFM3SourceConverter { assertNodeContent(src.charAt(pos) == tagEndChar, node, "Tag end not found"); print(tagEndChar); - printChildrenElements(node); + printChildElements(node); printCoreDirEndTag(node, tagName); } @@ -763,7 +913,7 @@ public class FM2ASTToFM3SourceConverter { if (startTagEndPos != elementEndPos) { // We have an end-tag assertNodeContent(src.charAt(startTagEndPos - 1) != '/', node, "Not expected \"/\" at the end of the start tag"); - printChildrenElements(node); + printChildElements(node); print(tagBeginChar); print("/@"); @@ -805,10 +955,10 @@ public class FM2ASTToFM3SourceConverter { printNode(conditionExp); printStartTagEnd(node, conditionExp, true); } else { - printCoreDirParameterlessStartTag(node, tagName); + printCoreDirStartTagParameterless(node, tagName); } - printChildrenElements(node); + printChildElements(node); if (!(node.getParentElement() instanceof IfBlock)) { printCoreDirEndTag(node, "if"); @@ -816,7 +966,7 @@ public class FM2ASTToFM3SourceConverter { } private void printDirIfElseElseIfContainer(IfBlock node) throws ConverterException { - printChildrenElements(node); + printChildElements(node); printCoreDirEndTag(node, "if"); } @@ -1280,13 +1430,19 @@ public class FM2ASTToFM3SourceConverter { rho.getBeginColumn(), rho.getBeginLine())); } - private void printChildrenElements(TemplateElement node) throws ConverterException { + private void printChildElements(TemplateElement node) throws ConverterException { int ln = node.getChildCount(); for (int i = 0; i < ln; i++) { printNode(node.getChild(i)); } } + /** + * Prints the start tag until the parameters come; this works even if there are no parameters, in whic case it + * prints until the tag end character. + * + * @return The position in the source after the printed part + */ private int printCoreDirStartTagBeforeParams(TemplateElement node, String fm3TagName) throws ConverterException { print(tagBeginChar); @@ -1295,18 +1451,62 @@ public class FM2ASTToFM3SourceConverter { return printWSAndExpComments(getPositionAfterTagName(node)); } - private int printCoreDirParameterlessStartTag(TemplateElement node, String tagName) + private int printCoreDirStartTagParameterless(TemplateElement node, String fm3TagName) throws ConverterException { - int pos = printCoreDirStartTagBeforeParams(node, tagName); - print(tagEndChar); + int pos = printCoreDirStartTagBeforeParams(node, fm3TagName); + printStartTagEnd(node, pos, true); return pos + 1; } - private void printCoreDirEndTag(TemplateElement node, String fm3TagName) throws UnexpectedNodeContentException { + private void printCoreDirEndTag(TemplateElement node, String tagName) throws UnexpectedNodeContentException { + printCoreDirEndTag(node, Collections.singleton(tagName), tagName, false); + } + + private void printCoreDirEndTag(TemplateElement node, Collection<String> fm2TagNames, String fm3TagName, + boolean optional) + throws UnexpectedNodeContentException { + int tagEndPos = getEndPositionInclusive(node); + { + char c = src.charAt(tagEndPos); + if (c != tagEndChar) { + if (optional) { + return; + } + throw new UnexpectedNodeContentException(node, "tagEndChar expected, found {}", c); + } + } + + int pos = tagEndPos - 1; + while (pos > 0 && Character.isWhitespace(src.charAt(pos))) { + pos--; + } + if (pos < 0 || !isCoreNameChar(src.charAt(pos))) { + if (optional) { + return; + } + throw new UnexpectedNodeContentException(node, "Can't find end tag name", null); + } + int nameEndPos = pos + 1; + + while (pos > 0 && src.charAt(pos) != '#') { + pos--; + } + String srcTagName = src.substring(pos + 1 /* skip '#' */, nameEndPos); + + if (!fm2TagNames.contains(srcTagName)) { + if (optional) { + return; + } + throw new UnexpectedNodeContentException(node, "Unexpected end tag name: {}", srcTagName); + } + print(tagBeginChar); print("/#"); + print(fm3TagName); - printEndTagSkippedTokens(node); + + printWithConvertedExpComments(src.substring(nameEndPos, tagEndPos)); + print(tagEndChar); } @@ -1386,7 +1586,7 @@ public class FM2ASTToFM3SourceConverter { * @param pos * The position where the first skipped character can occur (or the tag end character). */ - private int printStartTagEnd(TemplateElement node, int pos, boolean trimSlash) + private int printStartTagEnd(TemplateElement node, int pos, boolean removeSlash) throws ConverterException { final int startPos = pos; @@ -1398,7 +1598,16 @@ public class FM2ASTToFM3SourceConverter { char c = src.charAt(pos); if (c == '/' && pos + 1 < src.length() && src.charAt(pos + 1) == tagEndChar) { - printWithConvertedExpComments(src.substring(startPos, trimSlash ? pos : pos + 1)); + printWithConvertedExpComments(src.substring(startPos, pos)); + + if (removeSlash) { + // In <#foo param />, the space before the removed '/' should be removed: + if (out.length() > 0 && out.charAt(out.length() - 1) == ' ') { + out.setLength(out.length() - 1); + } + } else { + print('/'); + } print(tagEndChar); return pos + 1; } else if (c == tagEndChar) { @@ -1411,25 +1620,6 @@ public class FM2ASTToFM3SourceConverter { } } - private void printEndTagSkippedTokens(TemplateElement node) throws UnexpectedNodeContentException { - int tagEndPos = getEndPositionInclusive(node); - { - char c = src.charAt(tagEndPos); - assertNodeContent(c == tagEndChar, node, - "tagEndChar expected, found {}", c); - } - - int pos = tagEndPos - 1; - while (pos > 0 && Character.isWhitespace(src.charAt(pos))) { - pos--; - } - - assertNodeContent(pos > 0 && isCoreNameChar(src.charAt(pos)), node, - "Can't find end tag name"); - - printWithConvertedExpComments(src.substring(pos + 1, tagEndPos)); - } - private void printWithConvertedExpComments(String s) { // Later we might want to convert comment syntax here print(s); @@ -1584,7 +1774,7 @@ public class FM2ASTToFM3SourceConverter { return pos; } - private String readAndWSAndExpComments(int startPos) + private String readWSAndExpComments(int startPos) throws ConverterException { return src.substring(startPos, getPositionAfterWSAndExpComments(startPos)); } @@ -1608,7 +1798,7 @@ public class FM2ASTToFM3SourceConverter { } private int printWSAndExpComments(int pos) throws ConverterException { - String sep = readAndWSAndExpComments(pos); + String sep = readWSAndExpComments(pos); printWithConvertedExpComments(sep); pos += sep.length(); return pos; http://git-wip-us.apache.org/repos/asf/incubator-freemarker/blob/92db5891/freemarker-converter/src/main/java/org/apache/freemarker/converter/ConversionMarkers.java ---------------------------------------------------------------------- diff --git a/freemarker-converter/src/main/java/org/apache/freemarker/converter/ConversionMarkers.java b/freemarker-converter/src/main/java/org/apache/freemarker/converter/ConversionMarkers.java index dbfe939..fa92991 100644 --- a/freemarker-converter/src/main/java/org/apache/freemarker/converter/ConversionMarkers.java +++ b/freemarker-converter/src/main/java/org/apache/freemarker/converter/ConversionMarkers.java @@ -22,28 +22,42 @@ package org.apache.freemarker.converter; import java.util.ArrayList; import java.util.List; +import org.apache.freemarker.core.util._NullArgumentException; + /** - * Receives source code markings with positions. + * Stores markers that apply to a given position in the source or destination file. */ -public class ConversionMarkerReceiver { +public class ConversionMarkers { - private final List<Entry> sourceFileMarkers = new ArrayList<>(); - private final List<Entry> destinationFileMarkers = new ArrayList<>(); + private final List<Entry> sourceMarkers = new ArrayList<>(); + private final List<Entry> destinationMarkers = new ArrayList<>(); /** + * Adds a marker to the source file. * @param row * 1-based column in the source file * @param col * 1-based row in the source file * @param message - * Not {@code null} +* Not {@code null} */ - public void warnInSource(int row, int col, String message); + public void markInSource(int row, int col, Type type, String message) { + sourceMarkers.add(new Entry(row, col, Type.WARN, message)); + } /** - * Similar to {@link #warnInSource(int, int, String)}, but adds a tipInOutput instead of a warning message. + * Adds a marker to the destination file. + * + * @param row + * 1-based column in the source file + * @param col + * 1-based row in the source file + * @param message + * Not {@code null} */ - public void tipInOutput(int row, int col, String message); + public void markInDestination(int row, int col, Type type, String message) { + destinationMarkers.add(new Entry(row, col, type, message)); + } public enum Type { WARN, TIP @@ -52,11 +66,16 @@ public class ConversionMarkerReceiver { public class Entry { private final int row; private final int column; + private final Type type; private final String message; - public Entry(int row, int column, String message) { + public Entry(int row, int column, Type type, String message) { + _NullArgumentException.check("type", type); + _NullArgumentException.check("message", message); + this.row = row; this.column = column; + this.type = type; this.message = message; } @@ -68,9 +87,21 @@ public class ConversionMarkerReceiver { return column; } + public Type getType() { + return type; + } + public String getMessage() { return message; } } + public List<Entry> getSourceMarkers() { + return sourceMarkers; + } + + public List<Entry> getDestinationMarkers() { + return destinationMarkers; + } + } http://git-wip-us.apache.org/repos/asf/incubator-freemarker/blob/92db5891/freemarker-converter/src/main/java/org/apache/freemarker/converter/Converter.java ---------------------------------------------------------------------- diff --git a/freemarker-converter/src/main/java/org/apache/freemarker/converter/Converter.java b/freemarker-converter/src/main/java/org/apache/freemarker/converter/Converter.java index d9b3ea6..d35a34d 100644 --- a/freemarker-converter/src/main/java/org/apache/freemarker/converter/Converter.java +++ b/freemarker-converter/src/main/java/org/apache/freemarker/converter/Converter.java @@ -22,11 +22,14 @@ package org.apache.freemarker.converter; import java.io.File; import java.io.FileInputStream; import java.io.FileOutputStream; +import java.io.FileWriter; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.io.Writer; import java.util.HashSet; import java.util.Set; +import java.util.regex.Pattern; import org.apache.freemarker.core.util._NullArgumentException; import org.apache.freemarker.core.util._StringUtil; @@ -37,15 +40,17 @@ public abstract class Converter { public static final String PROPERTY_NAME_SOURCE = "source"; public static final String PROPERTY_NAME_DESTINATION_DIRECTORY = "destinationDirectory"; + public static final String CONVERSION_MARKERS_FILE_NAME = "__conversion-markers.txt"; private static final Logger LOG = LoggerFactory.getLogger(Converter.class); private File source; private File destinationDirectory; - private ConversionWarnReceiver conversionWarnReceiver = new LoggingWarnReceiver(); private boolean createDestinationDirectory; + private boolean executed; private Set<File> directoriesKnownToExist = new HashSet<>(); + private Writer conversionMarkersWriter; public File getSource() { return source; @@ -71,14 +76,6 @@ public abstract class Converter { this.createDestinationDirectory = createDestinationDirectory; } - public ConversionWarnReceiver getConversionWarnReceiver() { - return conversionWarnReceiver; - } - - public void setConversionWarnReceiver(ConversionWarnReceiver conversionWarnReceiver) { - this.conversionWarnReceiver = conversionWarnReceiver; - } - public final void execute() throws ConverterException { if (executed) { throw new IllegalStateException("This converted was already invoked once."); @@ -89,7 +86,20 @@ public abstract class Converter { LOG.debug("Source: {}", source); LOG.debug("Destination directory: {}", destinationDirectory); - convertFiles(source, destinationDirectory, true); + // Just so that no confusing marker file remains there: + File markerFile = new File(destinationDirectory, CONVERSION_MARKERS_FILE_NAME); + markerFile.delete(); + try { + convertFiles(source, destinationDirectory, true); + } finally { + if (conversionMarkersWriter != null) { + try { + conversionMarkersWriter.close(); + } catch (IOException e) { + throw new ConverterException("Marker file (" + markerFile + ") couldn't be written: ", e); + } + } + } } /** @@ -139,18 +149,17 @@ public abstract class Converter { } try { LOG.debug("Converting file: {}", src); - FileConversionContext fileTransCtx = null; + FileConversionContext ctx = null; try { - conversionWarnReceiver.setSourceFile(src); - fileTransCtx = new FileConversionContext(srcStream, src, dstDir, conversionWarnReceiver); - convertFile(fileTransCtx); + ctx = new FileConversionContext(srcStream, src, dstDir); + convertFile(ctx); + storeConversionMarkers(ctx.getConversionMarkers(), ctx); } catch (IOException e) { throw new ConverterException("I/O exception while converting " + _StringUtil.jQuote(src) + ".", e); } finally { - conversionWarnReceiver.setSourceFile(null); try { - if (fileTransCtx != null && fileTransCtx.outputStream != null) { - fileTransCtx.outputStream.close(); + if (ctx != null && ctx.outputStream != null) { + ctx.outputStream.close(); } } catch (IOException e) { throw new ConverterException("Failed to close destination file", e); @@ -165,6 +174,43 @@ public abstract class Converter { } } + private void storeConversionMarkers(ConversionMarkers conversionMarkers, FileConversionContext ctx) + throws ConverterException { + if (conversionMarkersWriter == null) { + File conversionMarkersFile = new File(destinationDirectory, CONVERSION_MARKERS_FILE_NAME); + try { + conversionMarkersWriter = new FileWriter(conversionMarkersFile); + } catch (IOException e) { + throw new ConverterException("Failed to create conversion marker file: " + conversionMarkersFile, e); + } + } + for (ConversionMarkers.Entry marker : conversionMarkers.getSourceMarkers()) { + try { + conversionMarkersWriter.write(toString(marker, ctx.getSourceFile())); + } catch (IOException e) { + throw new ConverterException("Failed to write conversion marker file", e); + } + } + for (ConversionMarkers.Entry marker : conversionMarkers.getDestinationMarkers()) { + try { + conversionMarkersWriter.write(toString(marker, ctx.getDestinationFile())); + } catch (IOException e) { + throw new ConverterException("Failed to write conversion marker file", e); + } + } + } + + private String toString(ConversionMarkers.Entry marker, File file) { + return "[" + marker.getType() + "] " + file + ":" + marker.getRow() + ":" + marker.getColumn() + + " " + addTabAfterLineBreaks(marker.getMessage()) + "\n"; + } + + private static final Pattern AFTER_LINE_BREAKS_PATTERN = Pattern.compile("\\n|\\r\\n?"); + + private String addTabAfterLineBreaks(String message) { + return AFTER_LINE_BREAKS_PATTERN.matcher(message).replaceAll("$0\t"); + } + private void ensureDirectoryExists(File dir) throws ConverterException { if (dir == null || fastIsDirectory(dir)) { return; @@ -209,16 +255,14 @@ public abstract class Converter { private final InputStream sourceStream; private final File sourceFile; private final File dstDir; - private final ConversionWarnReceiver conversionWarnReceiver; + private final ConversionMarkers conversionMarkers = new ConversionMarkers(); private String destinationFileName; private OutputStream outputStream; - public FileConversionContext( - InputStream sourceStream, File sourceFile, File dstDir, ConversionWarnReceiver conversionWarnReceiver) { + public FileConversionContext(InputStream sourceStream, File sourceFile, File dstDir) { this.sourceStream = sourceStream; this.sourceFile = sourceFile; this.dstDir = dstDir; - this.conversionWarnReceiver = conversionWarnReceiver; } /** @@ -252,7 +296,7 @@ public abstract class Converter { } ensureDirectoryExists(dstDir); - File dstFile = new File(dstDir, destinationFileName); + File dstFile = getDestinationFile(); try { outputStream = new FileOutputStream(dstFile); } catch (IOException e) { @@ -284,10 +328,16 @@ public abstract class Converter { this.destinationFileName = destinationFileName; } - public ConversionWarnReceiver getConversionWarnReceiver() { - return conversionWarnReceiver; + public ConversionMarkers getConversionMarkers() { + return conversionMarkers; } + public File getDestinationFile() { + if (destinationFileName == null) { + throw new IllegalStateException("FileConversionContext.destinationFileName wasn't yet set."); + } + return new File(dstDir, destinationFileName); + } } } http://git-wip-us.apache.org/repos/asf/incubator-freemarker/blob/92db5891/freemarker-converter/src/main/java/org/apache/freemarker/converter/FM2ToFM3Converter.java ---------------------------------------------------------------------- diff --git a/freemarker-converter/src/main/java/org/apache/freemarker/converter/FM2ToFM3Converter.java b/freemarker-converter/src/main/java/org/apache/freemarker/converter/FM2ToFM3Converter.java index db2a272..4dff35e 100644 --- a/freemarker-converter/src/main/java/org/apache/freemarker/converter/FM2ToFM3Converter.java +++ b/freemarker-converter/src/main/java/org/apache/freemarker/converter/FM2ToFM3Converter.java @@ -94,7 +94,7 @@ public class FM2ToFM3Converter extends Converter { protected void convertFile(FileConversionContext fileTransCtx) throws ConverterException, IOException { String src = IOUtils.toString(fileTransCtx.getSourceStream(), StandardCharsets.UTF_8); FM2ASTToFM3SourceConverter.Result result = FM2ASTToFM3SourceConverter.convert( - fileTransCtx.getSourceFile().getName(), src, fm2Cfg, fileTransCtx.getConversionWarnReceiver() + fileTransCtx.getSourceFile().getName(), src, fm2Cfg, fileTransCtx.getConversionMarkers() ); fileTransCtx.setDestinationFileName(getDestinationFileName(result.getFM2Template())); fileTransCtx.getDestinationStream().write( http://git-wip-us.apache.org/repos/asf/incubator-freemarker/blob/92db5891/freemarker-converter/src/test/java/org/freemarker/converter/FM2ToFM3ConverterTest.java ---------------------------------------------------------------------- diff --git a/freemarker-converter/src/test/java/org/freemarker/converter/FM2ToFM3ConverterTest.java b/freemarker-converter/src/test/java/org/freemarker/converter/FM2ToFM3ConverterTest.java index 362182c..138e048 100644 --- a/freemarker-converter/src/test/java/org/freemarker/converter/FM2ToFM3ConverterTest.java +++ b/freemarker-converter/src/test/java/org/freemarker/converter/FM2ToFM3ConverterTest.java @@ -20,6 +20,7 @@ package org.freemarker.converter; import static java.nio.charset.StandardCharsets.*; +import static org.hamcrest.Matchers.*; import static org.junit.Assert.*; import java.io.File; @@ -164,6 +165,8 @@ public class FM2ToFM3ConverterTest extends ConverterTest { assertConverted("<#if foo>1<#elseIf bar>2<#else>3</#if>", "<#if foo>1<#elseif bar>2<#else>3</#if>"); assertConvertedSame("<#if foo >1<#elseIf bar >2<#else >3</#if >"); + assertConverted("<#if foo>1<#elseIf bar>2<#else>3</#if>", "<#if foo>1<#elseif bar/>2<#else/>3</#if>"); + assertConverted("<#if foo>1<#elseIf bar>2<#else>3</#if>", "<#if foo>1<#elseif bar />2<#else />3</#if>"); assertConvertedSame("<#macro m>body</#macro>"); assertConvertedSame("<#macro <#--1--> m <#--2-->></#macro >"); @@ -238,8 +241,10 @@ public class FM2ToFM3ConverterTest extends ConverterTest { assertConvertedSame("<#include 'foo.ftl'>"); assertConverted("<#include 'foo.ftl' ignoreMissing=true>", "<#include 'foo.ftl' ignore_missing=true>"); + assertTrue(lastConversionMarkersFileContent.isEmpty()); assertConverted("<#include 'foo.ftl' ignoreMissing=true>", "<#include 'foo.ftl' ignore_missing=true encoding='utf-8' parse=false>"); + assertLastConversionMarkersFileContains("[WARN]", "encoding", "parse"); assertConverted("<#include 'foo.ftl' ignoreMissing=true>", "<#include 'foo.ftl' encoding='utf-8' ignore_missing=true parse=false>"); assertConverted("<#include 'foo.ftl' ignoreMissing=true>", @@ -253,6 +258,31 @@ public class FM2ToFM3ConverterTest extends ConverterTest { assertConverted("<#include <#--1--> 'foo.ftl' <#--2--> ignoreMissing=true <#--4-->>", "<#include <#--1--> 'foo.ftl' <#--2--> encoding='UTF-8' <#--3--> ignoreMissing=true <#--4--> " + "parse=true <#--5--> >"); + + assertConvertedSame("<#list xs as x>${x}</#list>"); + assertConvertedSame("<#list <#--1--> xs <#--2--> as <#--3--> x <#--4--> >${x}</#list >"); + assertConvertedSame("<#list xs as k, v>${k}${v}</#list>"); + assertConvertedSame("<#list <#--1--> xs <#--2--> as <#--3--> k <#--4-->, v <#--5--> >${k}${v}</#list >"); + + assertConverted("<#list xs as x>${x}</#list>", "<#foreach x in xs>${x}</#foreach>"); + assertConverted( + "<#list <#--1--> xs <#--XS--> as x <#--X-->>${x}</#list>", + "<#foreach <#--1--> x <#--X--> in xs <#--XS-->>${x}</#foreach>"); + + assertConvertedSame("<#list xs as x>${x}<#sep>, </#list>"); + assertConvertedSame("<#list xs as x>${x}<#sep>, </#sep></#list>"); + + assertConvertedSame("<#list xs as x>${x}<#else>-</#list>"); + assertConvertedSame("<#list xs as x>${x}<#else >-</#list >"); + assertConverted("<#list xs as x>${x}<#else>-</#list>", "<#list xs as x>${x}<#else/>-</#list>"); + + assertConvertedSame("<#list xs>[<#items as x>${x}<#sep>, </#items>]</#list>"); + assertConvertedSame("<#list xs>[<#items as <#--1--> x <#--2-->>${x}<#sep>, </#items>]</#list>"); + assertConvertedSame("<#list xs>[<#items as k, v>${h}${v}<#sep>, </#items>]</#list>"); + assertConvertedSame( + "<#list xs>[<#items as <#--1--> k <#--2-->, <#--3--> v <#--4-->>${h}${v}<#sep>, </#items>]</#list>"); + assertConvertedSame("<#list xs as x><#if x == 0><#break></#if>${x}</#list>"); + assertConvertedSame("<#list xs>[<#items as x>${x}<#sep>, </#sep >|</#items>]<#else>-</#list>"); } @Test @@ -347,10 +377,18 @@ public class FM2ToFM3ConverterTest extends ConverterTest { assertEquals(ftl3, convert(ftl2, squareBracketTagSyntax)); } + private void assertLastConversionMarkersFileContains(String... parts) { + for (String part : parts) { + assertThat(lastConversionMarkersFileContent, containsString(part)); + } + } + private String convert(String ftl2) throws IOException, ConverterException { return convert(ftl2, false); } + private String lastConversionMarkersFileContent; + private String convert(String ftl2, boolean squareBracketTagSyntax) throws IOException, ConverterException { File srcFile = new File(srcDir, "t"); FileUtils.write(srcFile, ftl2, UTF_8); @@ -373,6 +411,8 @@ public class FM2ToFM3ConverterTest extends ConverterTest { throw new IOException("Couldn't delete file: " + outputFile); } + lastConversionMarkersFileContent = readConversionMarkersFile(true); + return output; } http://git-wip-us.apache.org/repos/asf/incubator-freemarker/blob/92db5891/freemarker-converter/src/test/java/org/freemarker/converter/GenericConverterTest.java ---------------------------------------------------------------------- diff --git a/freemarker-converter/src/test/java/org/freemarker/converter/GenericConverterTest.java b/freemarker-converter/src/test/java/org/freemarker/converter/GenericConverterTest.java index 77abcfe..23fae4b 100644 --- a/freemarker-converter/src/test/java/org/freemarker/converter/GenericConverterTest.java +++ b/freemarker-converter/src/test/java/org/freemarker/converter/GenericConverterTest.java @@ -29,6 +29,7 @@ import java.io.IOException; import java.nio.charset.StandardCharsets; import org.apache.commons.io.IOUtils; +import org.apache.freemarker.converter.ConversionMarkers; import org.apache.freemarker.converter.MissingRequiredPropertyException; import org.apache.freemarker.converter.PropertyValidationException; import org.apache.freemarker.converter.Converter; @@ -164,15 +165,53 @@ public class GenericConverterTest extends ConverterTest { } } - public static class ToUpperCaseConverter extends Converter { + @Test + public void testMarksStored() throws IOException, ConverterException { + write(new File(srcDir, "warn.txt"), "[trigger warn]", UTF_8); + write(new File(srcDir, "tip.txt"), "[trigger tip]", UTF_8); + + ToUpperCaseConverter converter = new ToUpperCaseConverter(); + converter.setSource(srcDir); + converter.setDestinationDirectory(dstDir); + converter.execute(); + + String markersFileContent = readConversionMarkersFile(); + assertThat(markersFileContent, allOf( + containsString("[WARN]"), + containsString("warn.txt:1:2"), + containsString("Warn message"))); + assertThat(markersFileContent, allOf( + containsString("[TIP]"), + containsString("tip.txt.uc:1:2"), + containsString("Tip message"))); + } - public static final int BUFFER_SIZE = 4096; + @Test + public void emptyMarkFileCreated() throws IOException, ConverterException { + ToUpperCaseConverter converter = new ToUpperCaseConverter(); + converter.setSource(srcDir); + converter.setDestinationDirectory(dstDir); + converter.execute(); + + File markersFile = new File(dstDir, Converter.CONVERSION_MARKERS_FILE_NAME); + assertTrue(markersFile.exists()); + } + + public static class ToUpperCaseConverter extends Converter { @Override - protected void convertFile(FileConversionContext fileTransCtx) throws ConverterException, IOException { - String content = IOUtils.toString(fileTransCtx.getSourceStream(), StandardCharsets.UTF_8); - fileTransCtx.setDestinationFileName(fileTransCtx.getSourceFileName() + ".uc"); - IOUtils.write(content.toUpperCase(), fileTransCtx.getDestinationStream(), StandardCharsets.UTF_8); + protected void convertFile(FileConversionContext ctx) throws ConverterException, IOException { + String content = IOUtils.toString(ctx.getSourceStream(), StandardCharsets.UTF_8); + ctx.setDestinationFileName(ctx.getSourceFileName() + ".uc"); + if (content.contains("[trigger warn]")) { + ctx.getConversionMarkers().markInSource( + 1, 2, ConversionMarkers.Type.WARN, "Warn message"); + } + if (content.contains("[trigger tip]")) { + ctx.getConversionMarkers().markInDestination( + 1, 2, ConversionMarkers.Type.TIP, "Tip message"); + } + IOUtils.write(content.toUpperCase(), ctx.getDestinationStream(), StandardCharsets.UTF_8); } } http://git-wip-us.apache.org/repos/asf/incubator-freemarker/blob/92db5891/freemarker-converter/src/test/java/org/freemarker/converter/test/ConverterTest.java ---------------------------------------------------------------------- diff --git a/freemarker-converter/src/test/java/org/freemarker/converter/test/ConverterTest.java b/freemarker-converter/src/test/java/org/freemarker/converter/test/ConverterTest.java index 81df566..a7c9865 100644 --- a/freemarker-converter/src/test/java/org/freemarker/converter/test/ConverterTest.java +++ b/freemarker-converter/src/test/java/org/freemarker/converter/test/ConverterTest.java @@ -19,9 +19,14 @@ package org.freemarker.converter.test; +import static java.nio.charset.StandardCharsets.*; +import static org.apache.commons.io.FileUtils.*; +import static org.junit.Assert.*; + import java.io.File; import java.io.IOException; +import org.apache.freemarker.converter.Converter; import org.junit.Before; import org.junit.Rule; import org.junit.rules.TemporaryFolder; @@ -43,4 +48,22 @@ public abstract class ConverterTest { protected abstract void createSourceFiles() throws IOException; + protected String readConversionMarkersFile() throws IOException { + return readConversionMarkersFile(false); + } + + protected String readConversionMarkersFile(boolean delete) throws IOException { + File markersFile = getConversionMarkersFile(); + assertTrue(markersFile.isFile()); + String content = readFileToString(markersFile, UTF_8); + if (!markersFile.delete()) { + throw new IOException("Failed to delete file: " + markersFile); + } + return content; + } + + protected File getConversionMarkersFile() { + return new File(dstDir, Converter.CONVERSION_MARKERS_FILE_NAME); + } + }
