Continued FM2 to FM3 converter...
Project: http://git-wip-us.apache.org/repos/asf/incubator-freemarker/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-freemarker/commit/344b9541 Tree: http://git-wip-us.apache.org/repos/asf/incubator-freemarker/tree/344b9541 Diff: http://git-wip-us.apache.org/repos/asf/incubator-freemarker/diff/344b9541 Branch: refs/heads/3 Commit: 344b9541102b524a810f05b06bed9cdf93b23857 Parents: a0bc242 Author: ddekany <[email protected]> Authored: Sat Jul 8 00:59:50 2017 +0200 Committer: ddekany <[email protected]> Committed: Sat Jul 8 00:59:50 2017 +0200 ---------------------------------------------------------------------- freemarker-converter/build.gradle | 2 +- .../core/FM2ASTToFM3SourceConverter.java | 186 ++++++++++++++++--- .../freemarker/converter/FM2ToFM3Converter.java | 2 + .../converter/FM2ToFM3ConverterTest.java | 28 +++ 4 files changed, 192 insertions(+), 26 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-freemarker/blob/344b9541/freemarker-converter/build.gradle ---------------------------------------------------------------------- diff --git a/freemarker-converter/build.gradle b/freemarker-converter/build.gradle index c08bad7..ee42afe 100644 --- a/freemarker-converter/build.gradle +++ b/freemarker-converter/build.gradle @@ -25,7 +25,7 @@ inAggregateJavadoc = false dependencies { compile project(":freemarker-core") - compile "org.freemarker:freemarker:2.3.26-incubating" + compile "org.freemarker:freemarker-gae:2.3.27-incubating-SNAPSHOT" compile libraries.commonsCli compile libraries.commonsLang compile libraries.commonsIo http://git-wip-us.apache.org/repos/asf/incubator-freemarker/blob/344b9541/freemarker-converter/src/main/java/freemarker/core/FM2ASTToFM3SourceConverter.java ---------------------------------------------------------------------- diff --git a/freemarker-converter/src/main/java/freemarker/core/FM2ASTToFM3SourceConverter.java b/freemarker-converter/src/main/java/freemarker/core/FM2ASTToFM3SourceConverter.java index 1d6f484..de31532 100644 --- a/freemarker-converter/src/main/java/freemarker/core/FM2ASTToFM3SourceConverter.java +++ b/freemarker-converter/src/main/java/freemarker/core/FM2ASTToFM3SourceConverter.java @@ -236,22 +236,45 @@ public class FM2ASTToFM3SourceConverter { if (node instanceof MixedContent) { printChildElements(node); } else if (node instanceof TextBlock) { - print(getOnlyParam(node, ParameterRole.CONTENT, String.class)); + printText(node); } else if (node instanceof DollarVariable) { printDollarInterpolation((DollarVariable) node); } else if (node instanceof NumericalOutput) { printNumericalInterpolation((NumericalOutput) node); } else if (node instanceof Comment) { - print(tagBeginChar); - print("#--"); - print(getOnlyParam(node, ParameterRole.CONTENT, String.class)); - print("--"); - print(tagEndChar); + printComment((Comment) node); } else { printDir(node); } } + private void printText(TemplateElement node) throws ConverterException { + int startPos = getStartPosition(node); + int endPos = getEndPositionExclusive(node); + if (startPos < 0 || startPos >= endPos) { // empty text + return; + } + + boolean isNoParseBlock = src.startsWith(tagBeginChar + "#no", startPos); + if (isNoParseBlock) { + printCoreDirStartTagParameterless(node, "noParse"); + } + print(getOnlyParam(node, ParameterRole.CONTENT, String.class)); + if (isNoParseBlock) { + printCoreDirEndTag(node, NO_PARSE_FM_2_TAG_NAMES, "noParse"); + } + } + + private static final ImmutableList<String> NO_PARSE_FM_2_TAG_NAMES = ImmutableList.of("noparse", "noParse"); + + private void printComment(Comment node) throws UnexpectedNodeContentException { + print(tagBeginChar); + print("#--"); + print(getOnlyParam(node, ParameterRole.CONTENT, String.class)); + print("--"); + print(tagEndChar); + } + private void printNumericalInterpolation(NumericalOutput node) throws ConverterException { printWithParamsLeadingSkippedTokens("${", node); Expression content = getParam(node, 0, ParameterRole.CONTENT, Expression.class); @@ -320,8 +343,12 @@ public class FM2ASTToFM3SourceConverter { printDirCustom((UnifiedCall) node); } else if (node instanceof Macro) { printDirMacroOrFunction((Macro) node); + } else if (node instanceof BodyInstruction) { + printDirNested((BodyInstruction) node); } else if (node instanceof Assignment) { printDirAssignmentLonely((Assignment) node); + } else if (node instanceof BlockAssignment) { + printDirBlockAssignment((BlockAssignment) node); } else if (node instanceof AssignmentInstruction) { printDirAssignmentMultiple((AssignmentInstruction) node); } else if (node instanceof AttemptBlock) { @@ -356,11 +383,45 @@ public class FM2ASTToFM3SourceConverter { printDirItems((Items) node); } else if (node instanceof BreakInstruction) { printDirBreak((BreakInstruction) node); + } else if (node instanceof TrimInstruction) { + printDirTOrNtOrLtOrRt((TrimInstruction) node); } else { throw new ConverterException("Unhandled AST TemplateElement class: " + node.getClass().getName()); } } + private void printDirTOrNtOrLtOrRt(TrimInstruction node) throws ConverterException { + int subtype= getOnlyParam(node, ParameterRole.AST_NODE_SUBTYPE, Integer.class); + String tagName; + if (subtype == TrimInstruction.TYPE_T) { + tagName = "t"; + } else if (subtype == TrimInstruction.TYPE_LT) { + tagName = "lt"; + } else if (subtype == TrimInstruction.TYPE_RT) { + tagName = "rt"; + } else if (subtype == TrimInstruction.TYPE_NT) { + tagName = "nt"; + } else { + throw new UnexpectedNodeContentException(node, "Unhandled subtype {}.", subtype); + } + + printCoreDirStartTagParameterless(node, tagName); + } + + private void printDirNested(BodyInstruction node) throws ConverterException { + int pos = printCoreDirStartTagBeforeParams(node, "nested"); + int paramCnt = node.getParameterCount(); + for (int paramIdx = 0; paramIdx < paramCnt; paramIdx++) { + Expression passedValue = getParam(node, paramIdx, ParameterRole.PASSED_VALUE, Expression.class); + printExp(passedValue); + pos = getEndPositionExclusive(passedValue); + if (paramIdx < paramCnt - 1) { + printOptionalSeparatorAndWSAndExpComments(pos, ","); + } + } + printStartTagEnd(node, pos, true); + } + private void printDirBreak(BreakInstruction node) throws ConverterException { printCoreDirStartTagParameterless(node, "break"); } @@ -489,16 +550,14 @@ public class FM2ASTToFM3SourceConverter { printChildElements(node); if (printEndTag) { - printCoreDirEndTag(node, ImmutableList.of("list", "foreach", "forEach"), "list", false); + printCoreDirEndTag(node, LIST_FM_2_TAG_NAMES, "list", false); } } + private static final ImmutableList<String> LIST_FM_2_TAG_NAMES = ImmutableList.of("list", "foreach", "forEach"); + private void printDirInclude(Include node) throws ConverterException { - if (Configuration.getVersion().intValue() != Configuration.VERSION_2_3_26.intValue()) { - throw new BugException("Fix things at [broken in 2.3.26] comments; version was: " - + Configuration.getVersion()); - } - // assertParamCount(node, 4); // [broken in 2.3.26] + assertParamCount(node, 4); printCoreDirStartTagBeforeParams(node, "include"); @@ -521,9 +580,7 @@ public class FM2ASTToFM3SourceConverter { + "encoding than the configured default."); } - // Can't use as parameterCount is [broken in 2.3.26]: - // Expression ignoreMissingParam = getParam(node, 3, ParameterRole.IGNORE_MISSING_PARAMETER, Expression.class); - Expression ignoreMissingParam = (Expression) node.getParameterValue(3); + Expression ignoreMissingParam = getParam(node, 3, ParameterRole.IGNORE_MISSING_PARAMETER, Expression.class); List<Expression> sortedExps = sortExpressionsByPosition(templateName, parseParam, encodingParam, ignoreMissingParam); @@ -611,9 +668,11 @@ public class FM2ASTToFM3SourceConverter { } private void printDirNoEscape(NoEscapeBlock node) throws ConverterException { - printDirGenericParameterlessWithNestedContent(node, "noEscape"); + printDirGenericParameterlessWithNestedContent(node, NO_ESCAPE_FM_2_TAG_NAMES, "noEscape"); } + private static final ImmutableList<String> NO_ESCAPE_FM_2_TAG_NAMES = ImmutableList.of("noescape", "noEscape"); + private void printDirEscape(EscapeBlock node) throws ConverterException { assertParamCount(node, 2); @@ -638,20 +697,30 @@ public class FM2ASTToFM3SourceConverter { } private void printDirAutoEsc(AutoEscBlock node) throws ConverterException { - printDirGenericParameterlessWithNestedContent(node, "autoEsc"); + printDirGenericParameterlessWithNestedContent(node, AUTO_ESC_FM_2_TAG_NAMES, "autoEsc"); } + private static final ImmutableList<String> AUTO_ESC_FM_2_TAG_NAMES = ImmutableList.of("autoesc", "autoEsc"); + private void printDirNoAutoEsc(NoAutoEscBlock node) throws ConverterException { - printDirGenericParameterlessWithNestedContent(node, "noAutoEsc"); + printDirGenericParameterlessWithNestedContent(node, NO_AUTO_ESC_FM_2_TAG_NAMES, "noAutoEsc"); + } + + private static final ImmutableList<String> NO_AUTO_ESC_FM_2_TAG_NAMES = ImmutableList.of("noautoesc", "noAutoEsc"); + + private void printDirGenericParameterlessWithNestedContent(TemplateElement node, String fm3TagName) + throws ConverterException { + printDirGenericParameterlessWithNestedContent(node, Collections.singleton(fm3TagName), fm3TagName); } - private void printDirGenericParameterlessWithNestedContent(TemplateElement node, String tagName) + private void printDirGenericParameterlessWithNestedContent(TemplateElement node, + Collection<String> fm2TagName, String fm3TagName) throws ConverterException { assertParamCount(node, 0); - printCoreDirStartTagParameterless(node, tagName); + printCoreDirStartTagParameterless(node, fm3TagName); printChildElements(node); - printCoreDirEndTag(node, tagName); + printCoreDirEndTag(node, fm2TagName, fm3TagName); } private void printDirGenericParameterlessWithoutNestedContent(TemplateElement node, String name) @@ -674,9 +743,11 @@ public class FM2ASTToFM3SourceConverter { printChildElements(recoverDir); // In FM2 this could be </#recover> as well, but we normalize it - printCoreDirEndTag(node, ImmutableList.of("attempt", "recover"), "attempt", false); + printCoreDirEndTag(node, ATTEMPT_RECOVER_FM_2_TAG_NAMES, "attempt", false); } + private static final ImmutableList<String> ATTEMPT_RECOVER_FM_2_TAG_NAMES = ImmutableList.of("attempt", "recover"); + private void printDirAssignmentMultiple(AssignmentInstruction node) throws ConverterException { assertParamCount(node, 2); @@ -702,6 +773,28 @@ public class FM2ASTToFM3SourceConverter { printDirAssignmentCommonTagAfterLastAssignmentExp(node, 4, pos); } + private void printDirBlockAssignment(BlockAssignment node) throws ConverterException { + assertParamCount(node, 3); + + int pos = printDirAssignmentCommonTagTillAssignmentExp(node, 1); + + print(FTLUtil.escapeIdentifier(getParam(node, 0, ParameterRole.ASSIGNMENT_TARGET, String.class))); + pos = getPositionAfterAssignmentTargetIdentifier(pos); + + Expression namespace = getParam(node, 2, ParameterRole.NAMESPACE, Expression.class); + if (namespace != null) { + printSeparatorAndWSAndExpComments(pos, "in"); + printExp(namespace); + pos = getEndPositionExclusive(namespace); + } + + printStartTagEnd(node, pos, true); + + printChildElements(node); + + printCoreDirEndTag(node, getAssignmentDirTagName(node, 1)); + } + private void printDirAssignmentCommonTagAfterLastAssignmentExp(TemplateElement node, int nsParamIdx, int pos) throws ConverterException { Expression ns = getParam(node, nsParamIdx, ParameterRole.NAMESPACE, Expression.class); @@ -719,6 +812,11 @@ public class FM2ASTToFM3SourceConverter { private int printDirAssignmentCommonTagTillAssignmentExp(TemplateElement node, int scopeParamIdx) throws ConverterException { + return printCoreDirStartTagBeforeParams(node, getAssignmentDirTagName(node, scopeParamIdx)); + } + + private String getAssignmentDirTagName(TemplateElement node, int scopeParamIdx) + throws UnexpectedNodeContentException { int scope = getParam(node, scopeParamIdx, ParameterRole.VARIABLE_SCOPE, Integer.class); String tagName; if (scope == Assignment.NAMESPACE) { @@ -730,7 +828,7 @@ public class FM2ASTToFM3SourceConverter { } else { throw new UnexpectedNodeContentException(node, "Unhandled scope: {}", scope); } - return printCoreDirStartTagBeforeParams(node, tagName); + return tagName; } private int printDirAssignmentCommonExp(Assignment node, int pos) throws ConverterException { @@ -1360,7 +1458,7 @@ public class FM2ASTToFM3SourceConverter { { int pos = getStartPosition(node); quote = src.charAt(pos); - while ((quote == '\\' || quote == '{' /* 2.3.26 bug workaround */ || quote == 'r') + while ((quote == '\\' || quote == '{' /* [broken in 2.3.26] */ || quote == 'r') && pos < src.length()) { pos++; if (quote == 'r') { @@ -1459,7 +1557,20 @@ public class FM2ASTToFM3SourceConverter { } private void printCoreDirEndTag(TemplateElement node, String tagName) throws UnexpectedNodeContentException { - printCoreDirEndTag(node, Collections.singleton(tagName), tagName, false); + printCoreDirEndTag(node, Collections.singleton(tagName), tagName); + } + + private void printCoreDirEndTag(TemplateElement node, Collection<String> fm2TagName, String fm3TagName) throws + UnexpectedNodeContentException { + if (fm2TagName.size() == 0) { + throw new IllegalArgumentException("You must specify at least 1 FM2 tag names"); + } + if (fm2TagName.size() == 1 && containsUpperCaseLetter(fm3TagName)) { + throw new IllegalArgumentException( + "You must specify multiple FM2 tag names when the FM3 tag name (" + + fm3TagName + ") contains upper case letters"); + } + printCoreDirEndTag(node, fm2TagName, fm3TagName, false); } private void printCoreDirEndTag(TemplateElement node, Collection<String> fm2TagNames, String fm3TagName, @@ -1717,6 +1828,9 @@ public class FM2ASTToFM3SourceConverter { * 1-based row */ private int getPosition(int column, int row) { + if (row == 0) { + return -1; + } if (rowStartPositions == null) { rowStartPositions = new ArrayList<>(); rowStartPositions.add(0); @@ -1945,4 +2059,26 @@ public class FM2ASTToFM3SourceConverter { return i != -1 ? s.substring(0, i + 1) : ""; } + private boolean isUpperCaseLetter(char c) { + return Character.isUpperCase(c) && Character.isLetter(c); + } + + private HashMap<String, Boolean> containsUpperCaseLetterResults = new HashMap<>(); + + private boolean containsUpperCaseLetter(String s) { + Boolean result = containsUpperCaseLetterResults.get(s); + if (result != null) { + return result; + } + + int i = 0; + while (i < s.length() && !isUpperCaseLetter(s.charAt(i))) { + i++; + } + result = i < s.length(); + + containsUpperCaseLetterResults.put(s, result); + return result; + } + } http://git-wip-us.apache.org/repos/asf/incubator-freemarker/blob/344b9541/freemarker-converter/src/main/java/org/apache/freemarker/converter/FM2ToFM3Converter.java ---------------------------------------------------------------------- diff --git a/freemarker-converter/src/main/java/org/apache/freemarker/converter/FM2ToFM3Converter.java b/freemarker-converter/src/main/java/org/apache/freemarker/converter/FM2ToFM3Converter.java index 4dff35e..1e024e5 100644 --- a/freemarker-converter/src/main/java/org/apache/freemarker/converter/FM2ToFM3Converter.java +++ b/freemarker-converter/src/main/java/org/apache/freemarker/converter/FM2ToFM3Converter.java @@ -33,6 +33,7 @@ import org.slf4j.LoggerFactory; import freemarker.core.FM2ASTToFM3SourceConverter; import freemarker.template.Configuration; import freemarker.template.Template; +import freemarker.template._TemplateAPI; public class FM2ToFM3Converter extends Converter { @@ -59,6 +60,7 @@ public class FM2ToFM3Converter extends Converter { fm2Cfg = new Configuration(Configuration.VERSION_2_3_19 /* To fix ignored initial unknown tags */); fm2Cfg.setWhitespaceStripping(false); fm2Cfg.setTabSize(1); + _TemplateAPI.setPreventStrippings(fm2Cfg, true); try { fm2Cfg.setSettings(freeMarker2Settings); } catch (Exception e) { http://git-wip-us.apache.org/repos/asf/incubator-freemarker/blob/344b9541/freemarker-converter/src/test/java/org/freemarker/converter/FM2ToFM3ConverterTest.java ---------------------------------------------------------------------- diff --git a/freemarker-converter/src/test/java/org/freemarker/converter/FM2ToFM3ConverterTest.java b/freemarker-converter/src/test/java/org/freemarker/converter/FM2ToFM3ConverterTest.java index 138e048..1243251 100644 --- a/freemarker-converter/src/test/java/org/freemarker/converter/FM2ToFM3ConverterTest.java +++ b/freemarker-converter/src/test/java/org/freemarker/converter/FM2ToFM3ConverterTest.java @@ -191,6 +191,12 @@ public class FM2ToFM3ConverterTest extends ConverterTest { assertConvertedSame("<#macro m\\-1 p\\-1></#macro>"); // Only works with " now, as it doesn't keep the literal kind. Later we will escape differently anyway: assertConvertedSame("<#macro \"m 1\"></#macro>"); + assertConvertedSame("<#macro m><#nested x + 1, 2, 3></#macro>"); + assertConvertedSame("<#macro m><#nested <#--1--> x + 1 <#--2-->, <#--3--> 2 <#--4-->></#macro>"); + // [FM3] Will be different (comma) + assertConvertedSame("<#macro m><#nested x + 1 2 3></#macro>"); + assertConvertedSame("<#macro m><#nested <#--1--> x + 1 <#--2--> 2 <#--3-->></#macro>"); + assertConverted("<#macro m><#nested x></#macro>", "<#macro m><#nested x /></#macro>"); assertConvertedSame("<#assign x = 1>"); assertConvertedSame("<#global x = 1>"); @@ -207,6 +213,14 @@ public class FM2ToFM3ConverterTest extends ConverterTest { // Only works with " now, as it doesn't keep the literal kind. Later we will escape differently anyway: assertConvertedSame("<#assign \"x y\" = 1>"); + assertConvertedSame("<#assign x>t</#assign>"); + assertConvertedSame("<#assign x in ns>t</#assign>"); + assertConvertedSame("<#assign x\\-y>t</#assign>"); + assertConvertedSame("<#assign \"x y\">t</#assign>"); + assertConvertedSame("<#global x>t</#global>"); + assertConvertedSame("<#macro m><#local x>t</#local></#macro>"); + assertConvertedSame("<#assign <#--1--> x <#--2--> in <#--3--> ns <#--4-->>t</#assign >"); + assertConvertedSame("<#attempt>a<#recover>r</#attempt>"); assertConvertedSame("<#attempt >a<#recover >r</#attempt >"); assertConverted("<#attempt>a<#recover>r</#attempt>", "<#attempt>a<#recover>r</#recover>"); @@ -225,6 +239,9 @@ public class FM2ToFM3ConverterTest extends ConverterTest { assertConvertedSame("<#ftl outputFormat='XML'><#noAutoEsc><#autoEsc>${x}</#autoEsc></#noAutoEsc>"); assertConvertedSame("<#ftl outputFormat='XML'><#noAutoEsc ><#autoEsc\t>${x}</#autoEsc\n></#noAutoEsc\r>"); + assertConverted( + "<#ftl outputFormat='XML'><#noAutoEsc>${x}</#noAutoEsc>", + "<#ftl output_format='XML'><#noautoesc>${x}</#noautoesc>"); assertConvertedSame("<#compress>x</#compress>"); assertConvertedSame("<#compress >x</#compress >"); @@ -283,6 +300,17 @@ public class FM2ToFM3ConverterTest extends ConverterTest { "<#list xs>[<#items as <#--1--> k <#--2-->, <#--3--> v <#--4-->>${h}${v}<#sep>, </#items>]</#list>"); assertConvertedSame("<#list xs as x><#if x == 0><#break></#if>${x}</#list>"); assertConvertedSame("<#list xs>[<#items as x>${x}<#sep>, </#sep >|</#items>]<#else>-</#list>"); + + assertConvertedSame("<#noParse><#foo>${1}<#----></#noParse>"); + assertConverted("<#noParse >t</#noParse >", "<#noparse >t</#noparse >"); + + assertConvertedSame("<#assign x = 1><#t>"); + assertConvertedSame("a<#t>\nb"); + assertConvertedSame("<#t><#nt><#lt><#rt>"); + assertConvertedSame("<#t ><#nt ><#lt ><#rt >"); + assertConverted("<#t><#nt><#lt><#rt>", "<#t /><#nt /><#lt /><#rt />"); + + assertConvertedSame("<#ftl stripText='true'>\n\n<#macro m>\nx\n</#macro>\n"); } @Test
