Repository: incubator-freemarker Updated Branches: refs/heads/3 80e4c3424 -> 4763581a7
Continued working on FM2 to FM3 converter... Project: http://git-wip-us.apache.org/repos/asf/incubator-freemarker/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-freemarker/commit/4763581a Tree: http://git-wip-us.apache.org/repos/asf/incubator-freemarker/tree/4763581a Diff: http://git-wip-us.apache.org/repos/asf/incubator-freemarker/diff/4763581a Branch: refs/heads/3 Commit: 4763581a7c9b42058ca604efef6117639b8e32bd Parents: 80e4c34 Author: ddekany <[email protected]> Authored: Mon Jul 10 01:07:20 2017 +0200 Committer: ddekany <[email protected]> Committed: Mon Jul 10 01:07:20 2017 +0200 ---------------------------------------------------------------------- .../apache/freemarker/converter/Converter.java | 116 +++++++++++++++++++ .../freemarker/converter/FM2ToFM3Converter.java | 38 ++++-- .../converter/FM2ToFM3ConverterTest.java | 28 ++++- .../converter/GenericConverterTest.java | 39 +++++++ 4 files changed, 209 insertions(+), 12 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-freemarker/blob/4763581a/freemarker-converter/src/main/java/org/apache/freemarker/converter/Converter.java ---------------------------------------------------------------------- diff --git a/freemarker-converter/src/main/java/org/apache/freemarker/converter/Converter.java b/freemarker-converter/src/main/java/org/apache/freemarker/converter/Converter.java index 2b3d8eb..2c4fa0b 100644 --- a/freemarker-converter/src/main/java/org/apache/freemarker/converter/Converter.java +++ b/freemarker-converter/src/main/java/org/apache/freemarker/converter/Converter.java @@ -27,7 +27,9 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.io.Writer; +import java.nio.file.Path; import java.util.HashSet; +import java.util.Iterator; import java.util.Set; import java.util.regex.Pattern; @@ -36,16 +38,26 @@ import org.apache.freemarker.core.util._StringUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +/** + * Abstract superclass for converters. A converter converts the files in the source directory (and in its + * subdirectories, recursively), or of a single source file, and writes the converted files into the the destination + * directory. + */ public abstract class Converter { + public static final String PROPERTY_NAME_INCLUDE = "include"; + public static final String PROPERTY_NAME_EXCLUDE = "exclude"; public static final String PROPERTY_NAME_SOURCE = "source"; public static final String PROPERTY_NAME_DESTINATION_DIRECTORY = "destinationDirectory"; + public static final String CONVERSION_MARKERS_FILE_NAME = "__conversion-markers.txt"; public static ThreadLocal<FileConversionContext> FILE_CONVERSION_CONTEXT_TLS = new ThreadLocal<>(); private static final Logger LOG = LoggerFactory.getLogger(Converter.class); + private Pattern include; + private Pattern exclude; private File source; private File destinationDirectory; private boolean createDestinationDirectory; @@ -54,30 +66,106 @@ public abstract class Converter { private Set<File> directoriesKnownToExist = new HashSet<>(); private Writer conversionMarkersWriter; + public Converter() { + include = getDefaultInclude(); + } + + /** + * Getter pair of {@link #setSource(File)}. + */ public File getSource() { return source; } + /** + * Sets the directory that contains the files to be converted (subdirectories will be also searched, and so on + * recursively), or the single file that should be converted. Can't be {@code null}. + */ public void setSource(File source) { + _NullArgumentException.check("source", source); this.source = source != null ? source.getAbsoluteFile() : null; } + /** + * Getter pair of {@link #setDestinationDirectory(File)}. + */ public File getDestinationDirectory() { return destinationDirectory; } + /** + * Sets the directory into which the converted files will be written. Unless + * {@linkplain #setCreateDestinationDirectory(boolean) createDestinationDirectory property} was set to {@code true}, + * the directory must already exits. + * <p> + * Already existing files will be overwritten without warning. Already existing files and subdirectories for which + * there's no clashing output will be left as is. + */ public void setDestinationDirectory(File destinationDirectory) { this.destinationDirectory = destinationDirectory != null ? destinationDirectory.getAbsoluteFile() : null; } + /** + * Getter pair of {@link #setCreateDestinationDirectory(boolean)}. + */ public boolean isCreateDestinationDirectory() { return createDestinationDirectory; } + /** + * Sets whether the destination directory should be created if it doesn't exist. Defaults to {@code false}. + * Subdirectories inside the destination directory will be always automatically created, regardless of this + * property. + */ public void setCreateDestinationDirectory(boolean createDestinationDirectory) { this.createDestinationDirectory = createDestinationDirectory; } + /** + * Getter pair of {@link #setInclude(Pattern)}. + */ + public Pattern getInclude() { + return include; + } + + /** + * Sets what subset of the files selected by the {@linkplain #setSource(File) source property} will be + * processed (unless they are excluded by the {@linkplain #setExclude(Pattern) exclude} property.) + * Default is {@link #getDefaultInclude()}. + * + * @param include Matched against the source directory relative path. In the matched path, backslashes are + * replaced with slash. If {@code null}, then matches everything. + */ + public void setInclude(Pattern include) { + this.include = include; + } + + /** + * Returns the default value of the {@linkplain #setInclude(Pattern) include property}. + */ + protected abstract Pattern getDefaultInclude(); + + /** + * Getter pair of {@link #setExclude(Pattern)}. + */ + public Pattern getExclude() { + return exclude; + } + + /** + * Sets what subset of the files selected by the {@linkplain #setSource(File) source property} will not be + * processed. Default is {@code null}. + * + * @param exclude Matched against the source directory relative path. In the matched path, backslashes are + * replaced with slash. If {@code null}, then matches noting (nothing will be excluded). + */ + public void setExclude(Pattern exclude) { + this.exclude = exclude; + } + + /** + * Executes the file conversions. + */ public final void execute() throws ConverterException { if (executed) { throw new IllegalStateException("This converted was already invoked once."); @@ -143,6 +231,10 @@ public abstract class Converter { } private void convertFile(File src, File dstDir) throws ConverterException { + if (!isToBeProcessed(src)) { + return; + } + InputStream srcStream; try { srcStream = new FileInputStream(src); @@ -181,6 +273,30 @@ public abstract class Converter { } } + private boolean isToBeProcessed(File src) { + String relSrcPath; + File source = getSource(); + if (source.isFile()) { + relSrcPath = source.getName(); + } else { + relSrcPath = pathToStringWithSlashes(source.toPath().relativize(src.toPath()).normalize()); + } + + return (include == null || include.matcher(relSrcPath).matches()) + && (exclude == null || !exclude.matcher(relSrcPath).matches()); + } + + private String pathToStringWithSlashes(Path path) { + StringBuilder sb = new StringBuilder(); + for (Iterator<Path> it = path.iterator(); it.hasNext();) { + if (sb.length() != 0) { + sb.append('/'); + } + sb.append(it.next()); + } + return sb.toString(); + } + private void storeConversionMarkers(ConversionMarkers conversionMarkers, FileConversionContext ctx) throws ConverterException { if (conversionMarkersWriter == null) { http://git-wip-us.apache.org/repos/asf/incubator-freemarker/blob/4763581a/freemarker-converter/src/main/java/org/apache/freemarker/converter/FM2ToFM3Converter.java ---------------------------------------------------------------------- diff --git a/freemarker-converter/src/main/java/org/apache/freemarker/converter/FM2ToFM3Converter.java b/freemarker-converter/src/main/java/org/apache/freemarker/converter/FM2ToFM3Converter.java index 1e024e5..0e2e114 100644 --- a/freemarker-converter/src/main/java/org/apache/freemarker/converter/FM2ToFM3Converter.java +++ b/freemarker-converter/src/main/java/org/apache/freemarker/converter/FM2ToFM3Converter.java @@ -24,20 +24,37 @@ import java.nio.charset.StandardCharsets; import java.util.HashMap; import java.util.Map; import java.util.Properties; +import java.util.regex.Pattern; import org.apache.commons.io.IOUtils; import org.apache.freemarker.core.util._NullArgumentException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import freemarker.core.FM2ASTToFM3SourceConverter; import freemarker.template.Configuration; import freemarker.template.Template; import freemarker.template._TemplateAPI; +/** + * Converts FreeMarker 2 templates to FreeMarker 3 templates, as far as it's possible automatically. While the output + * will contain syntactically correct FreeMarker 3 templates, the templates will have to be reviewed by humans, as + * due to the semantic differences (such as a different treatment of {@code null}). + * <p> + * This is work in progress... new conversion are mostly only added when the syntactical change was + * already implemented. Current conversions: + * <ul> + * <li>All FreeMarker defined names are converted camel case + * <li>Renamed setting names in the {@code ftl} heading and in {@code #setting} tags are replaced with the new name. + * <li>Renamed built-in variables are replaced with the new name + * <li>From {@code #else}, {@code #elseif} and {@code #recover} tags that end with {@code "/>"} or {@code "/]"} the + * "/" characters is removed (as it's now illegal) + * <li>{@code #else}, {@code #elseif} and {@code #recover} tags that end with {@code "/>"} or {@code "/]"} the + * "/" characters is removed (as it's now illegal) + * <li>The last tag in {@code <#attempt>...<#recover>...</#recover>} is replaced with {@code </#attempt>} + * </ul> + */ public class FM2ToFM3Converter extends Converter { - public static final Logger LOG = LoggerFactory.getLogger(Converter.class); + private static final Pattern DEFAULT_INCLUDE = Pattern.compile("(?i).*\\.(fm|ftl(x|h)?)"); private static final Map<String, String> DEFAULT_REPLACED_FILE_EXTENSIONS; static { @@ -55,16 +72,23 @@ public class FM2ToFM3Converter extends Converter { private Configuration fm2Cfg; @Override + protected Pattern getDefaultInclude() { + return DEFAULT_INCLUDE; + } + + @Override protected void prepare() throws ConverterException { super.prepare(); fm2Cfg = new Configuration(Configuration.VERSION_2_3_19 /* To fix ignored initial unknown tags */); fm2Cfg.setWhitespaceStripping(false); fm2Cfg.setTabSize(1); _TemplateAPI.setPreventStrippings(fm2Cfg, true); - try { - fm2Cfg.setSettings(freeMarker2Settings); - } catch (Exception e) { - throw new ConverterException("Error while configuring FreeMarker 2", e); + if (freeMarker2Settings != null) { + try { + fm2Cfg.setSettings(freeMarker2Settings); + } catch (Exception e) { + throw new ConverterException("Error while configuring FreeMarker 2", e); + } } } http://git-wip-us.apache.org/repos/asf/incubator-freemarker/blob/4763581a/freemarker-converter/src/test/java/org/freemarker/converter/FM2ToFM3ConverterTest.java ---------------------------------------------------------------------- diff --git a/freemarker-converter/src/test/java/org/freemarker/converter/FM2ToFM3ConverterTest.java b/freemarker-converter/src/test/java/org/freemarker/converter/FM2ToFM3ConverterTest.java index 7d691c9..07af58b 100644 --- a/freemarker-converter/src/test/java/org/freemarker/converter/FM2ToFM3ConverterTest.java +++ b/freemarker-converter/src/test/java/org/freemarker/converter/FM2ToFM3ConverterTest.java @@ -426,7 +426,27 @@ public class FM2ToFM3ConverterTest extends ConverterTest { } @Test - public void testFileExtensions() throws IOException, ConverterException { + public void testDefaultIncludes() throws IOException, ConverterException { + FileUtils.write(new File(srcDir, "t.txt"), "x", UTF_8); + FileUtils.write(new File(srcDir, "t.fm"), "x", UTF_8); + FileUtils.write(new File(srcDir, "t.ftl"), "x", UTF_8); + FileUtils.write(new File(srcDir, "t.ftlfoo"), "x", UTF_8); + FileUtils.write(new File(srcDir, "U.FTLH"), "x", UTF_8); + + FM2ToFM3Converter converter = new FM2ToFM3Converter(); + converter.setSource(srcDir); + converter.setDestinationDirectory(dstDir); + converter.execute(); + + assertFalse(new File(dstDir, "t.txt").exists()); + assertTrue(new File(dstDir, "t.fm3").exists()); + assertTrue(new File(dstDir, "t.fm3").exists()); + assertFalse(new File(dstDir, "t.ftlfoo").exists()); + assertTrue(new File(dstDir, "U.fm3h").exists()); + } + + @Test + public void testFileExtensionConversion() throws IOException, ConverterException { FileUtils.write(new File(srcDir, "t1"), "x", UTF_8); FileUtils.write(new File(srcDir, "t2.foo"), "x", UTF_8); FileUtils.write(new File(srcDir, "t3.ftl"), "x", UTF_8); @@ -441,10 +461,7 @@ public class FM2ToFM3ConverterTest extends ConverterTest { FM2ToFM3Converter converter = new FM2ToFM3Converter(); converter.setSource(srcDir); converter.setDestinationDirectory(dstDir); - Properties properties = new Properties(); - properties.setProperty(Configuration.DEFAULT_ENCODING_KEY, UTF_8.name()); - converter.setFreeMarker2Settings(properties); - + converter.setInclude(null); converter.execute(); assertTrue(new File(dstDir, "t1").exists()); @@ -496,6 +513,7 @@ public class FM2ToFM3ConverterTest extends ConverterTest { FM2ToFM3Converter converter = new FM2ToFM3Converter(); converter.setSource(srcFile); converter.setDestinationDirectory(dstDir); + converter.setInclude(null); Properties properties = new Properties(); properties.setProperty(Configuration.DEFAULT_ENCODING_KEY, UTF_8.name()); if (squareBracketTagSyntax) { http://git-wip-us.apache.org/repos/asf/incubator-freemarker/blob/4763581a/freemarker-converter/src/test/java/org/freemarker/converter/GenericConverterTest.java ---------------------------------------------------------------------- diff --git a/freemarker-converter/src/test/java/org/freemarker/converter/GenericConverterTest.java b/freemarker-converter/src/test/java/org/freemarker/converter/GenericConverterTest.java index 7fcee1a..0873133 100644 --- a/freemarker-converter/src/test/java/org/freemarker/converter/GenericConverterTest.java +++ b/freemarker-converter/src/test/java/org/freemarker/converter/GenericConverterTest.java @@ -27,6 +27,7 @@ import static org.junit.Assert.*; import java.io.File; import java.io.IOException; import java.nio.charset.StandardCharsets; +import java.util.regex.Pattern; import org.apache.commons.io.IOUtils; import org.apache.freemarker.converter.ConversionMarkers; @@ -212,9 +213,47 @@ public class GenericConverterTest extends ConverterTest { } } + @Test + public void testIncludeAndExclude() throws IOException, ConverterException { + write(new File(srcDir, "included.txt"), "x", UTF_8); + write(new File(srcDir, "not-included.doc"), "x", UTF_8); + write(new File(srcDir, "~excluded.txt"), "x", UTF_8); + + ToUpperCaseConverter converter = new ToUpperCaseConverter(); + converter.setSource(srcDir); + converter.setDestinationDirectory(dstDir); + converter.setInclude(Pattern.compile("(?i).*\\.txt")); + converter.setExclude(Pattern.compile("(.*/)?~[^/]*")); + converter.execute(); + + assertTrue(new File(dstDir, "included.txt.uc").exists()); + assertFalse(new File(dstDir, "not-included.doc.uc").exists()); + assertFalse(new File(dstDir, "~excluded.txt.uc").exists()); + } + + @Test + public void testIncludeAndExclude2() throws IOException, ConverterException { + write(new File(srcDir, "included.txt"), "x", UTF_8); + write(new File(srcDir, "included2.doc"), "x", UTF_8); + + ToUpperCaseConverter converter = new ToUpperCaseConverter(); + converter.setSource(srcDir); + converter.setDestinationDirectory(dstDir); + converter.setInclude(null); + converter.execute(); + + assertTrue(new File(dstDir, "included.txt.uc").exists()); + assertTrue(new File(dstDir, "included2.doc.uc").exists()); + } + public static class ToUpperCaseConverter extends Converter { @Override + protected Pattern getDefaultInclude() { + return Pattern.compile("(?i).*\\.txt"); + } + + @Override protected void convertFile(FileConversionContext ctx) throws ConverterException, IOException { String content = IOUtils.toString(ctx.getSourceStream(), StandardCharsets.UTF_8); ctx.setDestinationFileName(ctx.getSourceFileName() + ".uc");
