details: https://code.openbravo.com/erp/devel/pi/rev/f8c3d0a4f7a4
changeset: 32608:f8c3d0a4f7a4
user: Asier Lostalé <asier.lostale <at> openbravo.com>
date: Fri Aug 11 16:21:42 2017 +0200
summary: fixed bug 36755: inefficient translate task
Several improvements applied:
* A single invocation is performed instead one per file extension.
* Look for labels just once regardless number of installed translations.
* Cache all labels in memory so that a single query is performed to look for
them up instead of one per each label found in files.
* Execute all DB updates in a single transaction
* Reduced log vebosity (specially when no modules in development)
diffstat:
build.xml | 4 -
src-trl/src/org/openbravo/translate/Translate.java | 752 +++++----------
src-trl/src/org/openbravo/translate/Translate_data.xsql | 187 +---
src/build.xml | 55 +-
4 files changed, 296 insertions(+), 702 deletions(-)
diffs (truncated from 1288 to 300 lines):
diff -r b9227826b5dc -r f8c3d0a4f7a4 build.xml
--- a/build.xml Wed Aug 30 14:10:12 2017 +0200
+++ b/build.xml Fri Aug 11 16:21:42 2017 +0200
@@ -712,10 +712,6 @@
<ant dir="${base.src}" target="translate" inheritAll="true"
inheritRefs="true" />
</target>
- <target name="translate.modules" depends="init">
- <ant dir="${base.src}" target="translate.modules" inheritAll="true"
inheritRefs="true" />
- </target>
-
<target name="installWebService" depends="init">
<ant dir="${base.src}" target="installWebService" inheritAll="true"
inheritRefs="true">
<property name="wsdd" value="1" />
diff -r b9227826b5dc -r f8c3d0a4f7a4
src-trl/src/org/openbravo/translate/Translate.java
--- a/src-trl/src/org/openbravo/translate/Translate.java Wed Aug 30
14:10:12 2017 +0200
+++ b/src-trl/src/org/openbravo/translate/Translate.java Fri Aug 11
16:21:42 2017 +0200
@@ -18,13 +18,27 @@
*/
package org.openbravo.translate;
-import java.io.File;
-import java.io.FileInputStream;
import java.io.IOException;
+import java.io.InputStream;
import java.io.InputStreamReader;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.FileSystems;
+import java.nio.file.FileVisitResult;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.PathMatcher;
+import java.nio.file.Paths;
+import java.nio.file.SimpleFileVisitor;
+import java.nio.file.attribute.BasicFileAttributes;
+import java.sql.Connection;
+import java.sql.SQLException;
import java.util.ArrayList;
-import java.util.StringTokenizer;
-import java.util.Vector;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.regex.Pattern;
import javax.servlet.ServletException;
@@ -33,74 +47,63 @@
import org.apache.xerces.parsers.SAXParser;
import org.openbravo.database.CPStandAlone;
import org.openbravo.database.SessionInfo;
-import org.openbravo.utils.DirFilter;
+import org.openbravo.exception.NoConnectionAvailableException;
import org.xml.sax.Attributes;
import org.xml.sax.InputSource;
import org.xml.sax.XMLReader;
-import org.xml.sax.ext.LexicalHandler;
import org.xml.sax.helpers.DefaultHandler;
/**
+ * Looks for translatable elements in html, fo, srpt and jrxml files inserting
them in
+ * ad_textinterfaces table.
+ *
* @author Fernando Iriazabal
- *
- * Translate the HTML file of the folder especified
**/
-public class Translate extends DefaultHandler implements LexicalHandler {
- protected static CPStandAlone pool;
- static XMLReader parser;
- static TranslateData[] toLanguage;
- static String actualLanguage;
- static String fileTermination;
+public class Translate extends DefaultHandler {
+ private static CPStandAlone pool;
+ private static final Pattern LETTER_PATTERN = Pattern.compile("[a-zA-Z]");
+ private static final List<String> translatableExtensions =
Arrays.asList("html", "fo", "srpt",
+ "jrxml");
+ private static final Logger log = Logger.getLogger(Translate.class);
- static boolean isHtml = false;
+ private XMLReader parser;
+ private String extension;
- static String actualTag;
- static String actualFile;
- static String actualPrefix;
- static StringBuffer translationText;
- static int count = 0;
- static ArrayList<String> moduleDirectories;
- static String moduleName = "";
- static String moduleLang = "";
- static String moduleID = "";
- static boolean translateModule = true;
- static final String[] tokens = { "-", ":" };
+ private String actualTag;
+ private String actualFile;
+ private String actualPrefix;
+ private StringBuilder translationText;
+ private int count = 0;
- static Logger log4j = Logger.getLogger(Translate.class);
+ private String moduleLang = "";
+ private String moduleID = "";
+ private Path moduleBasePath;
+ private Connection conn;
+ private boolean canTranslateModule = true;
- /**
- * Constructor
- *
- * @param xmlPoolFile
- * Path to the Openbravo.properties file.
- * @throws ServletException
- */
- public Translate(String xmlPoolFile) throws ServletException {
+ private static Map<String, List<TranslateData>> allLabels;
+ private static HashSet<String> modsInDev = new HashSet<>();
+
+ private static void init(String xmlPoolFile) {
pool = new CPStandAlone(xmlPoolFile);
}
+ private Translate() {
+ }
+
/**
- * Constructor
- *
- * @param xmlPoolFile
- * Path to the Openbravo.properties file.
* @param _fileTermination
* File extension to filter.
- * @throws ServletException
*/
- public Translate(String xmlPoolFile, String _fileTermination) throws
ServletException {
- this(xmlPoolFile);
- fileTermination = _fileTermination;
- isHtml = fileTermination.toLowerCase().endsWith("html");
+ public Translate(String _fileTermination) throws ServletException {
+ extension = _fileTermination;
+ boolean isHtml = extension.toLowerCase().endsWith("html");
if (isHtml)
parser = new org.cyberneko.html.parsers.SAXParser();
else
parser = new SAXParser();
parser.setEntityResolver(new LocalEntityResolver());
parser.setContentHandler(this);
- toLanguage = TranslateData.systemLanguage(pool);
- if (toLanguage.length == 0)
- log4j.warn("No system languages defined, translation will parse all
files.");
}
/**
@@ -110,66 +113,71 @@
* List of arguments. There is 2 call ways, with 2 arguments; the
first one is the
* attribute to indicate if the AD_TEXTINTERFACES must be cleaned
("clean") and the
* second one is the Openbravo.properties path. The other way is
with more arguments,
- * where: 0- Openbravo.properties path. 1- File extension. 2- Path
where are the files to
- * translate. 3- Relative path.
- * @throws Exception
+ * where: 0- Openbravo.properties path. 1- Path where are the files
to translate.
*/
public static void main(String argv[]) throws Exception {
PropertyConfigurator.configure("log4j.lcf");
- String dirIni;
- boolean boolFilter;
- DirFilter dirFilter = null;
- String relativePath = "";
- if ((argv.length == 2)) {
- if (argv[0].equals("clean")) {
- log4j.debug("clean AD_TEXTINTERFACES");
- final Translate translate = new Translate(argv[1]);
- translate.clean();
- return;
- } else if (argv[0].equals("remove")) {
- log4j.debug("remove AD_TEXTINTERFACES");
- final Translate translate = new Translate(argv[1]);
- translate.remove();
- return;
+ if (argv.length != 2) {
+ log.error("Usage: Translate Openbravo.properties
[clean|remove|sourceDir]");
+ log.error("Received: " + Arrays.asList(argv));
+ return;
+ }
+
+ init(argv[0]);
+ Translate translate;
+ switch (argv[1]) {
+ case "clean":
+ log.debug("clean AD_TEXTINTERFACES");
+ translate = new Translate();
+ translate.clean();
+ return;
+ case "remove":
+ log.debug("remove AD_TEXTINTERFACES");
+ translate = new Translate();
+ translate.remove();
+ return;
+ }
+
+ Path obPath = Paths.get(argv[1]).normalize();
+
+ TranslateData[] mods = TranslateData.getModulesInDevelopment(pool);
+ for (TranslateData mod : mods) {
+ modsInDev.add(mod.id);
+ }
+ for (TranslateData mod : mods) {
+ Path path;
+ if ("0".equals(mod.id)) {
+ path = obPath.resolve("src");
+ } else {
+ path = obPath.resolve(Paths.get("modules", mod.javapackage, "src"));
+ if (!Files.exists(path)) {
+ continue;
+ }
+ }
+ log.info("Looking for translatable elements in " + mod.javapackage + "
(" + mod.name + ")");
+
+ for (String extension : translatableExtensions) {
+ translate = new Translate(extension);
+ translate.moduleLang = mod.lang;
+ translate.moduleID = mod.id;
+ translate.moduleBasePath = path;
+
+ translate.execute();
+ if (translate.count > 0) {
+ log.info(" parsed " + translate.count + " " + extension + " files");
+ }
+
+ if (!translate.canTranslateModule) {
+ break;
+ }
}
}
- if (argv.length < 3) {
- log4j.error("Usage: java Translate Openbravo.properties fileTermination
sourceDir");
- return;
+ if (mods.length == 0) {
+ log.info("No modules in development to look for translatable elements.");
}
-
- final Translate translate = new Translate(argv[0], argv[1]);
-
- dirIni = argv[2].replace("\\", "/");
-
- if (argv.length > 3) {
- moduleDirectories = getDirectories(argv[3]);
- log4j.info("Translation for modules");
- }
- boolFilter = true;
- dirFilter = new DirFilter(fileTermination);
- log4j.info("directory source: " + dirIni);
- log4j.info("file termination: " + fileTermination);
-
- final File path = new File(dirIni, relativePath);
- if (!path.exists()) {
- log4j.error("Can't find directory: " + dirIni);
- translate.destroy();
- return;
- }
- if (moduleDirectories == null) {
- if (TranslateData.isInDevelopmentModule(pool, "0")) {
- listDir(path, boolFilter, dirFilter, relativePath, true, "", 0, "");
- log4j.info("Translated files for " + fileTermination + ": " + count);
- } else
- log4j.info("Core is not in development: skipping it");
- } else {
- listDir(path, boolFilter, dirFilter, relativePath, false, "", 0, "");
- log4j.info("Translated files for " + fileTermination + ": " + count);
- }
- translate.destroy();
+ destroy();
}
/**
@@ -179,115 +187,52 @@
try {
TranslateData.clean(pool);
} catch (final Exception e) {
- log4j.error("clean error", e);
+ log.error("clean error", e);
}
}
private void remove() {
try {
- TranslateData.remove(pool);
+ int n = TranslateData.remove(pool);
+ if (n > 0) {
+ log.info("Removed " + n + " unused elements");
+ }
} catch (final Exception e) {
- log4j.error("remove error", e);
+ log.error("remove error", e);
}
------------------------------------------------------------------------------
Check out the vibrant tech community on one of the world's most
engaging tech sites, Slashdot.org! http://sdm.link/slashdot
_______________________________________________
Openbravo-commits mailing list
[email protected]
https://lists.sourceforge.net/lists/listinfo/openbravo-commits
