details: https://code.openbravo.com/erp/devel/pi/rev/694b153db283
changeset: 26105:694b153db283
user: Alvaro Ferraz <alvaro.ferraz <at> openbravo.com>
date: Fri Feb 20 09:46:32 2015 +0100
summary: Fixes issue 28817: OutOfMemory importing bank statement using the
csv importer
ScrollableResults object, used to scrolls OBQuery of business partners when
importing bank statements, was directly closed to release resources, between
each line's iteration of the csv file. This change avoids the memory overusing,
so now it is possible to import a csv file with 4000 lines or more. Also,
OBCriteria has been changed into Query in order to use arryas instead of DAL
objects what reduces the memory usage.
diffstat:
modules/org.openbravo.advpaymentmngt/src/org/openbravo/advpaymentmngt/utility/FIN_BankStatementImport.java
| 139 ++++++---
1 files changed, 91 insertions(+), 48 deletions(-)
diffs (197 lines):
diff -r f1360ce4b3d1 -r 694b153db283
modules/org.openbravo.advpaymentmngt/src/org/openbravo/advpaymentmngt/utility/FIN_BankStatementImport.java
---
a/modules/org.openbravo.advpaymentmngt/src/org/openbravo/advpaymentmngt/utility/FIN_BankStatementImport.java
Mon Mar 02 13:45:56 2015 +0100
+++
b/modules/org.openbravo.advpaymentmngt/src/org/openbravo/advpaymentmngt/utility/FIN_BankStatementImport.java
Fri Feb 20 09:46:32 2015 +0100
@@ -11,7 +11,7 @@
* under the License.
* The Original Code is Openbravo ERP.
* The Initial Developer of the Original Code is Openbravo SLU
- * All portions are Copyright (C) 2010-2013 Openbravo SLU
+ * All portions are Copyright (C) 2010-2015 Openbravo SLU
* All Rights Reserved.
* Contributor(s): ______________________________________.
*************************************************************************
@@ -31,6 +31,7 @@
import org.apache.commons.fileupload.FileItem;
import org.apache.commons.lang.StringUtils;
import org.apache.log4j.Logger;
+import org.hibernate.Query;
import org.hibernate.ScrollMode;
import org.hibernate.ScrollableResults;
import org.hibernate.criterion.Restrictions;
@@ -257,6 +258,7 @@
} else {
OBDal.getInstance().remove(bankStatementLine);
}
+
}
OBDal.getInstance().flush();
return counter;
@@ -360,6 +362,7 @@
final OBQuery<BusinessPartner> bp =
OBDal.getInstance().createQuery(BusinessPartner.class,
whereClause.toString(), parameters);
bp.setFilterOnReadableOrganization(false);
+ bp.setMaxResult(1);
List<BusinessPartner> matchedBP = bp.list();
if (matchedBP.size() == 0)
return null;
@@ -396,73 +399,113 @@
return null;
}
final StringBuilder whereClause = new StringBuilder();
- List<Object> parameters = new ArrayList<Object>();
OBContext.setAdminMode();
+ ScrollableResults businessPartnersScroll = null;
try {
- whereClause.append(" as b ");
+ whereClause.append("select b.id as id, b.name as name from ");
+ whereClause.append(" BusinessPartner b ");
whereClause.append(" where (");
for (String token : list) {
- whereClause.append(" lower(b." + BusinessPartner.PROPERTY_NAME + ")
like lower(?) or ");
- parameters.add("%" + token + "%");
+ whereClause.append(" lower(b." + BusinessPartner.PROPERTY_NAME + ")
like lower('%" + token
+ + "%') or ");
}
whereClause.delete(whereClause.length() - 3,
whereClause.length()).append(")");
whereClause.append(" and b." + BusinessPartner.PROPERTY_ORGANIZATION +
".id in (");
whereClause.append(Utility.getInStrSet(new
OrganizationStructureProvider()
.getNaturalTree(organization.getId())) + ") ");
- final OBQuery<BusinessPartner> bl =
OBDal.getInstance().createQuery(BusinessPartner.class,
- whereClause.toString(), parameters);
- bl.setFilterOnReadableOrganization(false);
+ final Query bl =
OBDal.getInstance().getSession().createQuery(whereClause.toString());
+ businessPartnersScroll = bl.scroll(ScrollMode.SCROLL_SENSITIVE);
- final ScrollableResults businessPartnersScroll =
bl.scroll(ScrollMode.SCROLL_SENSITIVE);
- int countBp = bl.count();
- if (countBp == 0) {
+ if (!businessPartnersScroll.next()) {
return null;
- } else if (countBp == 1) {
- businessPartnersScroll.next();
- return (BusinessPartner) businessPartnersScroll.get(0);
- } else {
- BusinessPartner closest = closest(businessPartnersScroll, partnername);
+ }
- return closest;
+ else {
+ final Object[] resultObject = (Object[]) businessPartnersScroll.get(0);
+ if (!businessPartnersScroll.next()) {
+ String strParnterId = "";
+ if (resultObject.getClass().isArray()) {
+ final Object[] values = (Object[]) resultObject;
+ strParnterId = (String) values[0];
+ }
+ BusinessPartner bp = OBDal.getInstance().get(BusinessPartner.class,
strParnterId);
+ return bp;
+ }
+
+ else {
+ String closestId = closest(businessPartnersScroll, partnername);
+ BusinessPartner closest =
OBDal.getInstance().get(BusinessPartner.class, closestId);
+ return closest;
+ }
}
} finally {
+ businessPartnersScroll.close();
OBContext.restorePreviousMode();
}
}
- private BusinessPartner closest(ScrollableResults businessPartners, String
partnername) {
- businessPartners.next();
- BusinessPartner targetBusinessPartner = (BusinessPartner)
businessPartners.get(0);
- businessPartners.beforeFirst();
- int distance = StringUtils.getLevenshteinDistance(partnername,
targetBusinessPartner.getName());
- String parsedPartnername = partnername.toLowerCase();
- // Remove exceptions
- for (String eliminate : stringExceptions) {
- parsedPartnername =
parsedPartnername.replaceAll(eliminate.toLowerCase(), "");
+ private String closest(ScrollableResults businessPartners, String
partnername) {
+ String targetBusinessPartnerId = "";
+ try {
+ businessPartners.beforeFirst();
+ businessPartners.next();
+ Object[] resultObject = (Object[]) businessPartners.get(0);
+
+ String targetBusinessPartnerName = "";
+ if (resultObject.getClass().isArray()) {
+ final Object[] values = (Object[]) resultObject;
+ targetBusinessPartnerId = (String) values[0];
+ targetBusinessPartnerName = (String) values[1];
+ }
+
+ int distance = StringUtils.getLevenshteinDistance(partnername,
targetBusinessPartnerName);
+ String parsedPartnername = partnername.toLowerCase();
+ // Remove exceptions
+ for (String eliminate : stringExceptions) {
+ parsedPartnername =
parsedPartnername.replaceAll(eliminate.toLowerCase(), "");
+ }
+
+ // Remove Numeric characters
+ char[] digits = { '0', '1', '2', '3', '4', '5', '6', '7', '8', '9' };
+ for (char character : digits) {
+ parsedPartnername = parsedPartnername.replace(character, ' ');
+ parsedPartnername = parsedPartnername.trim();
+ }
+
+ businessPartners.beforeFirst();
+ int i = 0;
+ while (businessPartners.next()) {
+ i++;
+ String bpId = "";
+ String bpName = "";
+ resultObject = (Object[]) businessPartners.get(0);
+ if (resultObject.getClass().isArray()) {
+ final Object[] values = (Object[]) resultObject;
+ bpId = (String) values[0];
+ bpName = (String) values[1];
+ }
+ // Calculates distance between two strings meaning number of changes
required for a string
+ // to
+ // convert in another string
+ int bpDistance = StringUtils
+ .getLevenshteinDistance(parsedPartnername, bpName.toLowerCase());
+ if (bpDistance < distance) {
+ distance = bpDistance;
+ targetBusinessPartnerId = bpId;
+ }
+ if (i % 100 == 0) {
+ OBDal.getInstance().flush();
+ OBDal.getInstance().getSession().clear();
+ }
+ }
+ return targetBusinessPartnerId;
+ } catch (Exception e) {
+ log4j.error(e.getStackTrace());
+ } finally {
+ return targetBusinessPartnerId;
}
- // Remove Numeric characters
- char[] digits = { '0', '1', '2', '3', '4', '5', '6', '7', '8', '9' };
- for (char character : digits) {
- parsedPartnername = parsedPartnername.replace(character, ' ');
- parsedPartnername = parsedPartnername.trim();
- }
- while (businessPartners.next()) {
- BusinessPartner bp = (BusinessPartner) businessPartners.get(0);
- // Calculates distance between two strings meaning number of changes
required for a string to
- // convert in another string
- int bpDistance = StringUtils.getLevenshteinDistance(parsedPartnername,
bp.getName()
- .toLowerCase());
- if (bpDistance < distance) {
- distance = bpDistance;
- OBDal.getInstance().getSession().evict(targetBusinessPartner);
- targetBusinessPartner = bp;
- } else {
- OBDal.getInstance().getSession().evict(bp);
- }
- // The clear is not done because the evict is being done previously
- }
- return targetBusinessPartner;
+
}
private BankFileFormat getBankFileFormat() {
------------------------------------------------------------------------------
Dive into the World of Parallel Programming The Go Parallel Website, sponsored
by Intel and developed in partnership with Slashdot Media, is your hub for all
things parallel software development, from weekly thought leadership blogs to
news, videos, case studies, tutorials and more. Take a look and join the
conversation now. http://goparallel.sourceforge.net/
_______________________________________________
Openbravo-commits mailing list
[email protected]
https://lists.sourceforge.net/lists/listinfo/openbravo-commits
