This is an automated email from the ASF dual-hosted git repository.
nmalin pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/ofbiz-framework.git
The following commit(s) were added to refs/heads/trunk by this push:
new adefd159d2 Improved: Implemented support to use limit and offset
properly to fetch only limited records instead of fetching all the records from
the DB (OFBIZ-13366)
adefd159d2 is described below
commit adefd159d237e7aee2818931553365f32230dda2
Author: pushpraj.jadoun <[email protected]>
AuthorDate: Wed Feb 25 12:57:46 2026 +0530
Improved: Implemented support to use limit and offset properly to fetch
only limited records instead of fetching all the records from the DB
(OFBIZ-13366)
Currently, the FindGeneric.groovy script is responsible for retrieving data
for entity views. However, its existing data-fetching approach becomes
inefficient when working with large datasets.
When entities contain millions of records, the current pagination mechanism
may load a significantly larger number of records into memory than required. In
some cases, the system retrieves most—or all—records before applying pagination
logic in memory.
For example, when a user requests to view only the last 20 records, the
system may still load a much larger dataset into memory before filtering the
results for display. This leads to unnecessary memory consumption and
performance degradation.
To solve it we refactor the data retrieval logic in the following script:
framework/webtools/groovyScripts/entity/FindGeneric.groovy
The refactoring will be used explicitly EntityFindOptions with proper limit
and offset values. By pushing pagination logic down to the database layer, only
the records required for the current view will be fetched.
This offer :
* Reduced Memory Usage
Only the records required for the current page are loaded into memory.
* Improved Performance
Faster response times when working with large entities, as pagination
is handled directly by the database.
* Improved Scalability
Enables Webtools to efficiently manage entities containing millions of
records without excessive resource consumption or risk of memory issues.
---
.../ofbiz/webtools/entity/FindGeneric.groovy | 101 ++++++++++-----------
1 file changed, 49 insertions(+), 52 deletions(-)
diff --git
a/framework/webtools/src/main/groovy/org/apache/ofbiz/webtools/entity/FindGeneric.groovy
b/framework/webtools/src/main/groovy/org/apache/ofbiz/webtools/entity/FindGeneric.groovy
index 4d6535a4b8..3479e6fa2e 100644
---
a/framework/webtools/src/main/groovy/org/apache/ofbiz/webtools/entity/FindGeneric.groovy
+++
b/framework/webtools/src/main/groovy/org/apache/ofbiz/webtools/entity/FindGeneric.groovy
@@ -24,7 +24,7 @@ import org.apache.ofbiz.entity.model.ModelEntity
import org.apache.ofbiz.entity.model.ModelFieldType
import org.apache.ofbiz.entity.model.ModelReader
import org.apache.ofbiz.entity.model.ModelViewEntity
-import org.apache.ofbiz.entity.model.ModelViewEntity.ModelAlias
+import org.apache.ofbiz.entity.util.EntityQuery
import org.apache.ofbiz.widget.model.FormFactory
import org.apache.ofbiz.widget.model.ModelForm
import org.apache.ofbiz.widget.renderer.FormRenderer
@@ -44,9 +44,9 @@ try {
}
if (modelEntity) {
- List<String> fieldsToSelect = getFieldsToSelect(modelEntity)
+ Set<String> fieldsToSelect = getFieldsToSelect(modelEntity)
- entityName = modelEntity.entityName
+ String entityName = modelEntity.entityName
context.entityName = entityName
ModelReader entityModelReader = delegator.getModelReader()
//create the search form with auto-fields-entity
@@ -64,19 +64,19 @@ if (modelEntity) {
//call modelEntity to complete information on the field type
modelEntity.getFieldsUnmodifiable().each {
modelField ->
- if (! modelEntity.getAutomaticFieldNames().contains(modelField.name)) {
- ModelFieldType type = delegator.getEntityFieldType(modelEntity,
modelField.getType())
- dynamicAutoEntityFieldSearchForm +=
- "<field name=\"${modelField.name}\"
tooltip=\"${modelField.getName()}" +
- (modelField.getIsPk() ? '* ' : ' ') +
- " / ${modelField.getType()} (${type.getJavaType()} -
${type.getSqlType()})\">"
+ if
(!modelEntity.getAutomaticFieldNames().contains(modelField.name)) {
+ ModelFieldType type =
delegator.getEntityFieldType(modelEntity, modelField.getType())
+ dynamicAutoEntityFieldSearchForm +=
+ "<field name=\"${modelField.name}\"
tooltip=\"${modelField.getName()}" +
+ (modelField.getIsPk() ? '* ' : ' ') +
+ " / ${modelField.getType()}
(${type.getJavaType()} - ${type.getSqlType()})\">"
- //In general when your research some entity on the pk field, you
check on element, so help by set as default equals comparison
- if (modelField.getIsPk() && type.getJavaType() == 'String') {
- dynamicAutoEntityFieldSearchForm += '<text-find
default-option="equals"/>'
+ //In general when your research some entity on the pk field,
you check on element, so help by set as default equals comparison
+ if (modelField.getIsPk() && type.getJavaType() == 'String') {
+ dynamicAutoEntityFieldSearchForm += '<text-find
default-option="equals"/>'
+ }
+ dynamicAutoEntityFieldSearchForm += '</field>'
}
- dynamicAutoEntityFieldSearchForm += '</field>'
- }
}
dynamicAutoEntityFieldSearchForm = dynamicAutoEntityFieldSearchForm +
'</form></forms>'
logVerbose(dynamicAutoEntityFieldSearchForm)
@@ -96,31 +96,45 @@ if (modelEntity) {
dynamicAutoEntitySearchFormRenderer.render(writer, context)
context.dynamicAutoEntitySearchForm = writer
- //prepare the result list from performFind
+ // Prepare the data retrieval using EntityQuery with limit and offset for
better performance
+ int viewIndex = [parameters.VIEW_INDEX_1, parameters.VIEW_INDEX,
parameters.viewIndex].find { String val -> val?.isInteger() }?.toInteger() ?: 0
+ int viewSize = [parameters.VIEW_SIZE_1, parameters.VIEW_SIZE,
parameters.viewSize].find { String val -> val?.isInteger() }?.toInteger() ?: 20
+
+ Map prepareFindResult = dispatcher.runSync('prepareFind', [
+ entityName: entityName,
+ inputFields: parameters,
+ orderBy: parameters.sortField,
+ noConditionFind: parameters.noConditionFind ?: 'Y'
+ ])
+
+ if (prepareFindResult.entityConditionList || parameters.noConditionFind ==
'Y') {
+ context.listIt = EntityQuery.use(delegator)
+ .from(entityName)
+ .where(prepareFindResult.entityConditionList ?: [:])
+ .orderBy(prepareFindResult.orderByList ?:
modelEntity.getPkFieldNames())
+ .select(fieldsToSelect)
+ .limit(viewSize)
+ .offset(viewIndex * viewSize)
+ .distinct(parameters.distinct == 'Y')
+ .cursorScrollInsensitive()
+ .queryIterator()
+ context.listSize = context.listIt.getResultsSizeAfterPartialList()
+ }
+
+ //prepare the result list
String dynamicAutoEntityFieldListForm = """<?xml version="1.0"
encoding="UTF-8"?>
<forms xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";
xmlns="http://ofbiz.apache.org/Widget-Form";
xsi:schemaLocation="http://ofbiz.apache.org/Widget-Form
http://ofbiz.apache.org/dtds/widget-form.xsd";>
<form name="ListGeneric" type="list"
target="entity/find/${entityName}" list-name="listIt"
paginate-target="entity/find/${entityName}"
+ override-list-size="${context.listSize ?: 0}"
odd-row-style="alternate-row" default-table-style="basic-table
light-grid hover-bar" header-row-style="header-row-2">
- <actions>
- <service service-name="performFind">
- <field-map field-name="inputFields"
from-field="parameters"/>
- <field-map field-name="entityName"
value="${entityName}"/>"""
- if (fieldsToSelect) {
- dynamicAutoEntityFieldListForm += """
- <field-map field-name="fieldList"
value="${fieldsToSelect}"/>"""
- }
- dynamicAutoEntityFieldListForm += """
- <field-map field-name="orderBy"
from-field="parameters.sortField"/>
- </service>
- </actions>
<auto-fields-entity entity-name="${entityName}"
default-field-type="display" include-internal="true"/>
<field name="_method"><hidden value="POST"/></field>
<field name="entityName"><hidden value="${entityName}"/></field>"""
modelEntity.getFieldsUnmodifiable().each {
modelField ->
- dynamicAutoEntityFieldListForm +=
+ dynamicAutoEntityFieldListForm +=
"<field name=\"${modelField.name}\" sort-field=\"true\"/>"
}
dynamicAutoEntityFieldListForm += """
@@ -145,30 +159,13 @@ if (modelEntity) {
context.dynamicAutoEntityListForm = writerList
}
-List<String> getFieldsToSelect(ModelEntity modelEntity) {
- groupByFields = []
- functionFields = []
-
+static Set<String> getFieldsToSelect(ModelEntity modelEntity) {
if (modelEntity instanceof ModelViewEntity) {
- aliases = modelEntity.getAliasesCopy()
- for (ModelAlias alias : aliases) {
- if (alias.getGroupBy()) {
- groupByFields.add(alias.getName())
- } else if (alias.getFunction()) {
- functionFields.add(alias.getName())
- }
- }
- }
- List<String> fieldsToSelect = []
-
- if (groupByFields || functionFields) {
- for (String groupByField : groupByFields) {
- fieldsToSelect.add(groupByField)
- }
-
- for (String functionField : functionFields) {
- fieldsToSelect.add(functionField)
- }
+ List groupByFields = modelEntity.getAliasesCopy()
+ .find { it.getGroupBy() }*.getName()
+ List functionFields = modelEntity.getAliasesCopy()
+ .find { it.getFunction() }*.getName()
+ return [*groupByFields, *functionFields] as Set
}
- return fieldsToSelect
+ return [] as Set
}
