This is an automated email from the ASF dual-hosted git repository.
jkevan pushed a commit to branch async-save-export-config
in repository https://gitbox.apache.org/repos/asf/unomi.git
The following commit(s) were added to refs/heads/async-save-export-config by
this push:
new ec1c0a15c Perf improvements for oneshot exports
ec1c0a15c is described below
commit ec1c0a15cd5ff24c55f05848141d83ec1b3b8767
Author: Kevan <[email protected]>
AuthorDate: Thu Mar 30 17:04:15 2023 +0200
Perf improvements for oneshot exports
---
.../router/services/ProfileExportServiceImpl.java | 36 ++++++++++++++++++----
.../resources/OSGI-INF/blueprint/blueprint.xml | 2 ++
2 files changed, 32 insertions(+), 6 deletions(-)
diff --git
a/extensions/router/router-service/src/main/java/org/apache/unomi/router/services/ProfileExportServiceImpl.java
b/extensions/router/router-service/src/main/java/org/apache/unomi/router/services/ProfileExportServiceImpl.java
index 8b9471b54..c3197b61e 100644
---
a/extensions/router/router-service/src/main/java/org/apache/unomi/router/services/ProfileExportServiceImpl.java
+++
b/extensions/router/router-service/src/main/java/org/apache/unomi/router/services/ProfileExportServiceImpl.java
@@ -18,9 +18,12 @@ package org.apache.unomi.router.services;
import org.apache.commons.lang3.BooleanUtils;
import org.apache.commons.lang3.StringUtils;
+import org.apache.unomi.api.PartialList;
import org.apache.unomi.api.Profile;
import org.apache.unomi.api.PropertyType;
+import org.apache.unomi.api.conditions.Condition;
import org.apache.unomi.api.services.ConfigSharingService;
+import org.apache.unomi.api.services.DefinitionsService;
import org.apache.unomi.persistence.spi.PersistenceService;
import org.apache.unomi.router.api.ExportConfiguration;
import org.apache.unomi.router.api.RouterConstants;
@@ -40,28 +43,45 @@ public class ProfileExportServiceImpl implements
ProfileExportService {
private PersistenceService persistenceService;
+ private DefinitionsService definitionsService;
private ConfigSharingService configSharingService;
public void setPersistenceService(PersistenceService persistenceService) {
this.persistenceService = persistenceService;
}
+ public void setDefinitionsService(DefinitionsService definitionsService) {
+ this.definitionsService = definitionsService;
+ }
+
public void setConfigSharingService(ConfigSharingService
configSharingService) {
this.configSharingService = configSharingService;
}
public String extractProfilesBySegment(ExportConfiguration
exportConfiguration) {
- List<Profile> profileList = persistenceService.query("segments",
(String) exportConfiguration.getProperty("segment"), null, Profile.class);
+ Collection<PropertyType> propertiesDef =
persistenceService.query("target", "profiles", null, PropertyType.class);
+
+ Condition segmentCondition = new Condition();
+
segmentCondition.setConditionType(definitionsService.getConditionType("profileSegmentCondition"));
+ segmentCondition.setParameter("segments",
Collections.singletonList((String) exportConfiguration.getProperty("segment")));
+ segmentCondition.setParameter("matchType", "in");
+
StringBuilder csvContent = new StringBuilder();
- for (Profile profile : profileList) {
- csvContent.append(convertProfileToCSVLine(profile,
exportConfiguration));
-
csvContent.append(RouterUtils.getCharFromLineSeparator(exportConfiguration.getLineSeparator()));
+ PartialList<Profile> profiles =
persistenceService.query(segmentCondition, null, Profile.class, 0, 1000, "10m");
+ int counter = 0;
+ while (profiles != null && profiles.getList().size() > 0) {
+ List<Profile> scrolledProfiles = profiles.getList();
+ for (Profile profile : scrolledProfiles) {
+ csvContent.append(convertProfileToCSVLine(profile,
exportConfiguration, propertiesDef));
+
csvContent.append(RouterUtils.getCharFromLineSeparator(exportConfiguration.getLineSeparator()));
+ }
+ counter += scrolledProfiles.size();
+ profiles = persistenceService.continueScrollQuery(Profile.class,
profiles.getScrollIdentifier(), profiles.getScrollTimeValidity());
}
- logger.debug("Exporting {} extracted profiles.", profileList.size());
Map execution = new HashMap();
execution.put(RouterConstants.KEY_EXECS_DATE, new Date().getTime());
- execution.put(RouterConstants.KEY_EXECS_EXTRACTED, profileList.size());
+ execution.put(RouterConstants.KEY_EXECS_EXTRACTED, counter);
exportConfiguration = (ExportConfiguration)
RouterUtils.addExecutionEntry(exportConfiguration, execution,
Integer.parseInt((String)
configSharingService.getProperty(RouterConstants.KEY_HISTORY_SIZE)));
persistenceService.save(exportConfiguration);
@@ -72,6 +92,10 @@ public class ProfileExportServiceImpl implements
ProfileExportService {
public String convertProfileToCSVLine(Profile profile, ExportConfiguration
exportConfiguration) {
// TODO: UNOMI-759 querying this everytimes
Collection<PropertyType> propertiesDef =
persistenceService.query("target", "profiles", null, PropertyType.class);
+ return convertProfileToCSVLine(profile, exportConfiguration,
propertiesDef);
+ }
+
+ public String convertProfileToCSVLine(Profile profile, ExportConfiguration
exportConfiguration, Collection<PropertyType> propertiesDef) {
Map<String, String> mapping = (Map<String, String>)
exportConfiguration.getProperty("mapping");
String lineToWrite = "";
for (int i = 0; i < mapping.size(); i++) {
diff --git
a/extensions/router/router-service/src/main/resources/OSGI-INF/blueprint/blueprint.xml
b/extensions/router/router-service/src/main/resources/OSGI-INF/blueprint/blueprint.xml
index 74e67907a..8b187cad6 100644
---
a/extensions/router/router-service/src/main/resources/OSGI-INF/blueprint/blueprint.xml
+++
b/extensions/router/router-service/src/main/resources/OSGI-INF/blueprint/blueprint.xml
@@ -22,6 +22,7 @@
<reference id="persistenceService"
interface="org.apache.unomi.persistence.spi.PersistenceService"/>
<reference id="configSharingService"
interface="org.apache.unomi.api.services.ConfigSharingService"/>
+ <reference id="definitionsService"
interface="org.apache.unomi.api.services.DefinitionsService"/>
<bean id="importConfigurationServiceImpl"
class="org.apache.unomi.router.services.ImportConfigurationServiceImpl">
<property name="persistenceService" ref="persistenceService"/>
@@ -59,6 +60,7 @@
<bean id="profileExportServiceImpl"
class="org.apache.unomi.router.services.ProfileExportServiceImpl">
<property name="persistenceService" ref="persistenceService"/>
<property name="configSharingService" ref="configSharingService" />
+ <property name="definitionsService" ref="definitionsService" />
</bean>
<service id="profileExportService" ref="profileExportServiceImpl">
<interfaces>
