This is an automated email from the ASF dual-hosted git repository. shuber pushed a commit to branch UNOMI-281-profile-lastupdated in repository https://gitbox.apache.org/repos/asf/unomi.git
commit 20c3f7c7b230b26d1ffbe024dcc373f481a454e2 Author: Serge Huber <[email protected]> AuthorDate: Fri Feb 28 13:59:55 2020 +0100 UNOMI-281 Add lastUpdated system property to Profiles - This patch adds the lastUpdated system property to profiles - Added as a system property to avoid any issues with migration. If it doesn't exist in existing installations it will simply be added the next time a profile is updated. - Any profile modification (including changing consents, adding new segments or scoring plans) will update the last update date on profiles - This patch also includes some various minor improvements to logging, that were used while developing this feature --- .../main/java/org/apache/unomi/api/Profile.java | 14 +++++ .../apache/unomi/services/UserListServiceImpl.java | 14 +++-- .../unomi/privacy/internal/PrivacyServiceImpl.java | 3 + .../ElasticSearchPersistenceServiceImpl.java | 14 ++--- .../unomi/persistence/spi/PersistenceService.java | 3 +- .../actions/MergeProfilesOnPropertyAction.java | 10 ++-- .../unomi/plugins/mail/actions/SendMailAction.java | 5 +- plugins/pom.xml | 1 + .../apache/unomi/services/impl/ParserHelper.java | 21 ++++--- .../services/impl/profiles/ProfileServiceImpl.java | 3 + .../services/impl/segments/SegmentServiceImpl.java | 65 ++++++++++++++++------ .../apache/unomi/shell/commands/ProfileList.java | 10 +++- 12 files changed, 116 insertions(+), 47 deletions(-) diff --git a/api/src/main/java/org/apache/unomi/api/Profile.java b/api/src/main/java/org/apache/unomi/api/Profile.java index 69c6c21..90a3a90 100644 --- a/api/src/main/java/org/apache/unomi/api/Profile.java +++ b/api/src/main/java/org/apache/unomi/api/Profile.java @@ -132,6 +132,20 @@ public class Profile extends Item { } /** + * Sets a system property, overwriting an existing one if it existed. This call will also created the system + * properties hash map if it didn't exist. + * @param key the key for the system property hash map + * @param value the value for the system property hash map + * @return the previous value object if it existing. + */ + public Object setSystemProperty(String key, Object value) { + if (this.systemProperties == null) { + this.systemProperties = new LinkedHashMap<>(); + } + return this.systemProperties.put(key, value); + } + + /** * {@inheritDoc} * * Note that Profiles are always in the shared system scope ({@link Metadata#SYSTEM_SCOPE}). diff --git a/extensions/lists-extension/services/src/main/java/org/apache/unomi/services/UserListServiceImpl.java b/extensions/lists-extension/services/src/main/java/org/apache/unomi/services/UserListServiceImpl.java index 237a52a..dc3bbc8 100644 --- a/extensions/lists-extension/services/src/main/java/org/apache/unomi/services/UserListServiceImpl.java +++ b/extensions/lists-extension/services/src/main/java/org/apache/unomi/services/UserListServiceImpl.java @@ -25,6 +25,7 @@ import org.apache.unomi.api.services.DefinitionsService; import org.apache.unomi.lists.UserList; import org.apache.unomi.persistence.spi.PersistenceService; +import java.util.Date; import java.util.LinkedList; import java.util.List; import java.util.Map; @@ -85,14 +86,15 @@ public class UserListServiceImpl implements UserListService { query.setParameter("propertyValue", listId); List<Profile> profiles = persistenceService.query(query, null, Profile.class); - Map<String, Object> profileProps; + Map<String, Object> profileSystemProperties; for (Profile p : profiles) { - profileProps = p.getSystemProperties(); - if(profileProps != null && profileProps.get("lists") != null) { - int index = ((List) profileProps.get("lists")).indexOf(listId); + profileSystemProperties = p.getSystemProperties(); + if(profileSystemProperties != null && profileSystemProperties.get("lists") != null) { + int index = ((List) profileSystemProperties.get("lists")).indexOf(listId); if(index != -1){ - ((List) profileProps.get("lists")).remove(index); - persistenceService.update(p.getItemId(), null, Profile.class, "systemProperties", profileProps); + ((List) profileSystemProperties.get("lists")).remove(index); + profileSystemProperties.put("lastUpdated", new Date()); + persistenceService.update(p.getItemId(), null, Profile.class, "systemProperties", profileSystemProperties); } } } diff --git a/extensions/privacy-extension/services/src/main/java/org/apache/unomi/privacy/internal/PrivacyServiceImpl.java b/extensions/privacy-extension/services/src/main/java/org/apache/unomi/privacy/internal/PrivacyServiceImpl.java index 247a7f4..d4db874 100644 --- a/extensions/privacy-extension/services/src/main/java/org/apache/unomi/privacy/internal/PrivacyServiceImpl.java +++ b/extensions/privacy-extension/services/src/main/java/org/apache/unomi/privacy/internal/PrivacyServiceImpl.java @@ -88,6 +88,9 @@ public class PrivacyServiceImpl implements PrivacyService { if (profile == null) { return false; } + Event profileDeletedEvent = new Event("profileDeleted", null, profile, null, null, profile, new Date()); + profileDeletedEvent.setPersistent(true); + eventService.send(profileDeletedEvent); // we simply overwrite the existing profile with an empty one. Profile emptyProfile = new Profile(profileId); profileService.save(emptyProfile); diff --git a/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java b/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java index 63ff129..b3fd181 100644 --- a/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java +++ b/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java @@ -785,16 +785,16 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService, logger.error("Failure : cause={} , message={}", failure.getCause(), failure.getMessage()); } } else { - logger.info("Update By Query has processed {} in {}.", response.getUpdated(), response.getTook().toString()); + logger.info("Update with query and script processed {} entries in {}.", response.getUpdated(), response.getTook().toString()); } if (response.isTimedOut()) { - logger.error("Update By Query ended with timeout!"); + logger.error("Update with query and script ended with timeout!"); } if (response.getVersionConflicts() > 0) { - logger.warn("Update By Query ended with {} Version Conflicts!", response.getVersionConflicts()); + logger.warn("Update with query and script ended with {} version conflicts!", response.getVersionConflicts()); } if (response.getNoops() > 0) { - logger.warn("Update By Query ended with {} noops!", response.getNoops()); + logger.warn("Update Bwith query and script ended with {} noops!", response.getNoops()); } } return true; @@ -803,8 +803,6 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService, } catch (ScriptException e) { logger.error("Error in the update script : {}\n{}\n{}", e.getScript(), e.getDetailedMessage(), e.getScriptStack()); throw new Exception("Error in the update script"); - } finally { - return false; } } }.catchingExecuteInClassLoader(true); @@ -1901,9 +1899,9 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService, public T catchingExecuteInClassLoader(boolean logError, Object... args) { try { return executeInClassLoader(timerName, args); - } catch (Exception e) { + } catch (Throwable t) { if (logError) { - logger.error("Error while executing in class loader", e); + logger.error("Error while executing in class loader", t); } } return null; diff --git a/persistence-spi/src/main/java/org/apache/unomi/persistence/spi/PersistenceService.java b/persistence-spi/src/main/java/org/apache/unomi/persistence/spi/PersistenceService.java index 0151b9c..7311353 100644 --- a/persistence-spi/src/main/java/org/apache/unomi/persistence/spi/PersistenceService.java +++ b/persistence-spi/src/main/java/org/apache/unomi/persistence/spi/PersistenceService.java @@ -116,7 +116,8 @@ public interface PersistenceService { boolean updateWithScript(String itemId, Date dateHint, Class<?> clazz, String script, Map<String, Object> scriptParams); /** - * Updates the items of the specified class by a query with a new property value for the specified property name based on a provided script. + * Updates the items of the specified class by a query with a new property value for the specified property name + * based on provided scripts and script parameters * * @param dateHint a Date helping in identifying where the item is located * @param clazz the Item subclass of the item to update diff --git a/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/actions/MergeProfilesOnPropertyAction.java b/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/actions/MergeProfilesOnPropertyAction.java index 2f4d0cf..8d3b54f 100644 --- a/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/actions/MergeProfilesOnPropertyAction.java +++ b/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/actions/MergeProfilesOnPropertyAction.java @@ -34,9 +34,7 @@ import org.slf4j.LoggerFactory; import javax.servlet.ServletResponse; import javax.servlet.http.Cookie; import javax.servlet.http.HttpServletResponse; -import java.util.Arrays; -import java.util.List; -import java.util.UUID; +import java.util.*; public class MergeProfilesOnPropertyAction implements ActionExecutor { private static final Logger logger = LoggerFactory.getLogger(MergeProfilesOnPropertyAction.class.getName()); @@ -183,7 +181,11 @@ public class MergeProfilesOnPropertyAction implements ActionExecutor { // we must mark all the profiles that we merged into the master as merged with the master, and they will // be deleted upon next load profile.setMergedWith(masterProfileId); - persistenceService.update(profile.getItemId(), null, Profile.class, "mergedWith", masterProfileId); + Map<String,Object> sourceMap = new HashMap<>(); + sourceMap.put("mergedWith", masterProfile); + profile.setSystemProperty("lastUpdated", new Date()); + sourceMap.put("systemProperties", profile.getSystemProperties()); + persistenceService.update(profile.getItemId(), null, Profile.class, sourceMap); } } } catch (Exception e) { diff --git a/plugins/mail/src/main/java/org/apache/unomi/plugins/mail/actions/SendMailAction.java b/plugins/mail/src/main/java/org/apache/unomi/plugins/mail/actions/SendMailAction.java index 9e1aadb..8e03175 100644 --- a/plugins/mail/src/main/java/org/apache/unomi/plugins/mail/actions/SendMailAction.java +++ b/plugins/mail/src/main/java/org/apache/unomi/plugins/mail/actions/SendMailAction.java @@ -33,6 +33,7 @@ import org.stringtemplate.v4.ST; import java.net.MalformedURLException; import java.net.URL; +import java.util.Date; import java.util.HashMap; import java.util.Map; @@ -112,7 +113,9 @@ public class SendMailAction implements ActionExecutor { } } - event.getProfile().getSystemProperties().put("notificationAck", profileNotif); + event.getProfile().setSystemProperty("notificationAck", profileNotif); + event.getProfile().setSystemProperty("lastUpdated", new Date()); + persistenceService.update(event.getProfile().getItemId(), null, Profile.class, "systemProperties", event.getProfile().getSystemProperties()); ST stringTemplate = new ST(template, '$', '$'); diff --git a/plugins/pom.xml b/plugins/pom.xml index acca9f6..2cf4bec 100644 --- a/plugins/pom.xml +++ b/plugins/pom.xml @@ -38,6 +38,7 @@ <module>hover-event</module> <module>past-event</module> <module>tracked-event</module> + <module>kafka-injector</module> </modules> <dependencies> diff --git a/services/src/main/java/org/apache/unomi/services/impl/ParserHelper.java b/services/src/main/java/org/apache/unomi/services/impl/ParserHelper.java index 6984e6e..a861f6c 100644 --- a/services/src/main/java/org/apache/unomi/services/impl/ParserHelper.java +++ b/services/src/main/java/org/apache/unomi/services/impl/ParserHelper.java @@ -27,9 +27,7 @@ import org.apache.unomi.api.services.DefinitionsService; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; +import java.util.*; /** * Helper class to resolve condition, action and values types when loading definitions from JSON files @@ -38,6 +36,9 @@ public class ParserHelper { private static final Logger logger = LoggerFactory.getLogger(ParserHelper.class); + private static final Set<String> unresolvedActionTypes = new HashSet<>(); + private static final Set<String> unresolvedConditionTypes = new HashSet<>(); + public static boolean resolveConditionType(final DefinitionsService definitionsService, Condition rootCondition) { if (rootCondition == null) { return false; @@ -49,16 +50,18 @@ public class ParserHelper { if (condition.getConditionType() == null) { ConditionType conditionType = definitionsService.getConditionType(condition.getConditionTypeId()); if (conditionType != null) { + unresolvedConditionTypes.remove(condition.getConditionTypeId()); condition.setConditionType(conditionType); } else { result.add(condition.getConditionTypeId()); + if (!unresolvedConditionTypes.contains(condition.getConditionTypeId())) { + unresolvedConditionTypes.add(condition.getConditionTypeId()); + logger.warn("Couldn't resolve condition type: " + condition.getConditionTypeId()); + } } } } }); - if (!result.isEmpty()) { - logger.warn("Couldn't resolve condition types : " + result); - } return result.isEmpty(); } @@ -105,9 +108,13 @@ public class ParserHelper { if (action.getActionType() == null) { ActionType actionType = definitionsService.getActionType(action.getActionTypeId()); if (actionType != null) { + unresolvedActionTypes.remove(action.getActionTypeId()); action.setActionType(actionType); } else { - logger.warn("Couldn't resolve action types : " + action.getActionTypeId()); + if (!unresolvedActionTypes.contains(action.getActionTypeId())) { + logger.warn("Couldn't resolve action type : " + action.getActionTypeId()); + unresolvedActionTypes.add(action.getActionTypeId()); + } return false; } } diff --git a/services/src/main/java/org/apache/unomi/services/impl/profiles/ProfileServiceImpl.java b/services/src/main/java/org/apache/unomi/services/impl/profiles/ProfileServiceImpl.java index 18ffbc3..94fb5c5 100644 --- a/services/src/main/java/org/apache/unomi/services/impl/profiles/ProfileServiceImpl.java +++ b/services/src/main/java/org/apache/unomi/services/impl/profiles/ProfileServiceImpl.java @@ -521,6 +521,7 @@ public class ProfileServiceImpl implements ProfileService, SynchronousBundleList if (profile.getItemId() == null) { return null; } + profile.setSystemProperty("lastUpdated", new Date()); if (persistenceService.save(profile)) { if (forceRefresh) { // triggering a load will force an in-place refresh, that may be expensive in performance but will make data immediately available. @@ -534,6 +535,7 @@ public class ProfileServiceImpl implements ProfileService, SynchronousBundleList public Profile saveOrMerge(Profile profile) { Profile previousProfile = persistenceService.load(profile.getItemId(), Profile.class); + profile.setSystemProperty("lastUpdated", new Date()); if (previousProfile == null) { if (persistenceService.save(profile)) { return profile; @@ -551,6 +553,7 @@ public class ProfileServiceImpl implements ProfileService, SynchronousBundleList } public Persona savePersona(Persona profile) { + profile.setSystemProperty("lastUpdated", new Date()); if (persistenceService.load(profile.getItemId(), Persona.class) == null) { Session session = new PersonaSession(UUID.randomUUID().toString(), profile, new Date()); persistenceService.save(profile); diff --git a/services/src/main/java/org/apache/unomi/services/impl/segments/SegmentServiceImpl.java b/services/src/main/java/org/apache/unomi/services/impl/segments/SegmentServiceImpl.java index 212c1b1..9c362ed 100644 --- a/services/src/main/java/org/apache/unomi/services/impl/segments/SegmentServiceImpl.java +++ b/services/src/main/java/org/apache/unomi/services/impl/segments/SegmentServiceImpl.java @@ -346,10 +346,18 @@ public class SegmentServiceImpl extends AbstractServiceImpl implements SegmentSe segmentCondition.setParameter("propertyValue", segmentId); List<Profile> previousProfiles = persistenceService.query(segmentCondition, null, Profile.class); + long updatedProfileCount = 0; + long profileRemovalStartTime = System.currentTimeMillis(); for (Profile profileToRemove : previousProfiles) { profileToRemove.getSegments().remove(segmentId); - persistenceService.update(profileToRemove.getItemId(), null, Profile.class, "segments", profileToRemove.getSegments()); + Map<String,Object> sourceMap = new HashMap<>(); + sourceMap.put("segments", profileToRemove.getSegments()); + profileToRemove.setSystemProperty("lastUpdated", new Date()); + sourceMap.put("systemProperties", profileToRemove.getSystemProperties()); + persistenceService.update(profileToRemove.getItemId(), null, Profile.class, sourceMap); + updatedProfileCount++; } + logger.info("Removed segment from {} profiles in {} ms", updatedProfileCount, System.currentTimeMillis() - profileRemovalStartTime); // update impacted segments for (Segment segment : impactedSegments) { @@ -786,6 +794,7 @@ public class SegmentServiceImpl extends AbstractServiceImpl implements SegmentSe Map<String, Object> systemProperties = new HashMap<>(); systemProperties.put("pastEvents", pastEventCounts); try { + systemProperties.put("lastUpdated", new Date()); persistenceService.update(profileId, null, Profile.class, "systemProperties", systemProperties); } catch (Exception e) { logger.error("Error updating profile {} past event system properties", profileId, e); @@ -825,7 +834,7 @@ public class SegmentServiceImpl extends AbstractServiceImpl implements SegmentSe } private void updateExistingProfilesForSegment(Segment segment) { - long t = System.currentTimeMillis(); + long updateProfilesForSegmentStartTime = System.currentTimeMillis(); Condition segmentCondition = new Condition(); long updatedProfileCount = 0; @@ -862,32 +871,40 @@ public class SegmentServiceImpl extends AbstractServiceImpl implements SegmentSe PartialList<Profile> profilesToAdd = persistenceService.query(profilesToAddCondition, null, Profile.class, 0, segmentUpdateBatchSize, "10m"); while (profilesToAdd.getList().size() > 0) { - long t2= System.currentTimeMillis(); + long profilesToAddStartTime = System.currentTimeMillis(); for (Profile profileToAdd : profilesToAdd.getList()) { profileToAdd.getSegments().add(segment.getItemId()); - persistenceService.update(profileToAdd.getItemId(), null, Profile.class, "segments", profileToAdd.getSegments()); + Map<String,Object> sourceMap = new HashMap<>(); + sourceMap.put("segments", profileToAdd.getSegments()); + profileToAdd.setSystemProperty("lastUpdated", new Date()); + sourceMap.put("systemProperties", profileToAdd.getSystemProperties()); + persistenceService.update(profileToAdd.getItemId(), null, Profile.class, sourceMap); Event profileUpdated = new Event("profileUpdated", null, profileToAdd, null, null, profileToAdd, new Date()); profileUpdated.setPersistent(false); eventService.send(profileUpdated); updatedProfileCount++; } - logger.info("{} profiles added in segment in {}ms", profilesToAdd.size(), System.currentTimeMillis() - t2); + logger.info("{} profiles added to segment in {}ms", profilesToAdd.size(), System.currentTimeMillis() - profilesToAddStartTime); profilesToAdd = persistenceService.continueScrollQuery(Profile.class, profilesToAdd.getScrollIdentifier(), profilesToAdd.getScrollTimeValidity()); if (profilesToAdd == null || profilesToAdd.getList().size() == 0) { break; } } while (profilesToRemove.getList().size() > 0) { - long t2= System.currentTimeMillis(); + long profilesToRemoveStartTime = System.currentTimeMillis(); for (Profile profileToRemove : profilesToRemove.getList()) { profileToRemove.getSegments().remove(segment.getItemId()); - persistenceService.update(profileToRemove.getItemId(), null, Profile.class, "segments", profileToRemove.getSegments()); + Map<String,Object> sourceMap = new HashMap<>(); + sourceMap.put("segments", profileToRemove.getSegments()); + profileToRemove.setSystemProperty("lastUpdated", new Date()); + sourceMap.put("systemProperties", profileToRemove.getSystemProperties()); + persistenceService.update(profileToRemove.getItemId(), null, Profile.class, sourceMap); Event profileUpdated = new Event("profileUpdated", null, profileToRemove, null, null, profileToRemove, new Date()); profileUpdated.setPersistent(false); eventService.send(profileUpdated); updatedProfileCount++; } - logger.info("{} profiles removed from segment in {}ms", profilesToRemove.size(), System.currentTimeMillis() - t2); + logger.info("{} profiles removed from segment in {}ms", profilesToRemove.size(), System.currentTimeMillis() - profilesToRemoveStartTime ); profilesToRemove = persistenceService.continueScrollQuery(Profile.class, profilesToRemove.getScrollIdentifier(), profilesToRemove.getScrollTimeValidity()); if (profilesToRemove == null || profilesToRemove.getList().size() == 0) { break; @@ -897,22 +914,31 @@ public class SegmentServiceImpl extends AbstractServiceImpl implements SegmentSe } else { PartialList<Profile> profilesToRemove = persistenceService.query(segmentCondition, null, Profile.class, 0, 200, "10m"); while (profilesToRemove.getList().size() > 0) { + long profilesToRemoveStartTime = System.currentTimeMillis(); for (Profile profileToRemove : profilesToRemove.getList()) { profileToRemove.getSegments().remove(segment.getItemId()); - persistenceService.update(profileToRemove.getItemId(), null, Profile.class, "segments", profileToRemove.getSegments()); + Map<String,Object> sourceMap = new HashMap<>(); + sourceMap.put("segments", profileToRemove.getSegments()); + profileToRemove.setSystemProperty("lastUpdated", new Date()); + sourceMap.put("systemProperties", profileToRemove.getSystemProperties()); + persistenceService.update(profileToRemove.getItemId(), null, Profile.class, sourceMap); + Event profileUpdated = new Event("profileUpdated", null, profileToRemove, null, null, profileToRemove, new Date()); + profileUpdated.setPersistent(false); + eventService.send(profileUpdated); updatedProfileCount++; } + logger.info("{} profiles removed from segment in {}ms", profilesToRemove.size(), System.currentTimeMillis() - profilesToRemoveStartTime); profilesToRemove = persistenceService.continueScrollQuery(Profile.class, profilesToRemove.getScrollIdentifier(), profilesToRemove.getScrollTimeValidity()); if (profilesToRemove == null || profilesToRemove.getList().size() == 0) { break; } } } - logger.info("{} profiles updated in {}ms", updatedProfileCount, System.currentTimeMillis() - t); + logger.info("{} profiles updated in {}ms", updatedProfileCount, System.currentTimeMillis() - updateProfilesForSegmentStartTime); } private void updateExistingProfilesForScoring(Scoring scoring) { - long t = System.currentTimeMillis(); + long startTime = System.currentTimeMillis(); Condition scoringCondition = new Condition(); scoringCondition.setConditionType(definitionsService.getConditionType("profilePropertyCondition")); scoringCondition.setParameter("propertyName", "scores." + scoring.getItemId()); @@ -922,13 +948,17 @@ public class SegmentServiceImpl extends AbstractServiceImpl implements SegmentSe HashMap<String, Object>[] scriptParams = new HashMap[scoring.getElements().size() + 1]; Condition[] conditions = new Condition[scoring.getElements().size() + 1]; + String lastUpdatedScriptPart = " if (!ctx._source.containsKey(\"systemProperties\")) { ctx._source.put(\"systemProperties\", [:]) } ctx._source.systemProperties.put(\"lastUpdated\", ZonedDateTime.ofInstant(Instant.ofEpochMilli(System.currentTimeMillis()), ZoneId.of(\"Z\")))"; + scriptParams[0] = new HashMap<String, Object>(); scriptParams[0].put("scoringId", scoring.getItemId()); - scripts[0] = "if( ctx._source.containsKey(\"systemProperties\") && ctx._source.systemProperties.containsKey(\"scoreModifiers\") && ctx._source.systemProperties.scoreModifiers.containsKey(params.scoringId) ) { ctx._source.scores.put(params.scoringId, ctx._source.systemProperties.scoreModifiers.get(params.scoringId)) } else { ctx._source.scores.remove(params.scoringId) }"; + scripts[0] = "if (ctx._source.containsKey(\"systemProperties\") && ctx._source.systemProperties.containsKey(\"scoreModifiers\") && ctx._source.systemProperties.scoreModifiers.containsKey(params.scoringId) ) { ctx._source.scores.put(params.scoringId, ctx._source.systemProperties.scoreModifiers.get(params.scoringId)) } else { ctx._source.scores.remove(params.scoringId) } " + + lastUpdatedScriptPart; conditions[0] = scoringCondition; if (scoring.getMetadata().isEnabled()) { - String scriptToAdd = "if( !ctx._source.containsKey(\"scores\") ){ ctx._source.put(\"scores\", [:])} if( ctx._source.scores.containsKey(params.scoringId) ) { ctx._source.scores.put(params.scoringId, ctx._source.scores.get(params.scoringId)+params.scoringValue) } else { ctx._source.scores.put(params.scoringId, params.scoringValue) }"; + String scriptToAdd = "if (!ctx._source.containsKey(\"scores\")) { ctx._source.put(\"scores\", [:])} if (ctx._source.scores.containsKey(params.scoringId) ) { ctx._source.scores.put(params.scoringId, ctx._source.scores.get(params.scoringId)+params.scoringValue) } else { ctx._source.scores.put(params.scoringId, params.scoringValue) } " + + lastUpdatedScriptPart; int idx = 1; for (ScoringElement element : scoring.getElements()) { scriptParams[idx] = new HashMap<>(); @@ -939,13 +969,12 @@ public class SegmentServiceImpl extends AbstractServiceImpl implements SegmentSe idx++; } } - persistenceService.updateWithQueryAndScript(null, Profile.class, scripts, scriptParams, conditions); - logger.info("Profiles updated in {}ms", System.currentTimeMillis() - t); + logger.info("Updated scoring for profiles in {}ms", System.currentTimeMillis() - startTime); } private void updateExistingProfilesForRemovedScoring(String scoringId) { - long t = System.currentTimeMillis(); + long startTime = System.currentTimeMillis(); Condition scoringCondition = new Condition(); scoringCondition.setConditionType(definitionsService.getConditionType("profilePropertyCondition")); scoringCondition.setParameter("propertyName", "scores." + scoringId); @@ -958,11 +987,11 @@ public class SegmentServiceImpl extends AbstractServiceImpl implements SegmentSe scriptParams[0].put("scoringId", scoringId); String[] script = new String[1]; - script[0] = "ctx._source.scores.remove(params.scoringId)"; + script[0] = "ctx._source.scores.remove(params.scoringId); if (!ctx._source.containsKey(\"systemProperties\")) { ctx._source.put(\"systemProperties\", [:]) } ctx._source.systemProperties.put(\"lastUpdated\", ZonedDateTime.ofInstant(Instant.ofEpochMilli(System.currentTimeMillis()), ZoneId.of(\"Z\")))"; persistenceService.updateWithQueryAndScript(null, Profile.class, script, scriptParams, conditions); - logger.info("Profiles updated in {}ms", System.currentTimeMillis() - t); + logger.info("Removed scoring from profiles in {}ms", System.currentTimeMillis() - startTime); } public void bundleChanged(BundleEvent event) { diff --git a/tools/shell-dev-commands/src/main/java/org/apache/unomi/shell/commands/ProfileList.java b/tools/shell-dev-commands/src/main/java/org/apache/unomi/shell/commands/ProfileList.java index 14f16a9..24f3a2c 100644 --- a/tools/shell-dev-commands/src/main/java/org/apache/unomi/shell/commands/ProfileList.java +++ b/tools/shell-dev-commands/src/main/java/org/apache/unomi/shell/commands/ProfileList.java @@ -51,14 +51,15 @@ public class ProfileList extends ListCommandSupport { "Scope", "Segments", "Consents", - "Last modification", + "Last visit", + "Last update" }; } @java.lang.Override protected DataTable buildDataTable() { Query query = new Query(); - query.setSortby("properties.lastVisit:desc"); + query.setSortby("systemProperties.lastUpdated:desc,properties.lastVisit:desc"); query.setLimit(maxEntries); Condition matchAllCondition = new Condition(definitionsService.getConditionType("matchAllCondition")); query.setCondition(matchAllCondition); @@ -71,6 +72,11 @@ public class ProfileList extends ListCommandSupport { rowData.add(StringUtils.join(profile.getSegments(), ",")); rowData.add(StringUtils.join(profile.getConsents().keySet(), ",")); rowData.add((String) profile.getProperty("lastVisit")); + if (profile.getSystemProperties() != null && profile.getSystemProperties().get("lastUpdated") != null) { + rowData.add((String) profile.getSystemProperties().get("lastUpdated")); + } else { + rowData.add(""); + } dataTable.addRow(rowData.toArray(new Comparable[rowData.size()])); } return dataTable;
