Repository: incubator-unomi Updated Branches: refs/heads/master 94253a952 -> 57aa6fcec
UNOMI-137 : Do not store personal identifiers in session objects Project: http://git-wip-us.apache.org/repos/asf/incubator-unomi/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-unomi/commit/57aa6fce Tree: http://git-wip-us.apache.org/repos/asf/incubator-unomi/tree/57aa6fce Diff: http://git-wip-us.apache.org/repos/asf/incubator-unomi/diff/57aa6fce Branch: refs/heads/master Commit: 57aa6fcece85f0efe8728f6d83a54c110cc1fdf4 Parents: fdb1dd5 Author: Abdelkader Midani <[email protected]> Authored: Tue Nov 7 14:16:43 2017 +0100 Committer: Abdelkader Midani <[email protected]> Committed: Tue Nov 7 14:16:59 2017 +0100 ---------------------------------------------------------------------- .../services/services/ProfileServiceImpl.java | 19 +++++++++++++ .../properties/profiles/basic/firstName.json | 3 +- .../cxs/properties/profiles/basic/lastName.json | 3 +- .../properties/profiles/contact/address.json | 3 +- .../cxs/properties/profiles/contact/email.json | 3 +- .../profiles/contact/phoneNumber.json | 3 +- .../properties/profiles/social/facebookId.json | 3 +- .../properties/profiles/social/googleid.json | 3 +- .../properties/profiles/social/linkedInId.json | 3 +- .../properties/profiles/social/twitterId.json | 3 +- .../shell/migration/impl/MigrationTo130.java | 29 +++++++++----------- 11 files changed, 50 insertions(+), 25 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/57aa6fce/services/src/main/java/org/apache/unomi/services/services/ProfileServiceImpl.java ---------------------------------------------------------------------- diff --git a/services/src/main/java/org/apache/unomi/services/services/ProfileServiceImpl.java b/services/src/main/java/org/apache/unomi/services/services/ProfileServiceImpl.java index 5ec9ceb..1c8747a 100644 --- a/services/src/main/java/org/apache/unomi/services/services/ProfileServiceImpl.java +++ b/services/src/main/java/org/apache/unomi/services/services/ProfileServiceImpl.java @@ -66,6 +66,8 @@ public class ProfileServiceImpl implements ProfileService, SynchronousBundleList private List<PropertyType> allPropertyTypes; + private final String PERSONAL_IDENTIFIER_TAG_NAME = "personalIdentifierProperties"; + public ProfileServiceImpl() { logger.info("Initializing profile service..."); } @@ -559,9 +561,26 @@ public class ProfileServiceImpl implements ProfileService, SynchronousBundleList if (session.getItemId() == null) { return null; } + if(session.getProfile()!=null && session.getProfile().getProperties()!=null) { + session.getProfile().setProperties(removePersonalIdentifiersFromSessionProfile(session.getProfile().getProperties())); + } return persistenceService.save(session) ? session : null; } + private Map removePersonalIdentifiersFromSessionProfile(final Map<String, Object> profileProperties){ + Set<PropertyType> personalIdsProps = getPropertyTypeBySystemTag(PERSONAL_IDENTIFIER_TAG_NAME); + final List personalIdsPropsNames = new ArrayList<String>(); + personalIdsProps.forEach(propType -> personalIdsPropsNames.add(propType.getMetadata().getId())); + Set propsToRemove = new HashSet<String>(); + profileProperties.keySet().forEach( propKey -> { + if(personalIdsPropsNames.contains(propKey)) { + propsToRemove.add(propKey); + } + }); + propsToRemove.forEach(propId -> profileProperties.remove(propId)); + return profileProperties; + } + public PartialList<Session> findProfileSessions(String profileId) { return persistenceService.query("profileId", profileId, "timeStamp:desc", Session.class, 0, 50); } http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/57aa6fce/services/src/main/resources/META-INF/cxs/properties/profiles/basic/firstName.json ---------------------------------------------------------------------- diff --git a/services/src/main/resources/META-INF/cxs/properties/profiles/basic/firstName.json b/services/src/main/resources/META-INF/cxs/properties/profiles/basic/firstName.json index 865f7f6..a54869c 100644 --- a/services/src/main/resources/META-INF/cxs/properties/profiles/basic/firstName.json +++ b/services/src/main/resources/META-INF/cxs/properties/profiles/basic/firstName.json @@ -5,7 +5,8 @@ "systemTags": [ "properties", "profileProperties", - "basicProfileProperties" + "basicProfileProperties", + "personalIdentifierProperties" ] }, "type": "string", http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/57aa6fce/services/src/main/resources/META-INF/cxs/properties/profiles/basic/lastName.json ---------------------------------------------------------------------- diff --git a/services/src/main/resources/META-INF/cxs/properties/profiles/basic/lastName.json b/services/src/main/resources/META-INF/cxs/properties/profiles/basic/lastName.json index e745550..041d3fc 100644 --- a/services/src/main/resources/META-INF/cxs/properties/profiles/basic/lastName.json +++ b/services/src/main/resources/META-INF/cxs/properties/profiles/basic/lastName.json @@ -5,7 +5,8 @@ "systemTags": [ "properties", "profileProperties", - "basicProfileProperties" + "basicProfileProperties", + "personalIdentifierProperties" ] }, "type": "string", http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/57aa6fce/services/src/main/resources/META-INF/cxs/properties/profiles/contact/address.json ---------------------------------------------------------------------- diff --git a/services/src/main/resources/META-INF/cxs/properties/profiles/contact/address.json b/services/src/main/resources/META-INF/cxs/properties/profiles/contact/address.json index 5b5799e..ea9a7c3 100644 --- a/services/src/main/resources/META-INF/cxs/properties/profiles/contact/address.json +++ b/services/src/main/resources/META-INF/cxs/properties/profiles/contact/address.json @@ -5,7 +5,8 @@ "systemTags": [ "properties", "profileProperties", - "contactProfileProperties" + "contactProfileProperties", + "personalIdentifierProperties" ] }, "type": "string", http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/57aa6fce/services/src/main/resources/META-INF/cxs/properties/profiles/contact/email.json ---------------------------------------------------------------------- diff --git a/services/src/main/resources/META-INF/cxs/properties/profiles/contact/email.json b/services/src/main/resources/META-INF/cxs/properties/profiles/contact/email.json index 662358c..49ec6ed 100644 --- a/services/src/main/resources/META-INF/cxs/properties/profiles/contact/email.json +++ b/services/src/main/resources/META-INF/cxs/properties/profiles/contact/email.json @@ -5,7 +5,8 @@ "systemTags": [ "properties", "profileProperties", - "contactProfileProperties" + "contactProfileProperties", + "personalIdentifierProperties" ] }, "type": "email", http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/57aa6fce/services/src/main/resources/META-INF/cxs/properties/profiles/contact/phoneNumber.json ---------------------------------------------------------------------- diff --git a/services/src/main/resources/META-INF/cxs/properties/profiles/contact/phoneNumber.json b/services/src/main/resources/META-INF/cxs/properties/profiles/contact/phoneNumber.json index 8b7a2bf..de90c18 100644 --- a/services/src/main/resources/META-INF/cxs/properties/profiles/contact/phoneNumber.json +++ b/services/src/main/resources/META-INF/cxs/properties/profiles/contact/phoneNumber.json @@ -5,7 +5,8 @@ "systemTags": [ "properties", "profileProperties", - "contactProfileProperties" + "contactProfileProperties", + "personalIdentifierProperties" ] }, "type": "string", http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/57aa6fce/services/src/main/resources/META-INF/cxs/properties/profiles/social/facebookId.json ---------------------------------------------------------------------- diff --git a/services/src/main/resources/META-INF/cxs/properties/profiles/social/facebookId.json b/services/src/main/resources/META-INF/cxs/properties/profiles/social/facebookId.json index 157c4dd..ab969ac 100644 --- a/services/src/main/resources/META-INF/cxs/properties/profiles/social/facebookId.json +++ b/services/src/main/resources/META-INF/cxs/properties/profiles/social/facebookId.json @@ -5,7 +5,8 @@ "systemTags": [ "properties", "profileProperties", - "socialProfileProperties" + "socialProfileProperties", + "personalIdentifierProperties" ] }, "type": "string", http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/57aa6fce/services/src/main/resources/META-INF/cxs/properties/profiles/social/googleid.json ---------------------------------------------------------------------- diff --git a/services/src/main/resources/META-INF/cxs/properties/profiles/social/googleid.json b/services/src/main/resources/META-INF/cxs/properties/profiles/social/googleid.json index c0f0c12..00a2a92 100644 --- a/services/src/main/resources/META-INF/cxs/properties/profiles/social/googleid.json +++ b/services/src/main/resources/META-INF/cxs/properties/profiles/social/googleid.json @@ -5,7 +5,8 @@ "systemTags": [ "properties", "profileProperties", - "socialProfileProperties" + "socialProfileProperties", + "personalIdentifierProperties" ] }, "type": "string", http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/57aa6fce/services/src/main/resources/META-INF/cxs/properties/profiles/social/linkedInId.json ---------------------------------------------------------------------- diff --git a/services/src/main/resources/META-INF/cxs/properties/profiles/social/linkedInId.json b/services/src/main/resources/META-INF/cxs/properties/profiles/social/linkedInId.json index c8ec497..111891e 100644 --- a/services/src/main/resources/META-INF/cxs/properties/profiles/social/linkedInId.json +++ b/services/src/main/resources/META-INF/cxs/properties/profiles/social/linkedInId.json @@ -5,7 +5,8 @@ "systemTags": [ "properties", "profileProperties", - "socialProfileProperties" + "socialProfileProperties", + "personalIdentifierProperties" ] }, "type": "string", http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/57aa6fce/services/src/main/resources/META-INF/cxs/properties/profiles/social/twitterId.json ---------------------------------------------------------------------- diff --git a/services/src/main/resources/META-INF/cxs/properties/profiles/social/twitterId.json b/services/src/main/resources/META-INF/cxs/properties/profiles/social/twitterId.json index f4a75b3..93c0654 100644 --- a/services/src/main/resources/META-INF/cxs/properties/profiles/social/twitterId.json +++ b/services/src/main/resources/META-INF/cxs/properties/profiles/social/twitterId.json @@ -5,7 +5,8 @@ "systemTags": [ "properties", "profileProperties", - "socialProfileProperties" + "socialProfileProperties", + "personalIdentifierProperties" ] }, "type": "string", http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/57aa6fce/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/impl/MigrationTo130.java ---------------------------------------------------------------------- diff --git a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/impl/MigrationTo130.java b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/impl/MigrationTo130.java index c596175..a176f0f 100644 --- a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/impl/MigrationTo130.java +++ b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/impl/MigrationTo130.java @@ -66,30 +66,26 @@ public class MigrationTo130 implements Migration { initTagsStructurePriorTo130(); String hostAddress = ConsoleUtils.askUserWithDefaultAnswer(session, "Host address (default = http://localhost:9200): ", "http://localhost:9200";); String tagsOperation = ConsoleUtils.askUserWithAuthorizedAnswer(session, "How to manage tags?\n1. copy: will duplicate tags in systemTags property\n2. move: will move tags in systemTags property\n[1 - 2]: ", Arrays.asList("1", "2")); - String removeNamespaceOnSystemTags = ConsoleUtils.askUserWithAuthorizedAnswer(session,"As we will copy/move the tags, do you wish to remove existing namespace on tags before copy/move in systemTags? (e.g: hidden.) (yes/no): ", Arrays.asList("yes", "no")); + String removeNamespaceOnSystemTags = ConsoleUtils.askUserWithAuthorizedAnswer(session, "As we will copy/move the tags, do you wish to remove existing namespace on tags before copy/move in systemTags? (e.g: hidden.) (yes/no): ", Arrays.asList("yes", "no")); List<String> typeToMigrate = Arrays.asList("actionType", "conditionType", "campaign", "goal", "rule", "scoring", "segment", "userList"); for (String type : typeToMigrate) { - migrateTypeTags(hostAddress, type, tagsOperation, removeNamespaceOnSystemTags.equals("yes")); + migrateTagsInResult(hostAddress, type, 10, true, tagsOperation, removeNamespaceOnSystemTags.equals("yes"), null); } - migratePropertyTypesTags(hostAddress, tagsOperation, removeNamespaceOnSystemTags.equals("yes")); + migrateTagsInResult(hostAddress, "propertyType", 10, false, tagsOperation, removeNamespaceOnSystemTags.equals("yes"), null); } - private void migrateTypeTags(String hostAddress, String type, String tagsOperation, boolean removeNamespaceOnSystemTags) throws IOException { - JSONObject responseJSON = MigrationUtils.queryWithScroll(httpClient, hostAddress + "/context/" + type + "/_search"); + private void migrateTagsInResult(String hostAddress, String type, int currentOffset, + boolean tagsInMetadata, String tagsOperation, boolean removeNamespaceOnSystemTags, String scrollId) throws IOException { - migrateTagsInResult(responseJSON, hostAddress, type,10, true, tagsOperation, removeNamespaceOnSystemTags); - } - - private void migratePropertyTypesTags(String hostAddress, String tagsOperation, boolean removeNamespaceOnSystemTags) throws IOException { - JSONObject responseJSON = MigrationUtils.queryWithScroll(httpClient,hostAddress + "/context/propertyType/_search"); - - migrateTagsInResult(responseJSON, hostAddress, "propertyType", 10, false, tagsOperation, removeNamespaceOnSystemTags); - } + JSONObject responseJSON; + if (StringUtils.isNotBlank(scrollId)) { + responseJSON = MigrationUtils.continueQueryWithScroll(httpClient, hostAddress, scrollId); + } else { + responseJSON = MigrationUtils.queryWithScroll(httpClient, hostAddress + "/context/" + type + "/_search"); + } - private void migrateTagsInResult(JSONObject responseJSON, String hostAddress, String type, int currentOffset, - boolean tagsInMetadata, String tagsOperation, boolean removeNamespaceOnSystemTags) throws IOException { if (responseJSON.has("hits")) { JSONObject hitsObject = responseJSON.getJSONObject("hits"); if (hitsObject.has("hits")) { @@ -115,7 +111,7 @@ public class MigrationTo130 implements Migration { } if (hitsObject.getInt("total") > currentOffset) { - migrateTagsInResult(MigrationUtils.continueQueryWithScroll(httpClient, hostAddress, responseJSON.getString("_scroll_id")), hostAddress, type,currentOffset + 10, tagsInMetadata, tagsOperation, removeNamespaceOnSystemTags); + migrateTagsInResult(hostAddress, type, currentOffset + 10, tagsInMetadata, tagsOperation, removeNamespaceOnSystemTags, responseJSON.getString("_scroll_id")); } } } @@ -215,6 +211,7 @@ public class MigrationTo130 implements Migration { tagsStructurePriorTo130.put("socialProfileProperties", Arrays.asList("properties", "profileProperties")); tagsStructurePriorTo130.put("personalProfileProperties", Arrays.asList("properties", "profileProperties")); tagsStructurePriorTo130.put("workProfileProperties", Arrays.asList("properties", "profileProperties")); + tagsStructurePriorTo130.put("personalIdentifierProperties", Arrays.asList("properties", "profileProperties")); tagsStructurePriorTo130.put("sessionProperties", Collections.singletonList("properties")); tagsStructurePriorTo130.put("geographicSessionProperties", Arrays.asList("properties", "sessionProperties"));
