UNOMI-118 : Ranking test, profile import
Project: http://git-wip-us.apache.org/repos/asf/incubator-unomi/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-unomi/commit/a8de6e1f Tree: http://git-wip-us.apache.org/repos/asf/incubator-unomi/tree/a8de6e1f Diff: http://git-wip-us.apache.org/repos/asf/incubator-unomi/diff/a8de6e1f Branch: refs/heads/feature-UNOMI-117 Commit: a8de6e1f73c05229a27fd82aa6e340d852c84a63 Parents: 777b13e Author: Abdelkader Midani <[email protected]> Authored: Mon Aug 14 01:18:49 2017 +0200 Committer: Abdelkader Midani <[email protected]> Committed: Mon Aug 14 01:18:49 2017 +0200 ---------------------------------------------------------------------- .../services/ProfileImportServiceImpl.java | 69 +++---- .../java/org/apache/unomi/itests/AllITs.java | 3 +- .../java/org/apache/unomi/itests/BaseIT.java | 2 + .../unomi/itests/ProfileImportRankingIT.java | 180 +++++++++++++++++++ .../itests/ProfileImportSurfersDeleteIT.java | 6 +- .../unomi/itests/ProfileImportSurfersIT.java | 6 +- itests/src/test/resources/4-ranking-test.csv | 26 +++ 7 files changed, 253 insertions(+), 39 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/a8de6e1f/extensions/router/router-service/src/main/java/org/apache/unomi/router/services/ProfileImportServiceImpl.java ---------------------------------------------------------------------- diff --git a/extensions/router/router-service/src/main/java/org/apache/unomi/router/services/ProfileImportServiceImpl.java b/extensions/router/router-service/src/main/java/org/apache/unomi/router/services/ProfileImportServiceImpl.java index b7ab45a..4ca2a09 100644 --- a/extensions/router/router-service/src/main/java/org/apache/unomi/router/services/ProfileImportServiceImpl.java +++ b/extensions/router/router-service/src/main/java/org/apache/unomi/router/services/ProfileImportServiceImpl.java @@ -34,43 +34,48 @@ public class ProfileImportServiceImpl extends AbstractCustomServiceImpl implemen private static final Logger logger = LoggerFactory.getLogger(ProfileImportServiceImpl.class.getName()); public boolean saveMergeDeleteImportedProfile(ProfileToImport profileToImport) throws InvocationTargetException, IllegalAccessException { - logger.debug("Importing profile with ID : {}", profileToImport.getItemId()); + logger.info("Importing profile with ID : {}", profileToImport.getItemId()); Profile existingProfile = new Profile(); - List<Profile> existingProfiles = persistenceService.query("properties." + profileToImport.getMergingProperty(), profileToImport.getProperties().get(profileToImport.getMergingProperty()).toString(), null, Profile.class); - logger.debug("Query existing profile with mergingProperty: {}. Found: {}", profileToImport.getMergingProperty(), existingProfiles.size()); + if(profileToImport.getProperties().get(profileToImport.getMergingProperty()) != null) { + List<Profile> existingProfiles = persistenceService.query("properties." + profileToImport.getMergingProperty(), profileToImport.getProperties().get(profileToImport.getMergingProperty()).toString(), null, Profile.class); + logger.info("Query existing profile with mergingProperty: {}. Found: {}", profileToImport.getMergingProperty(), existingProfiles.size()); - //Profile already exist, and import config allow to overwrite profiles - if (existingProfiles.size() == 1) { - existingProfile = existingProfiles.get(0); - if (profileToImport.isProfileToDelete()) { - logger.debug("Profile is to delete!"); - persistenceService.remove(existingProfile.getItemId(), Profile.class); - return true; - } - List<String> propertiesToOverwrite = profileToImport.getPropertiesToOverwrite(); - if (profileToImport.isOverwriteExistingProfiles() && propertiesToOverwrite != null && propertiesToOverwrite.size() > 0) { // We overwrite only properties marked to overwrite - logger.debug("Properties to overwrite: {}", propertiesToOverwrite); - for (String propName : propertiesToOverwrite) { - existingProfile.getProperties().put(propName, profileToImport.getProperties().get(propName)); + //Profile already exist, and import config allow to overwrite profiles + if (existingProfiles.size() == 1) { + existingProfile = existingProfiles.get(0); + if (profileToImport.isProfileToDelete()) { + logger.info("Profile is to delete!"); + persistenceService.remove(existingProfile.getItemId(), Profile.class); + return true; + } + List<String> propertiesToOverwrite = profileToImport.getPropertiesToOverwrite(); + if (profileToImport.isOverwriteExistingProfiles() && propertiesToOverwrite != null && propertiesToOverwrite.size() > 0) { // We overwrite only properties marked to overwrite + logger.info("Properties to overwrite: {}", propertiesToOverwrite); + for (String propName : propertiesToOverwrite) { + existingProfile.getProperties().put(propName, profileToImport.getProperties().get(propName)); + } + } else { //If no property is marked to overwrite we replace the whole properties map + logger.info("Overwrite all properties"); + existingProfile.setProperties(profileToImport.getProperties()); } - } else { //If no property is marked to overwrite we replace the whole properties map - logger.debug("Overwrite all properties"); - existingProfile.setProperties(profileToImport.getProperties()); + //update segments and scores + existingProfile.setSegments(profileToImport.getSegments()); + existingProfile.setScores(profileToImport.getScores()); + } else if (existingProfiles.size() == 0 && !profileToImport.isProfileToDelete()) { + logger.info("New profile to add..."); + BeanUtils.copyProperties(existingProfile, profileToImport); + } else { + logger.warn("{} occurences found for profile with {} = {}. Profile import is skipped", existingProfiles.size(), + profileToImport.getMergingProperty(), profileToImport.getProperties().get(profileToImport.getMergingProperty()).toString()); + } + logger.info("-------------------------------------"); + if (!profileToImport.isProfileToDelete()) { + return persistenceService.save(existingProfile, true); + } else { + return false; } - //update segments and scores - existingProfile.setSegments(profileToImport.getSegments()); - existingProfile.setScores(profileToImport.getScores()); - } else if (existingProfiles.size() == 0 && !profileToImport.isProfileToDelete()) { - logger.debug("New profile to add..."); - BeanUtils.copyProperties(existingProfile, profileToImport); - } else { - logger.warn("{} occurences found for profile with {} = {}. Profile import is skipped", existingProfiles.size(), - profileToImport.getMergingProperty(), profileToImport.getProperties().get(profileToImport.getMergingProperty()).toString()); - } - logger.debug("-------------------------------------"); - if (!profileToImport.isProfileToDelete()) { - return persistenceService.save(existingProfile, true); } else { + logger.error("Merging property '{}' has no value in the current profile."); return false; } } http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/a8de6e1f/itests/src/test/java/org/apache/unomi/itests/AllITs.java ---------------------------------------------------------------------- diff --git a/itests/src/test/java/org/apache/unomi/itests/AllITs.java b/itests/src/test/java/org/apache/unomi/itests/AllITs.java index d32e627..3673289 100644 --- a/itests/src/test/java/org/apache/unomi/itests/AllITs.java +++ b/itests/src/test/java/org/apache/unomi/itests/AllITs.java @@ -35,7 +35,8 @@ import org.junit.runners.Suite.SuiteClasses; ProfileServiceIT.class, ProfileImportBasicIT.class, ProfileImportSurfersIT.class, - ProfileImportSurfersDeleteIT.class + ProfileImportSurfersDeleteIT.class, + ProfileImportRankingIT.class }) public class AllITs { } http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/a8de6e1f/itests/src/test/java/org/apache/unomi/itests/BaseIT.java ---------------------------------------------------------------------- diff --git a/itests/src/test/java/org/apache/unomi/itests/BaseIT.java b/itests/src/test/java/org/apache/unomi/itests/BaseIT.java index 465f3b5..8f70f2f 100644 --- a/itests/src/test/java/org/apache/unomi/itests/BaseIT.java +++ b/itests/src/test/java/org/apache/unomi/itests/BaseIT.java @@ -101,6 +101,8 @@ public abstract class BaseIT { "src/test/resources/2-surfers-test.csv")), replaceConfigurationFile("data/tmp/recurrent_import/3-surfers-delete-test.csv", new File( "src/test/resources/3-surfers-delete-test.csv")), + replaceConfigurationFile("data/tmp/recurrent_import/4-ranking-test.csv", new File( + "src/test/resources/4-ranking-test.csv")), keepRuntimeFolder(), configureConsole().ignoreLocalConsole(), logLevel(LogLevel.INFO), http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/a8de6e1f/itests/src/test/java/org/apache/unomi/itests/ProfileImportRankingIT.java ---------------------------------------------------------------------- diff --git a/itests/src/test/java/org/apache/unomi/itests/ProfileImportRankingIT.java b/itests/src/test/java/org/apache/unomi/itests/ProfileImportRankingIT.java new file mode 100644 index 0000000..e96175a --- /dev/null +++ b/itests/src/test/java/org/apache/unomi/itests/ProfileImportRankingIT.java @@ -0,0 +1,180 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License + */ +package org.apache.unomi.itests; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.http.HttpResponse; +import org.apache.http.client.methods.HttpPut; +import org.apache.http.entity.StringEntity; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClients; +import org.apache.unomi.api.Metadata; +import org.apache.unomi.api.PartialList; +import org.apache.unomi.api.Profile; +import org.apache.unomi.api.PropertyType; +import org.apache.unomi.api.services.ProfileService; +import org.apache.unomi.router.api.ImportConfiguration; +import org.apache.unomi.router.api.RouterConstants; +import org.apache.unomi.router.api.services.ImportExportConfigurationService; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.ops4j.pax.exam.junit.PaxExam; +import org.ops4j.pax.exam.spi.reactors.ExamReactorStrategy; +import org.ops4j.pax.exam.spi.reactors.PerSuite; +import org.ops4j.pax.exam.util.Filter; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.inject.Inject; +import javax.ws.rs.core.MediaType; +import java.io.File; +import java.io.IOException; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.MatcherAssert.assertThat; + +/** + * Created by amidani on 09/08/2017. + */ +@RunWith(PaxExam.class) +@ExamReactorStrategy(PerSuite.class) +public class ProfileImportRankingIT extends BaseIT { + + @Inject + @Filter("(configDiscriminator=IMPORT)") + protected ImportExportConfigurationService<ImportConfiguration> importConfigurationService; + + @Inject + protected ProfileService profileService; + private Logger logger = LoggerFactory.getLogger(ProfileImportRankingIT.class); + + + @Before + public void setUp() throws IOException { + + /*** Create Missing Properties ***/ + PropertyType propertyTypeUciId = new PropertyType(new Metadata("integration", "uciId", "UCI ID", "UCI ID")); + propertyTypeUciId.setValueTypeId("string"); + propertyTypeUciId.setTagIds(Collections.singleton("basicProfileProperties")); + propertyTypeUciId.setTarget("profiles"); + + profileService.setPropertyType(propertyTypeUciId); + + PropertyType propertyTypeRank = new PropertyType(new Metadata("integration", "rank", "Rank", "Rank")); + propertyTypeRank.setValueTypeId("integer"); + propertyTypeRank.setTagIds(Collections.singleton("basicProfileProperties")); + propertyTypeRank.setTarget("profiles"); + + profileService.setPropertyType(propertyTypeRank); + + /*** Surfers Test ***/ + ImportConfiguration importConfigRanking = new ImportConfiguration(); + importConfigRanking.setItemId("4-ranking-test"); + importConfigRanking.setConfigType(RouterConstants.IMPORT_EXPORT_CONFIG_TYPE_RECURRENT); + importConfigRanking.setMergingProperty("rank"); + importConfigRanking.setOverwriteExistingProfiles(true); + importConfigRanking.setColumnSeparator(";"); + importConfigRanking.setHasHeader(true); + importConfigRanking.setHasDeleteColumn(false); + + Map mappingRanking = new HashMap(); + mappingRanking.put("rank", 0); + mappingRanking.put("uciId", 1); + mappingRanking.put("lastName", 2); + mappingRanking.put("nationality", 3); + mappingRanking.put("age", 4); + + importConfigRanking.getProperties().put("mapping", mappingRanking); + File importSurfersFile = new File("data/tmp/recurrent_import/"); + importConfigRanking.getProperties().put("source", "file://" + importSurfersFile.getAbsolutePath() + "?fileName=4-ranking-test.csv&consumer.delay=10m&move=.done"); + importConfigRanking.setActive(true); + + ImportConfiguration savedImportConfig = importConfigurationService.save(importConfigRanking); + + CloseableHttpClient httpclient = HttpClients.createDefault(); + HttpPut httpPut = new HttpPut(URL + "/configUpdate/importConfigAdmin"); + + String json = new ObjectMapper().writeValueAsString(savedImportConfig); + StringEntity entity = new StringEntity(json); + entity.setContentType(MediaType.APPLICATION_JSON); + httpPut.setEntity(entity); + + HttpResponse response = httpclient.execute(httpPut); + assertThat(response.getStatusLine().getStatusCode(), equalTo(200)); + + httpclient.close(); + + } + + @Test + public void testCheckImportConfigListRanking() { + + List<ImportConfiguration> importConfigurations = importConfigurationService.getAll(); + Assert.assertEquals(4, importConfigurations.size()); + + } + + + @Test + public void testCheckAddedPropertiesRanking() throws IOException, InterruptedException { + + //Wait for data to be processed + Thread.sleep(1000); + + PropertyType propUciId = profileService.getPropertyType("uciId"); + Assert.assertNotNull(propUciId); + + PropertyType propRankId = profileService.getPropertyType("rank"); + Assert.assertNotNull(propRankId); + + } + + @Test + public void testImport4Ranking() throws InterruptedException { + + //Wait for data to be processed + //Check import config status + ImportConfiguration importConfiguration = importConfigurationService.load("4-ranking-test"); + while (importConfiguration != null && !RouterConstants.CONFIG_STATUS_COMPLETE_SUCCESS.equals(importConfiguration.getStatus())) { + logger.info("$$$$ : testImport4Ranking : Waiting for data to be processed ..."); + Thread.sleep(1000); + importConfiguration = importConfigurationService.load("4-ranking-test"); + } + Thread.sleep(10000); + + Assert.assertEquals(1, importConfiguration.getExecutions().size()); + + Assert.assertEquals(28, profileService.getAllProfilesCount()); + + PartialList<Profile> gregProfile = profileService.findProfilesByPropertyValue("properties.uciId", "10004451371", 0, 10, null); + Assert.assertEquals(1, gregProfile.getList().size()); + Assert.assertNotNull(gregProfile.get(0)); + Assert.assertEquals(1, gregProfile.get(0).getProperty("rank")); + Assert.assertEquals("VAN AVERMAET Greg", gregProfile.get(0).getProperty("lastName")); + Assert.assertEquals("BELGIUM", gregProfile.get(0).getProperty("nationality")); + Assert.assertEquals(32, gregProfile.get(0).getProperty("age")); + + + } + +} http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/a8de6e1f/itests/src/test/java/org/apache/unomi/itests/ProfileImportSurfersDeleteIT.java ---------------------------------------------------------------------- diff --git a/itests/src/test/java/org/apache/unomi/itests/ProfileImportSurfersDeleteIT.java b/itests/src/test/java/org/apache/unomi/itests/ProfileImportSurfersDeleteIT.java index d605693..e04956a 100644 --- a/itests/src/test/java/org/apache/unomi/itests/ProfileImportSurfersDeleteIT.java +++ b/itests/src/test/java/org/apache/unomi/itests/ProfileImportSurfersDeleteIT.java @@ -90,7 +90,7 @@ public class ProfileImportSurfersDeleteIT extends BaseIT { importConfigSurfersDelete.getProperties().put("mapping", mappingSurfers); - importConfigSurfersDelete.getProperties().put("source", "file://" + importSurfersFile.getAbsolutePath() + "?fileName=3-surfers-delete-test.csv&consumer.delay=10m&consumer.initialDelay=25s&move=.done"); + importConfigSurfersDelete.getProperties().put("source", "file://" + importSurfersFile.getAbsolutePath() + "?fileName=3-surfers-delete-test.csv&consumer.delay=10m&move=.done"); importConfigSurfersDelete.setActive(true); ImportConfiguration savedImportConfigDelete = importConfigurationService.save(importConfigSurfersDelete); @@ -111,7 +111,7 @@ public class ProfileImportSurfersDeleteIT extends BaseIT { } @Test - public void testCheckImportConfigList() { + public void testCheckImportConfigListSurfers() { List<ImportConfiguration> importConfigurations = importConfigurationService.getAll(); Assert.assertEquals(3, importConfigurations.size()); @@ -122,7 +122,7 @@ public class ProfileImportSurfersDeleteIT extends BaseIT { public void testImport3SurfersDelete() throws IOException, InterruptedException { //Wait for data to be processed - Thread.sleep(30000); + Thread.sleep(5000); Assert.assertEquals(3, profileService.getAllProfilesCount()); PartialList<Profile> jordyProfile = profileService.findProfilesByPropertyValue("properties.email", "[email protected]", 0, 10, null); http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/a8de6e1f/itests/src/test/java/org/apache/unomi/itests/ProfileImportSurfersIT.java ---------------------------------------------------------------------- diff --git a/itests/src/test/java/org/apache/unomi/itests/ProfileImportSurfersIT.java b/itests/src/test/java/org/apache/unomi/itests/ProfileImportSurfersIT.java index 58347f5..b67c78a 100644 --- a/itests/src/test/java/org/apache/unomi/itests/ProfileImportSurfersIT.java +++ b/itests/src/test/java/org/apache/unomi/itests/ProfileImportSurfersIT.java @@ -99,7 +99,7 @@ public class ProfileImportSurfersIT extends BaseIT { importConfigSurfers.getProperties().put("mapping", mappingSurfers); File importSurfersFile = new File("data/tmp/recurrent_import/"); - importConfigSurfers.getProperties().put("source", "file://" + importSurfersFile.getAbsolutePath() + "?fileName=2-surfers-test.csv&consumer.delay=10m&consumer.initialDelay=5s&move=.done"); + importConfigSurfers.getProperties().put("source", "file://" + importSurfersFile.getAbsolutePath() + "?fileName=2-surfers-test.csv&consumer.delay=10m&move=.done"); importConfigSurfers.setActive(true); ImportConfiguration savedImportConfig = importConfigurationService.save(importConfigSurfers); @@ -120,10 +120,10 @@ public class ProfileImportSurfersIT extends BaseIT { } @Test - public void testCheckAddedProperties() throws IOException, InterruptedException { + public void testCheckAddedPropertiesSurfers() throws IOException, InterruptedException { //Wait for data to be processed - Thread.sleep(5000); + Thread.sleep(1000); PropertyType propAlive = profileService.getPropertyType("alive"); http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/a8de6e1f/itests/src/test/resources/4-ranking-test.csv ---------------------------------------------------------------------- diff --git a/itests/src/test/resources/4-ranking-test.csv b/itests/src/test/resources/4-ranking-test.csv new file mode 100644 index 0000000..8545e7b --- /dev/null +++ b/itests/src/test/resources/4-ranking-test.csv @@ -0,0 +1,26 @@ +Rank;UCI ID;Name;Nationality;Age +1;10004451371;VAN AVERMAET Greg;BELGIUM;32 +2;10005460373;SAGAN Peter;SLOVAKIA;27 +3;10006738854;QUINTANA Nairo;COLOMBIA;27 +4;10001457509;VALVERDE Alejandro;SPAIN;37 +5;10004393676;FROOME Chris;GREAT BRITAIN;32 +6;10007389259;DUMOULIN Tom;NETHERLANDS;27 +7;10002416694;GILBERT Philippe;BELGIUM;35 +8;10005760164;PORTE Richie;AUSTRALIA;32 +9;10003224828;MARTIN Daniel;IRELAND;31 +10;10051842945;CONTADOR VELASCO Alberto;SPAIN;35 +11;10003269183;KRISTOFF Alexander;NORWAY;30 +12;10006291947;MATTHEWS Michael;AUSTRALIA;27 +13;10006491708;BARDET Romain;FRANCE;27 +14;10005379339;DEGENKOLB John;GERMANY;28 +15;10005467952;ULISSI Diego;ITALY;28 +16;10005852821;KWIATKOWSKI Michal;POLAND;27 +17;10009365433;NAESEN Oliver;BELGIUM;27 +18;10042236612;IZAGIRRE INSAUSTI Ion;SPAIN;28 +19;10008656828;GAVIRIA RENDON Fernando;COLOMBIA;23 +20;10007587303;CHAVES RUBIO Jhoan Esteban;COLOMBIA;27 +21;10007058954;YATES Simon;GREAT BRITAIN;25 +22;10006467052;DEMARE Arnaud;FRANCE;26 +23;10006468062;COLBRELLI Sonny;ITALY;27 +24;10005982456;FELLINE Fabio;ITALY;27 +25;10005961440;BOUHANNI Nacer;FRANCE;27 \ No newline at end of file
