UNOMI-118 : Surfers test, profile import, and rename Basic test
Project: http://git-wip-us.apache.org/repos/asf/incubator-unomi/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-unomi/commit/b0286342 Tree: http://git-wip-us.apache.org/repos/asf/incubator-unomi/tree/b0286342 Diff: http://git-wip-us.apache.org/repos/asf/incubator-unomi/diff/b0286342 Branch: refs/heads/master Commit: b02863425db8ce65ceedfbc7cb835507024dd6ee Parents: 5c99fb2 Author: Abdelkader Midani <[email protected]> Authored: Thu Aug 10 02:20:34 2017 +0200 Committer: Abdelkader Midani <[email protected]> Committed: Thu Aug 10 02:20:34 2017 +0200 ---------------------------------------------------------------------- .../java/org/apache/unomi/itests/AllITs.java | 3 +- .../java/org/apache/unomi/itests/BaseIT.java | 2 + .../unomi/itests/ProfileImportBasicIT.java | 114 ++++++++++++ .../apache/unomi/itests/ProfileImportIT.java | 107 ----------- .../unomi/itests/ProfileImportSurfersIT.java | 177 +++++++++++++++++++ itests/src/test/resources/2-surfers-test.csv | 37 ++++ 6 files changed, 332 insertions(+), 108 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/b0286342/itests/src/test/java/org/apache/unomi/itests/AllITs.java ---------------------------------------------------------------------- diff --git a/itests/src/test/java/org/apache/unomi/itests/AllITs.java b/itests/src/test/java/org/apache/unomi/itests/AllITs.java index c2b6fd3..225513e 100644 --- a/itests/src/test/java/org/apache/unomi/itests/AllITs.java +++ b/itests/src/test/java/org/apache/unomi/itests/AllITs.java @@ -33,7 +33,8 @@ import org.junit.runners.Suite.SuiteClasses; ConditionESQueryBuilderIT.class, SegmentIT.class, ProfileServiceIT.class, - ProfileImportIT.class + ProfileImportBasicIT.class, + ProfileImportSurfersIT.class }) public class AllITs { } http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/b0286342/itests/src/test/java/org/apache/unomi/itests/BaseIT.java ---------------------------------------------------------------------- diff --git a/itests/src/test/java/org/apache/unomi/itests/BaseIT.java b/itests/src/test/java/org/apache/unomi/itests/BaseIT.java index fa56a74..38a2415 100644 --- a/itests/src/test/java/org/apache/unomi/itests/BaseIT.java +++ b/itests/src/test/java/org/apache/unomi/itests/BaseIT.java @@ -97,6 +97,8 @@ public abstract class BaseIT { "src/test/resources/org.apache.unomi.router.cfg")), replaceConfigurationFile("data/tmp/unomi_oneshot_import_configs/1-basic-test.csv", new File( "src/test/resources/1-basic-test.csv")), + replaceConfigurationFile("data/tmp/recurrent_import/2-surfers-test.csv", new File( + "src/test/resources/2-surfers-test.csv")), keepRuntimeFolder(), configureConsole().ignoreLocalConsole(), logLevel(LogLevel.INFO), http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/b0286342/itests/src/test/java/org/apache/unomi/itests/ProfileImportBasicIT.java ---------------------------------------------------------------------- diff --git a/itests/src/test/java/org/apache/unomi/itests/ProfileImportBasicIT.java b/itests/src/test/java/org/apache/unomi/itests/ProfileImportBasicIT.java new file mode 100644 index 0000000..a147029 --- /dev/null +++ b/itests/src/test/java/org/apache/unomi/itests/ProfileImportBasicIT.java @@ -0,0 +1,114 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License + */ +package org.apache.unomi.itests; + +import org.apache.unomi.api.PartialList; +import org.apache.unomi.api.Profile; +import org.apache.unomi.api.PropertyType; +import org.apache.unomi.api.services.ProfileService; +import org.apache.unomi.router.api.ImportConfiguration; +import org.apache.unomi.router.api.RouterConstants; +import org.apache.unomi.router.api.services.ImportExportConfigurationService; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.ops4j.pax.exam.junit.PaxExam; +import org.ops4j.pax.exam.spi.reactors.ExamReactorStrategy; +import org.ops4j.pax.exam.spi.reactors.PerSuite; +import org.ops4j.pax.exam.util.Filter; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.inject.Inject; +import java.io.IOException; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * Created by amidani on 03/08/2017. + */ +@RunWith(PaxExam.class) +@ExamReactorStrategy(PerSuite.class) +public class ProfileImportBasicIT extends BaseIT { + + @Inject + @Filter("(configDiscriminator=IMPORT)") + protected ImportExportConfigurationService<ImportConfiguration> importConfigurationService; + + @Inject + protected ProfileService profileService; + private Logger logger = LoggerFactory.getLogger(ProfileImportBasicIT.class); + + + @Before + public void setUp() { + + /*** Basic Test ***/ + ImportConfiguration importConfiguration = new ImportConfiguration(); + importConfiguration.setItemId("1-basic-test"); + importConfiguration.setConfigType(RouterConstants.IMPORT_EXPORT_CONFIG_TYPE_ONESHOT); + importConfiguration.setMergingProperty("email"); + importConfiguration.setOverwriteExistingProfiles(true); + + Map mapping = new HashMap(); + mapping.put("email", 0); + mapping.put("firstName", 1); + mapping.put("lastName", 2); + + importConfiguration.getProperties().put("mapping", mapping); + importConfiguration.setActive(true); + + importConfigurationService.save(importConfiguration); + + } + + @Test + public void testCheckImportConfigList() { + List<ImportConfiguration> importConfigurations = importConfigurationService.getAll(); + Assert.assertEquals(1, importConfigurations.size()); + } + + @Test + public void testImport1Basic() throws IOException, InterruptedException { + + //Wait for the csv to be processed + Thread.sleep(10000); + + //Check saved profiles + PartialList<Profile> profiles = profileService.findProfilesByPropertyValue("properties.email", "[email protected]", 0, 10, null); + Assert.assertEquals(3, profileService.getAllProfilesCount()); + Assert.assertEquals(1, profiles.getList().size()); + Assert.assertNotNull(profiles.get(0)); + Assert.assertEquals("Basic1", profiles.get(0).getProperty("firstName")); + Assert.assertEquals("User1", profiles.get(0).getProperty("lastName")); + + //Check import config status + ImportConfiguration importConfiguration = importConfigurationService.load("1-basic-test"); + Assert.assertEquals(RouterConstants.CONFIG_STATUS_COMPLETE_SUCCESS, importConfiguration.getStatus()); + Assert.assertEquals(1, importConfiguration.getExecutions().size()); + + } + + @After + public void tearDown() { + + } + +} http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/b0286342/itests/src/test/java/org/apache/unomi/itests/ProfileImportIT.java ---------------------------------------------------------------------- diff --git a/itests/src/test/java/org/apache/unomi/itests/ProfileImportIT.java b/itests/src/test/java/org/apache/unomi/itests/ProfileImportIT.java deleted file mode 100644 index afd3ce9..0000000 --- a/itests/src/test/java/org/apache/unomi/itests/ProfileImportIT.java +++ /dev/null @@ -1,107 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License - */ -package org.apache.unomi.itests; - -import org.apache.unomi.api.PartialList; -import org.apache.unomi.api.Profile; -import org.apache.unomi.api.services.ProfileService; -import org.apache.unomi.router.api.ImportConfiguration; -import org.apache.unomi.router.api.RouterConstants; -import org.apache.unomi.router.api.services.ImportExportConfigurationService; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.ops4j.pax.exam.junit.PaxExam; -import org.ops4j.pax.exam.spi.reactors.ExamReactorStrategy; -import org.ops4j.pax.exam.spi.reactors.PerSuite; -import org.ops4j.pax.exam.util.Filter; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import javax.inject.Inject; -import java.io.IOException; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -/** - * Created by amidani on 03/08/2017. - */ -@RunWith(PaxExam.class) -@ExamReactorStrategy(PerSuite.class) -public class ProfileImportIT extends BaseIT { - - @Inject - @Filter("(configDiscriminator=IMPORT)") - protected ImportExportConfigurationService<ImportConfiguration> importConfigurationService; - @Inject - protected ProfileService profileService; - private Logger logger = LoggerFactory.getLogger(ProfileImportIT.class); - - @Before - public void setUp() { - - /*** Basic Test ***/ - ImportConfiguration importConfiguration = new ImportConfiguration(); - importConfiguration.setItemId("1-basic-test"); - importConfiguration.setConfigType(RouterConstants.IMPORT_EXPORT_CONFIG_TYPE_ONESHOT); - importConfiguration.setMergingProperty("email"); - importConfiguration.setOverwriteExistingProfiles(true); - - Map mapping = new HashMap(); - mapping.put("email", 0); - mapping.put("firstName", 1); - mapping.put("lastName", 2); - - importConfiguration.getProperties().put("mapping", mapping); - importConfiguration.setActive(true); - - importConfigurationService.save(importConfiguration); - - } - - @Test - public void testGetImportConfigList() { - List<ImportConfiguration> importConfigurations = importConfigurationService.getAll(); - Assert.assertEquals(1, importConfigurations.size()); - } - - @Test - public void testBasic() throws IOException, InterruptedException { - - //Wait for the csv to be processed - Thread.sleep(10000); - - //Check saved profiles - PartialList<Profile> profiles = profileService.findProfilesByPropertyValue("properties.email", "[email protected]", 0, 10, null); - logger.info("$$$$ : Profile count : " + profileService.getAllProfilesCount()); - logger.info("$$$$ : List size : " + profiles.getList().size()); - Assert.assertEquals(3, profileService.getAllProfilesCount()); - Assert.assertEquals(1, profiles.getList().size()); - Assert.assertNotNull(profiles.get(0)); - Assert.assertEquals("Basic1", profiles.get(0).getProperty("firstName")); - Assert.assertEquals("User1", profiles.get(0).getProperty("lastName")); - - //Check import config status - ImportConfiguration importConfiguration = importConfigurationService.load("1-basic-test"); - Assert.assertEquals(RouterConstants.CONFIG_STATUS_COMPLETE_SUCCESS, importConfiguration.getStatus()); - Assert.assertEquals(1, importConfiguration.getExecutions().size()); - - } - -} http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/b0286342/itests/src/test/java/org/apache/unomi/itests/ProfileImportSurfersIT.java ---------------------------------------------------------------------- diff --git a/itests/src/test/java/org/apache/unomi/itests/ProfileImportSurfersIT.java b/itests/src/test/java/org/apache/unomi/itests/ProfileImportSurfersIT.java new file mode 100644 index 0000000..1be2a23 --- /dev/null +++ b/itests/src/test/java/org/apache/unomi/itests/ProfileImportSurfersIT.java @@ -0,0 +1,177 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License + */ +package org.apache.unomi.itests; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.http.HttpRequest; +import org.apache.http.HttpResponse; +import org.apache.http.auth.AuthenticationException; +import org.apache.http.auth.UsernamePasswordCredentials; +import org.apache.http.client.entity.UrlEncodedFormEntity; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.client.methods.HttpPut; +import org.apache.http.client.methods.HttpUriRequest; +import org.apache.http.entity.StringEntity; +import org.apache.http.impl.auth.BasicScheme; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClientBuilder; +import org.apache.http.impl.client.HttpClients; +import org.apache.unomi.api.Metadata; +import org.apache.unomi.api.PartialList; +import org.apache.unomi.api.Profile; +import org.apache.unomi.api.PropertyType; +import org.apache.unomi.api.services.ProfileService; +import org.apache.unomi.router.api.ImportConfiguration; +import org.apache.unomi.router.api.RouterConstants; +import org.apache.unomi.router.api.services.ImportExportConfigurationService; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.ops4j.pax.exam.junit.PaxExam; +import org.ops4j.pax.exam.spi.reactors.ExamReactorStrategy; +import org.ops4j.pax.exam.spi.reactors.PerSuite; +import org.ops4j.pax.exam.util.Filter; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.inject.Inject; +import javax.ws.rs.core.MediaType; +import java.io.File; +import java.io.IOException; +import java.util.*; + +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.MatcherAssert.assertThat; + +/** + * Created by amidani on 09/08/2017. + */ +@RunWith(PaxExam.class) +@ExamReactorStrategy(PerSuite.class) +public class ProfileImportSurfersIT extends BaseIT { + + @Inject + @Filter("(configDiscriminator=IMPORT)") + protected ImportExportConfigurationService<ImportConfiguration> importConfigurationService; + + @Inject + protected ProfileService profileService; + private Logger logger = LoggerFactory.getLogger(ProfileImportSurfersIT.class); + + + @Before + public void setUp() throws IOException, AuthenticationException { + + /*** Create Missing Properties ***/ + PropertyType propertyType = new PropertyType(new Metadata("integration", "alive", "Alive", "Is the person alive?")); + propertyType.setValueTypeId("boolean"); + propertyType.setTagIds(Collections.singleton("basicProfileProperties")); + propertyType.setTarget("profiles"); + + profileService.setPropertyType(propertyType); + + /*** Surfer Test ***/ + ImportConfiguration importConfigSurfers = new ImportConfiguration(); + importConfigSurfers.setItemId("2-surfers-test"); + importConfigSurfers.setConfigType(RouterConstants.IMPORT_EXPORT_CONFIG_TYPE_RECURRENT); + importConfigSurfers.setMergingProperty("linkedInId"); + importConfigSurfers.setOverwriteExistingProfiles(true); + importConfigSurfers.setColumnSeparator(";"); + importConfigSurfers.setHasHeader(true); + importConfigSurfers.setHasDeleteColumn(true); + + Map mappingSurfers = new HashMap(); + mappingSurfers.put("linkedInId", 0); + mappingSurfers.put("lastName", 1); + mappingSurfers.put("email", 2); + mappingSurfers.put("facebookId", 3); + mappingSurfers.put("gender", 4); + mappingSurfers.put("alive", 5); + + importConfigSurfers.getProperties().put("mapping", mappingSurfers); + File importSurfersFile = new File("data/tmp/recurrent_import/"); + importConfigSurfers.getProperties().put("source", "file://" + importSurfersFile.getAbsolutePath() + "?fileName=2-surfers-test.csv&consumer.delay=10m&consumer.initialDelay=5s&move=.done"); + importConfigSurfers.setActive(true); + + ImportConfiguration savedImportConfig = importConfigurationService.save(importConfigSurfers); + + CloseableHttpClient httpclient = HttpClients.createDefault(); + HttpPut httpPut = new HttpPut(URL + "/configUpdate/importConfigAdmin"); + + String json = new ObjectMapper().writeValueAsString(savedImportConfig); + StringEntity entity = new StringEntity(json); + entity.setContentType(MediaType.APPLICATION_JSON); + httpPut.setEntity(entity); + + HttpResponse response = httpclient.execute(httpPut); + assertThat(response.getStatusLine().getStatusCode(), equalTo(200)); + httpclient.close(); + + } + + @Test + public void testCheckAddedProperties() throws IOException, InterruptedException { + + //Wait for data to be processed + Thread.sleep(5000); + + PropertyType propAlive = profileService.getPropertyType("alive"); + + Assert.assertNotNull(propAlive); + + } + + @Test + public void testCheckImportConfigList() { + + List<ImportConfiguration> importConfigurations = importConfigurationService.getAll(); + Assert.assertEquals(2, importConfigurations.size()); + + } + + @Test + public void testImport2Surfers() throws IOException, InterruptedException { + + //Wait for data to be processed + Thread.sleep(10000); + + Assert.assertEquals(37, profileService.getAllProfilesCount()); + + //Profile not to delete + PartialList<Profile> jordyProfile = profileService.findProfilesByPropertyValue("properties.email", "[email protected]", 0, 10, null); + Assert.assertEquals(1, jordyProfile.getList().size()); + Assert.assertNotNull(jordyProfile.get(0)); + Assert.assertEquals("1", jordyProfile.get(0).getProperty("linkedInId")); + Assert.assertEquals("Jordy Smith", jordyProfile.get(0).getProperty("lastName")); + Assert.assertEquals("999", jordyProfile.get(0).getProperty("facebookId")); + Assert.assertEquals("male", jordyProfile.get(0).getProperty("gender")); + Assert.assertTrue((Boolean) jordyProfile.get(0).getProperty("alive")); + + //Profile to delete + PartialList<Profile> paulineProfile = profileService.findProfilesByPropertyValue("properties.lastName", "Pauline Ado", 0, 10, null); + Assert.assertEquals(0, paulineProfile.getList().size()); + + //Check import config status + ImportConfiguration importConfiguration = importConfigurationService.load("2-surfers-test"); + Assert.assertEquals(RouterConstants.CONFIG_STATUS_COMPLETE_SUCCESS, importConfiguration.getStatus()); + Assert.assertEquals(1, importConfiguration.getExecutions().size()); + + } + +} http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/b0286342/itests/src/test/resources/2-surfers-test.csv ---------------------------------------------------------------------- diff --git a/itests/src/test/resources/2-surfers-test.csv b/itests/src/test/resources/2-surfers-test.csv new file mode 100644 index 0000000..bee4ef9 --- /dev/null +++ b/itests/src/test/resources/2-surfers-test.csv @@ -0,0 +1,37 @@ +Linkedin ID;Last Name;Email;Facebook ID;Gender;Alive;To Delete +1;Jordy Smith;[email protected];999;male;TRUE;FALSE +2;Gabriel Medina;[email protected];998;male;TRUE;FALSE +3;Kolohe Andino;[email protected];997;male;TRUE;FALSE +4;Matt Wilkinson;[email protected];996;male;TRUE;FALSE +5;Michel Bourez;[email protected];995;male;TRUE;FALSE +6;Kelly Slater;[email protected];994;male;TRUE;FALSE +7;Julian Wilson;[email protected];993;male;TRUE;FALSE +8;Joel Parkinson;[email protected];992;male;TRUE;FALSE +9;Filipe Toledo;[email protected];991;male;TRUE;FALSE +10;Adriano De Souza;[email protected];990;male;TRUE;FALSE +11;Sebastian Zietz;[email protected];989;male;TRUE;TRUE +12;Josh Kerr;[email protected];988;male;TRUE;FALSE +13;Adrian Buchan;[email protected];987;male;TRUE;FALSE +14;Italo Ferreira;[email protected];986;male;TRUE;FALSE +15;Mick Fanning;[email protected];985;male;TRUE;FALSE +16;Miguel Pupo;[email protected];984;male;TRUE;FALSE +17;Nat Young;[email protected];983;male;TRUE;FALSE +18;Keanu Asing;;982;male;TRUE;FALSE +19;Tyler Wright;;981;male;TRUE;FALSE +20;Courtney Conlogue;;980;female;TRUE;FALSE +21;Carissa Moore;;979;female;TRUE;FALSE +22;Tatiana Weston-Webb;;978;female;TRUE;FALSE +23;Johanne Defay;;977;female;TRUE;FALSE +24;Stephanie Gilmore;;976;female;TRUE;FALSE +25;Malia Manuel;;975;female;TRUE;FALSE +26;Sally Fitzgibbons;;974;female;TRUE;FALSE +27;Sage Erickson;;973;female;TRUE;FALSE +28;Laura Enever;;972;female;TRUE;FALSE +29;Nikki Van Dijk;;971;female;TRUE;FALSE +30;Bianca Buitendag;;970;female;TRUE;FALSE +31;Coco Ho;;969;female;TRUE;FALSE +32;Alessa Quizon;;968;female;TRUE;FALSE +33;Lakey Peterson;;967;female;TRUE;FALSE +34;Bronte Macaulay;;966;female;TRUE;FALSE +35;Pauline Ado;;965;female;TRUE;TRUE +36;Silvana Lima;;964;female;TRUE;FALSE \ No newline at end of file
