incubator-unomi git commit: UNOMI-118 : Actors test, profile import
Repository: incubator-unomi Updated Branches: refs/heads/master bcb77c5c8 -> a597b5ea0 UNOMI-118 : Actors test, profile import Project: http://git-wip-us.apache.org/repos/asf/incubator-unomi/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-unomi/commit/a597b5ea Tree: http://git-wip-us.apache.org/repos/asf/incubator-unomi/tree/a597b5ea Diff: http://git-wip-us.apache.org/repos/asf/incubator-unomi/diff/a597b5ea Branch: refs/heads/master Commit: a597b5ea0996079ed23edbc13da4eda902fd1fbe Parents: bcb77c5 Author: Abdelkader Midani Authored: Tue Aug 15 00:26:46 2017 +0200 Committer: Abdelkader Midani Committed: Tue Aug 15 00:26:46 2017 +0200 -- .../java/org/apache/unomi/itests/AllITs.java| 3 +- .../java/org/apache/unomi/itests/BaseIT.java| 2 + .../unomi/itests/ProfileImportActorsIT.java | 182 +++ itests/src/test/resources/6-actors-test.csv | 7 + 4 files changed, 193 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/a597b5ea/itests/src/test/java/org/apache/unomi/itests/AllITs.java -- diff --git a/itests/src/test/java/org/apache/unomi/itests/AllITs.java b/itests/src/test/java/org/apache/unomi/itests/AllITs.java index 8251816..507a52e 100644 --- a/itests/src/test/java/org/apache/unomi/itests/AllITs.java +++ b/itests/src/test/java/org/apache/unomi/itests/AllITs.java @@ -37,7 +37,8 @@ import org.junit.runners.Suite.SuiteClasses; ProfileImportSurfersIT.class, ProfileImportSurfersOverwriteIT.class, ProfileImportSurfersDeleteIT.class, -ProfileImportRankingIT.class +ProfileImportRankingIT.class, +ProfileImportActorsIT.class }) public class AllITs { } http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/a597b5ea/itests/src/test/java/org/apache/unomi/itests/BaseIT.java -- diff --git a/itests/src/test/java/org/apache/unomi/itests/BaseIT.java b/itests/src/test/java/org/apache/unomi/itests/BaseIT.java index ae43a60..52f5abc 100644 --- a/itests/src/test/java/org/apache/unomi/itests/BaseIT.java +++ b/itests/src/test/java/org/apache/unomi/itests/BaseIT.java @@ -105,6 +105,8 @@ public abstract class BaseIT { "src/test/resources/4-surfers-delete-test.csv")), replaceConfigurationFile("data/tmp/recurrent_import/5-ranking-test.csv", new File( "src/test/resources/5-ranking-test.csv")), + replaceConfigurationFile("data/tmp/recurrent_import/6-actors-test.csv", new File( +"src/test/resources/6-actors-test.csv")), keepRuntimeFolder(), configureConsole().ignoreLocalConsole(), logLevel(LogLevel.INFO), http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/a597b5ea/itests/src/test/java/org/apache/unomi/itests/ProfileImportActorsIT.java -- diff --git a/itests/src/test/java/org/apache/unomi/itests/ProfileImportActorsIT.java b/itests/src/test/java/org/apache/unomi/itests/ProfileImportActorsIT.java new file mode 100644 index 000..8ac6478 --- /dev/null +++ b/itests/src/test/java/org/apache/unomi/itests/ProfileImportActorsIT.java @@ -0,0 +1,182 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License + */ +package org.apache.unomi.itests; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.http.HttpResponse; +import org.apache.http.client.methods.HttpPut; +import org.apache.http.entity.StringEntity; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClients; +import org.apache.unomi.api.Metadata; +import org.apache.unomi.api.PartialList; +import org.apache.unomi.api.Profile; +import org.apache.unomi.api.PropertyType; +import org.apache.unomi.api.services.ProfileService; +import org.apache.unomi.router.api.ImportConfiguration; +import org.apache.unomi.router.api
incubator-unomi git commit: UNOMI-101 : multi-value delimiter default value
Repository: incubator-unomi Updated Branches: refs/heads/master 2c32e4f5d -> bcb77c5c8 UNOMI-101 : multi-value delimiter default value Project: http://git-wip-us.apache.org/repos/asf/incubator-unomi/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-unomi/commit/bcb77c5c Tree: http://git-wip-us.apache.org/repos/asf/incubator-unomi/tree/bcb77c5c Diff: http://git-wip-us.apache.org/repos/asf/incubator-unomi/diff/bcb77c5c Branch: refs/heads/master Commit: bcb77c5c88e86231690ebbd2a77cd84226ec7367 Parents: 2c32e4f Author: Abdelkader Midani Authored: Tue Aug 15 00:24:55 2017 +0200 Committer: Abdelkader Midani Committed: Tue Aug 15 00:24:55 2017 +0200 -- .../core/processor/LineSplitProcessor.java | 19 +-- 1 file changed, 13 insertions(+), 6 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/bcb77c5c/extensions/router/router-core/src/main/java/org/apache/unomi/router/core/processor/LineSplitProcessor.java -- diff --git a/extensions/router/router-core/src/main/java/org/apache/unomi/router/core/processor/LineSplitProcessor.java b/extensions/router/router-core/src/main/java/org/apache/unomi/router/core/processor/LineSplitProcessor.java index 081a9ef..1e66374 100644 --- a/extensions/router/router-core/src/main/java/org/apache/unomi/router/core/processor/LineSplitProcessor.java +++ b/extensions/router/router-core/src/main/java/org/apache/unomi/router/core/processor/LineSplitProcessor.java @@ -33,10 +33,7 @@ import org.apache.unomi.router.core.exception.BadProfileDataFormatException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.UUID; +import java.util.*; /** * Created by amidani on 29/12/2016. @@ -105,6 +102,8 @@ public class LineSplitProcessor implements Processor { for (String fieldMappingKey : fieldsMapping.keySet()) { PropertyType propertyType = RouterUtils.getPropertyTypeById(profileService.getAllPropertyTypes("profiles"), fieldMappingKey); +logger.debug(" : LineSplitProcessor : PropType value : {}", profileData[fieldsMapping.get(fieldMappingKey)].trim()); + if (profileData.length > fieldsMapping.get(fieldMappingKey)) { try { if (propertyType.getValueTypeId().equals("string") || propertyType.getValueTypeId().equals("email")) { @@ -113,8 +112,16 @@ public class LineSplitProcessor implements Processor { if (StringUtils.isNotBlank(multiValueDelimiter) && multiValueDelimiter.length() == 2) { multivalueArray = multivalueArray.replaceAll("\\" + multiValueDelimiter.charAt(0), "").replaceAll("\\" + multiValueDelimiter.charAt(1), ""); } -String[] valuesArray = multivalueArray.split("\\" + multiValueSeparator); -properties.put(fieldMappingKey, valuesArray); + if(multivalueArray.contains(multiValueSeparator)) { +String[] valuesArray = multivalueArray.split("\\" + multiValueSeparator); +properties.put(fieldMappingKey, valuesArray); +} else { + if(StringUtils.isNotBlank(multivalueArray)) { +properties.put(fieldMappingKey, multivalueArray); +} else { +properties.put(fieldMappingKey, new String[]{}); +} +} } else { String singleValue = profileData[fieldsMapping.get(fieldMappingKey)].trim(); properties.put(fieldMappingKey, singleValue);
[jira] [Updated] (UNOMI-119) pastEventCondition hardcoded 5000 limit
[ https://issues.apache.org/jira/browse/UNOMI-119?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Don Hinshaw updated UNOMI-119: -- Description: https:///cxs/query/profile/count {code} { "type": "pastEventCondition", "parameterValues": { "eventCondition" : { "type" : "matchAllCondition", "parameterValues" : { } }, "numberOfDays" : 7 } } {code} in ElasticSearchPersistenceServiceImpl.java like 1485 there is a hardcoded limit of 5000. {{bucketsAggregation = AggregationBuilders.terms("buckets").field(aggregate.getField()).size(5000);}} This change was made on December 5 in commit 96a018ecb1b5b907930504e69f6da9357db8385e I think this value should be configurable. The way it is now, we are unable to get the count when our query has more that 5000 results. Thanks. was: https:///cxs/query/profile/count { "type": "pastEventCondition", "parameterValues": { "eventCondition" : { "type" : "matchAllCondition", "parameterValues" : { } }, "numberOfDays" : 7 } } in ElasticSearchPersistenceServiceImpl.java like 1485 there is a hardcoded limit of 5000. {{bucketsAggregation = AggregationBuilders.terms("buckets").field(aggregate.getField()).size(5000);}} This change was made on December 5 in commit 96a018ecb1b5b907930504e69f6da9357db8385e I think this value should be configurable. The way it is now, we are unable to get the count when our query has more that 5000 results. Thanks. > pastEventCondition hardcoded 5000 limit > --- > > Key: UNOMI-119 > URL: https://issues.apache.org/jira/browse/UNOMI-119 > Project: Apache Unomi > Issue Type: Improvement >Reporter: Don Hinshaw >Priority: Minor > > https:///cxs/query/profile/count > {code} > { > "type": "pastEventCondition", > "parameterValues": { > "eventCondition" : { > "type" : "matchAllCondition", > "parameterValues" : { > } > }, > "numberOfDays" : 7 > } > } > {code} > in ElasticSearchPersistenceServiceImpl.java like 1485 there is a hardcoded > limit of 5000. > {{bucketsAggregation = > AggregationBuilders.terms("buckets").field(aggregate.getField()).size(5000);}} > This change was made on December 5 in commit > 96a018ecb1b5b907930504e69f6da9357db8385e > I think this value should be configurable. The way it is now, we are unable > to get the count when our query has more that 5000 results. > Thanks. -- This message was sent by Atlassian JIRA (v6.4.14#64029)
[jira] [Updated] (UNOMI-119) pastEventCondition hardcoded 5000 limit
[ https://issues.apache.org/jira/browse/UNOMI-119?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Don Hinshaw updated UNOMI-119: -- Description: https:///cxs/query/profile/count {code} { "type": "pastEventCondition", "parameterValues": { "eventCondition" : { "type" : "matchAllCondition", "parameterValues" : { } }, "numberOfDays" : 7 } } {code} in ElasticSearchPersistenceServiceImpl.java like 1485 there is a hardcoded limit of 5000. {code}bucketsAggregation = AggregationBuilders.terms("buckets").field(aggregate.getField()).size(5000);{code} This change was made on December 5 in commit 96a018ecb1b5b907930504e69f6da9357db8385e I think this value should be configurable. The way it is now, we are unable to get the count when our query has more that 5000 results. Thanks. was: https:///cxs/query/profile/count {code} { "type": "pastEventCondition", "parameterValues": { "eventCondition" : { "type" : "matchAllCondition", "parameterValues" : { } }, "numberOfDays" : 7 } } {code} in ElasticSearchPersistenceServiceImpl.java like 1485 there is a hardcoded limit of 5000. {{bucketsAggregation = AggregationBuilders.terms("buckets").field(aggregate.getField()).size(5000);}} This change was made on December 5 in commit 96a018ecb1b5b907930504e69f6da9357db8385e I think this value should be configurable. The way it is now, we are unable to get the count when our query has more that 5000 results. Thanks. > pastEventCondition hardcoded 5000 limit > --- > > Key: UNOMI-119 > URL: https://issues.apache.org/jira/browse/UNOMI-119 > Project: Apache Unomi > Issue Type: Improvement >Reporter: Don Hinshaw >Priority: Minor > > https:///cxs/query/profile/count > {code} > { > "type": "pastEventCondition", > "parameterValues": { > "eventCondition" : { > "type" : "matchAllCondition", > "parameterValues" : { > } > }, > "numberOfDays" : 7 > } > } > {code} > in ElasticSearchPersistenceServiceImpl.java like 1485 there is a hardcoded > limit of 5000. > {code}bucketsAggregation = > AggregationBuilders.terms("buckets").field(aggregate.getField()).size(5000);{code} > This change was made on December 5 in commit > 96a018ecb1b5b907930504e69f6da9357db8385e > I think this value should be configurable. The way it is now, we are unable > to get the count when our query has more that 5000 results. > Thanks. -- This message was sent by Atlassian JIRA (v6.4.14#64029)
[jira] [Updated] (UNOMI-119) pastEventCondition hardcoded 5000 limit
[ https://issues.apache.org/jira/browse/UNOMI-119?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Don Hinshaw updated UNOMI-119: -- Description: https://{{host}}/cxs/query/profile/count {{{ "type": "pastEventCondition", "parameterValues": { "eventCondition" : { "type" : "matchAllCondition", "parameterValues" : { } }, "numberOfDays" : 7 } }}} in ElasticSearchPersistenceServiceImpl.java like 1485 there is a hardcoded limit of 5000. {{bucketsAggregation = AggregationBuilders.terms("buckets").field(aggregate.getField()).size(5000);}} This change was made on December 5 in commit 96a018ecb1b5b907930504e69f6da9357db8385e I think this value should be configurable. The way it is now, we are unable to get the count when our query has more that 5000 results. Thanks. was: https://{{host}}/cxs/query/profile/count {{ { "type": "pastEventCondition", "parameterValues": { "eventCondition" : { "type" : "matchAllCondition", "parameterValues" : { } }, "numberOfDays" : 7 } } }} in ElasticSearchPersistenceServiceImpl.java like 1485 there is a hardcoded limit of 5000. {{bucketsAggregation = AggregationBuilders.terms("buckets").field(aggregate.getField()).size(5000);}} This change was made on December 5 in commit 96a018ecb1b5b907930504e69f6da9357db8385e I think this value should be configurable. The way it is now, we are unable to get the count when our query has more that 5000 results. Thanks. > pastEventCondition hardcoded 5000 limit > --- > > Key: UNOMI-119 > URL: https://issues.apache.org/jira/browse/UNOMI-119 > Project: Apache Unomi > Issue Type: Improvement >Reporter: Don Hinshaw >Priority: Minor > > https://{{host}}/cxs/query/profile/count > {{{ > "type": "pastEventCondition", > "parameterValues": { > "eventCondition" : { > "type" : "matchAllCondition", > "parameterValues" : { > } > }, > "numberOfDays" : 7 > } > }}} > in ElasticSearchPersistenceServiceImpl.java like 1485 there is a hardcoded > limit of 5000. > {{bucketsAggregation = > AggregationBuilders.terms("buckets").field(aggregate.getField()).size(5000);}} > This change was made on December 5 in commit > 96a018ecb1b5b907930504e69f6da9357db8385e > I think this value should be configurable. The way it is now, we are unable > to get the count when our query has more that 5000 results. > Thanks. -- This message was sent by Atlassian JIRA (v6.4.14#64029)
[jira] [Updated] (UNOMI-119) pastEventCondition hardcoded 5000 limit
[ https://issues.apache.org/jira/browse/UNOMI-119?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Don Hinshaw updated UNOMI-119: -- Description: https:///cxs/query/profile/count { "type": "pastEventCondition", "parameterValues": { "eventCondition" : { "type" : "matchAllCondition", "parameterValues" : { } }, "numberOfDays" : 7 } } in ElasticSearchPersistenceServiceImpl.java like 1485 there is a hardcoded limit of 5000. {{bucketsAggregation = AggregationBuilders.terms("buckets").field(aggregate.getField()).size(5000);}} This change was made on December 5 in commit 96a018ecb1b5b907930504e69f6da9357db8385e I think this value should be configurable. The way it is now, we are unable to get the count when our query has more that 5000 results. Thanks. was: https://{{host}}/cxs/query/profile/count {{{ "type": "pastEventCondition", "parameterValues": { "eventCondition" : { "type" : "matchAllCondition", "parameterValues" : { } }, "numberOfDays" : 7 } }}} in ElasticSearchPersistenceServiceImpl.java like 1485 there is a hardcoded limit of 5000. {{bucketsAggregation = AggregationBuilders.terms("buckets").field(aggregate.getField()).size(5000);}} This change was made on December 5 in commit 96a018ecb1b5b907930504e69f6da9357db8385e I think this value should be configurable. The way it is now, we are unable to get the count when our query has more that 5000 results. Thanks. > pastEventCondition hardcoded 5000 limit > --- > > Key: UNOMI-119 > URL: https://issues.apache.org/jira/browse/UNOMI-119 > Project: Apache Unomi > Issue Type: Improvement >Reporter: Don Hinshaw >Priority: Minor > > https:///cxs/query/profile/count > { > "type": "pastEventCondition", > "parameterValues": { > "eventCondition" : { > "type" : "matchAllCondition", > "parameterValues" : { > } > }, > "numberOfDays" : 7 > } > } > in ElasticSearchPersistenceServiceImpl.java like 1485 there is a hardcoded > limit of 5000. > {{bucketsAggregation = > AggregationBuilders.terms("buckets").field(aggregate.getField()).size(5000);}} > This change was made on December 5 in commit > 96a018ecb1b5b907930504e69f6da9357db8385e > I think this value should be configurable. The way it is now, we are unable > to get the count when our query has more that 5000 results. > Thanks. -- This message was sent by Atlassian JIRA (v6.4.14#64029)
[jira] [Created] (UNOMI-119) pastEventCondition hardcoded 5000 limit
Don Hinshaw created UNOMI-119: - Summary: pastEventCondition hardcoded 5000 limit Key: UNOMI-119 URL: https://issues.apache.org/jira/browse/UNOMI-119 Project: Apache Unomi Issue Type: Improvement Reporter: Don Hinshaw Priority: Minor https://{{host}}/cxs/query/profile/count {{{ "type": "pastEventCondition", "parameterValues": { "eventCondition" : { "type" : "matchAllCondition", "parameterValues" : { } }, "numberOfDays" : 7 } }}} in ElasticSearchPersistenceServiceImpl.java like 1485 there is a hardcoded limit of 5000. {{bucketsAggregation = AggregationBuilders.terms("buckets").field(aggregate.getField()).size(5000);}} This change was made on December 5 in commit 96a018ecb1b5b907930504e69f6da9357db8385e I think this value should be configurable. The way it is now, we are unable to get the count when our query has more that 5000 results. Thanks. -- This message was sent by Atlassian JIRA (v6.4.14#64029)
[jira] [Updated] (UNOMI-119) pastEventCondition hardcoded 5000 limit
[ https://issues.apache.org/jira/browse/UNOMI-119?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Don Hinshaw updated UNOMI-119: -- Description: https://{{host}}/cxs/query/profile/count {{ { "type": "pastEventCondition", "parameterValues": { "eventCondition" : { "type" : "matchAllCondition", "parameterValues" : { } }, "numberOfDays" : 7 } } }} in ElasticSearchPersistenceServiceImpl.java like 1485 there is a hardcoded limit of 5000. {{bucketsAggregation = AggregationBuilders.terms("buckets").field(aggregate.getField()).size(5000);}} This change was made on December 5 in commit 96a018ecb1b5b907930504e69f6da9357db8385e I think this value should be configurable. The way it is now, we are unable to get the count when our query has more that 5000 results. Thanks. was: https://{{host}}/cxs/query/profile/count {{{ "type": "pastEventCondition", "parameterValues": { "eventCondition" : { "type" : "matchAllCondition", "parameterValues" : { } }, "numberOfDays" : 7 } }}} in ElasticSearchPersistenceServiceImpl.java like 1485 there is a hardcoded limit of 5000. {{bucketsAggregation = AggregationBuilders.terms("buckets").field(aggregate.getField()).size(5000);}} This change was made on December 5 in commit 96a018ecb1b5b907930504e69f6da9357db8385e I think this value should be configurable. The way it is now, we are unable to get the count when our query has more that 5000 results. Thanks. > pastEventCondition hardcoded 5000 limit > --- > > Key: UNOMI-119 > URL: https://issues.apache.org/jira/browse/UNOMI-119 > Project: Apache Unomi > Issue Type: Improvement >Reporter: Don Hinshaw >Priority: Minor > > https://{{host}}/cxs/query/profile/count > {{ > { > "type": "pastEventCondition", > "parameterValues": { > "eventCondition" : { > "type" : "matchAllCondition", > "parameterValues" : { > } > }, > "numberOfDays" : 7 > } > } > }} > in ElasticSearchPersistenceServiceImpl.java like 1485 there is a hardcoded > limit of 5000. > {{bucketsAggregation = > AggregationBuilders.terms("buckets").field(aggregate.getField()).size(5000);}} > This change was made on December 5 in commit > 96a018ecb1b5b907930504e69f6da9357db8385e > I think this value should be configurable. The way it is now, we are unable > to get the count when our query has more that 5000 results. > Thanks. -- This message was sent by Atlassian JIRA (v6.4.14#64029)