Repository: incubator-unomi Updated Branches: refs/heads/UNOMI-28-ES-2-X-UPGRADE 12641b9b3 -> fda0d9ce7
Use ElasticSearch's re-introduced support for dots in names and remove the old dot escaping code. Project: http://git-wip-us.apache.org/repos/asf/incubator-unomi/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-unomi/commit/fda0d9ce Tree: http://git-wip-us.apache.org/repos/asf/incubator-unomi/tree/fda0d9ce Diff: http://git-wip-us.apache.org/repos/asf/incubator-unomi/diff/fda0d9ce Branch: refs/heads/UNOMI-28-ES-2-X-UPGRADE Commit: fda0d9ce711ff189a8f4af42711d17a5d9d020ed Parents: 12641b9 Author: Serge Huber <[email protected]> Authored: Thu Sep 22 16:03:18 2016 +0200 Committer: Serge Huber <[email protected]> Committed: Thu Sep 22 16:03:18 2016 +0200 ---------------------------------------------------------------------- .../ElasticSearchPersistenceServiceImpl.java | 13 +- .../elasticsearch/FieldDotEscaper.java | 108 ------- .../elasticsearch/FieldDotJsonTransformer.java | 290 ------------------- .../elasticsearch/FieldDotEscapeTest.java | 95 ------ .../FieldDotJsonTransformerTest.java | 61 ---- .../core/src/test/resources/complex.json | 50 ---- .../core/src/test/resources/profile.json | 1 - 7 files changed, 6 insertions(+), 612 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/fda0d9ce/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java ---------------------------------------------------------------------- diff --git a/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java b/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java index b5f1c7a..16f1bc6 100644 --- a/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java +++ b/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java @@ -132,6 +132,7 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService, public static final String PATH_HOME = "path.home"; public static final String PATH_PLUGINS = "path.plugins"; public static final String INDEX_MAX_RESULT_WINDOW = "index.max_result_window"; + public static final String MAPPER_ALLOW_DOTS_IN_NAME = "mapper.allow_dots_in_name"; private Node node; private Client client; @@ -281,6 +282,9 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService, File homeFile = new File(getConfig(settings, PATH_HOME, new File(new File(karafHome), ELASTICSEARCH_HOME_DIRECTORY).getAbsolutePath())); File dataFile = new File(getConfig(settings, PATH_DATA, new File(new File(karafHome), ELASTICSEARCH_DATA_DIRECTORY).getAbsolutePath())); + // allow dots in mappings (re-introduced in ElasticSearch 2.4.0) + System.setProperty(MAPPER_ALLOW_DOTS_IN_NAME, "true"); + settingsBuilder.put(CLUSTER_NAME, clusterName) .put(NODE_DATA, nodeData) .put(PATH_DATA, dataFile.getAbsolutePath()) @@ -528,7 +532,7 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService, .actionGet(); if (response.isExists()) { String sourceAsString = response.getSourceAsString(); - final T value = CustomObjectMapper.getObjectMapper().readValue(FieldDotEscaper.unescapeJson(sourceAsString), clazz); + final T value = CustomObjectMapper.getObjectMapper().readValue(sourceAsString, clazz); value.setItemId(response.getId()); return value; } else { @@ -555,11 +559,6 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService, protected Boolean execute(Object... args) { try { String source = CustomObjectMapper.getObjectMapper().writeValueAsString(item); - Set<String> modifiedNames = new LinkedHashSet<>(); - source = FieldDotEscaper.escapeJson(source, modifiedNames); - if (modifiedNames.size() > 0) { - logger.warn("Found JSON property names with dot characters not allowed by ElasticSearch 2.x={} in item {}", modifiedNames, item); - } String itemType = item.getItemType(); String index = indexNames.containsKey(itemType) ? indexNames.get(itemType) : (itemsMonthlyIndexed.contains(itemType) ? getMonthlyIndex(((TimestampedItem) item).getTimeStamp()) : indexName); @@ -606,7 +605,7 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService, String index = indexNames.containsKey(itemType) ? indexNames.get(itemType) : (itemsMonthlyIndexed.contains(itemType) && dateHint != null ? getMonthlyIndex(dateHint) : indexName); - client.prepareUpdate(index, itemType, itemId).setDoc(FieldDotEscaper.escapeMap(source)) + client.prepareUpdate(index, itemType, itemId).setDoc(source) .execute() .actionGet(); return true; http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/fda0d9ce/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/FieldDotEscaper.java ---------------------------------------------------------------------- diff --git a/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/FieldDotEscaper.java b/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/FieldDotEscaper.java deleted file mode 100644 index 62ccacc..0000000 --- a/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/FieldDotEscaper.java +++ /dev/null @@ -1,108 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.unomi.persistence.elasticsearch; - -import java.util.LinkedHashMap; -import java.util.Map; -import java.util.Properties; -import java.util.Set; - -/** - * A utility class to un/escape field names that contain dots that are not allowed in ElasticSearch 2.x - */ -public class FieldDotEscaper { - - public static final String DOT_ESCAPE_MARKER = "__DOT__"; - - public static String escapeJson(final String jsonInput) { - return escapeJson(jsonInput, null); - } - - public static String escapeJson(final String jsonInput, final Set<String> modifiedNames) { - if (!jsonInput.contains(".")) { // optimization in case no dot is present at all - return jsonInput; - } - StringBuffer result = new StringBuffer(); - FieldDotJsonTransformer jsonTransformer = new FieldDotJsonTransformer(jsonInput, result, DOT_ESCAPE_MARKER); - Set<String> pathsModified = jsonTransformer.transform(); - if (modifiedNames != null) { - modifiedNames.addAll(pathsModified); - } - return result.toString(); - } - - public static String unescapeJson(final String jsonInput) { - return unescapeString(jsonInput); - } - - public static String escapeString(final String stringInput) { - return stringInput.replaceAll("\\.", DOT_ESCAPE_MARKER); - } - - public static String unescapeString(final String stringInput) { - return stringInput.replaceAll(DOT_ESCAPE_MARKER, "."); - } - - public static Map<? extends String, ?> escapeMap(final Map<? extends String, ?> mapInput) { - Map<String,Object> result = new LinkedHashMap<>(mapInput.size()); - for (Map.Entry<? extends String, ? extends Object> entry : mapInput.entrySet()) { - String entryKey = entry.getKey(); - if (entryKey.contains(".")) { - entryKey = escapeString(entryKey); - } - result.put(entryKey, entry.getValue()); - } - return result; - } - - public static Map<? extends String, ?> unescapeMap(final Map<? extends String, ?> mapInput) { - Map<String, Object> result = new LinkedHashMap<>(mapInput.size()); - for (Map.Entry<? extends String, ?> entry : mapInput.entrySet()) { - String entryKey = entry.getKey(); - if (entryKey.contains(DOT_ESCAPE_MARKER)) { - entryKey = unescapeString(entryKey); - } - result.put(entryKey, entry.getValue()); - } - return result; - } - - public static Properties escapeProperties(final Properties input) { - Properties result = new Properties(); - for (String propertyName : input.stringPropertyNames()) { - String newPropertyName = propertyName; - if (propertyName.contains(".")) { - newPropertyName = escapeString(propertyName); - } - result.put(newPropertyName, input.getProperty(propertyName)); - } - return result; - } - - public static Properties unescapeProperties(final Properties input) { - Properties result = new Properties(); - for (String propertyName : input.stringPropertyNames()) { - String newPropertyName = propertyName; - if (propertyName.contains(DOT_ESCAPE_MARKER)) { - newPropertyName = unescapeString(propertyName); - } - result.put(newPropertyName, input.getProperty(propertyName)); - } - return result; - } - -} http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/fda0d9ce/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/FieldDotJsonTransformer.java ---------------------------------------------------------------------- diff --git a/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/FieldDotJsonTransformer.java b/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/FieldDotJsonTransformer.java deleted file mode 100644 index a26e3f4..0000000 --- a/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/FieldDotJsonTransformer.java +++ /dev/null @@ -1,290 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.unomi.persistence.elasticsearch; - -import org.apache.commons.lang3.StringUtils; - -import java.util.Deque; -import java.util.LinkedHashSet; -import java.util.LinkedList; -import java.util.Set; - -/** - * A custom JSON transformer that can replace dot characters in field names with a marker. This is useful for tools like - * ElasticSearch 2.x that doesn't allow dot characters in field names since version 2.x - */ -public class FieldDotJsonTransformer { - - public static final char BEGIN_ARRAY_CHAR = '['; - public static final char END_ARRAY_CHAR = ']'; - public static final char BEGIN_OBJECT_CHAR = '{'; - public static final char END_OBJECT_CHAR = '}'; - public static final char NAME_SEPARATOR_CHAR = ':'; - public static final char VALUE_SEPARATOR_CHAR = ','; - - public static final char STRING_BEGIN_OR_END_CHAR = '"'; - public static final char STRING_ESCAPE_CHAR = '\\'; - public static final char STRING_UNICODE_CHAR = 'u'; - public static final char STRING_DOT_CHAR = '.'; - - public static final String WHITESPACE_CHARS = " \t\n\r"; - public static final String NUMBER_CHARS = "+-0123456789eE."; - - String jsonInput; - int pos = -1; - StringBuffer output; - String dotReplacement = null; - Set<String> modifiedNames = new LinkedHashSet<>(); - Deque<String> currentPath = new LinkedList<>(); - - public FieldDotJsonTransformer(String jsonInput, StringBuffer output, String dotReplacement) { - this.jsonInput = jsonInput; - this.output = output; - this.dotReplacement = dotReplacement; - } - - public Set<String> transform() { - parseValue(); - return modifiedNames; - } - - protected Character getNextChar() { - pos++; - char ch = jsonInput.charAt(pos); - if (pos >= jsonInput.length()) { - return null; - } - return ch; - } - - protected Character peekNextToken() { - if (pos + 1 >= jsonInput.length()) { - return null; - } - int i = 1; - Character ch = jsonInput.charAt(pos + i); - while (WHITESPACE_CHARS.indexOf(ch) > -1 && (pos + i < jsonInput.length())) { - i++; - ch = jsonInput.charAt(pos + i); - } - return ch; - } - - protected Character getNextToken() { - Character ch = getNextChar(); - while ((ch != null) && (WHITESPACE_CHARS.indexOf(ch) > -1)) { - output.append(ch); - ch = getNextChar(); - } - return ch; - } - - protected void parseBooleanValue(boolean expectedValue) { - if (expectedValue) { - StringBuilder sb = new StringBuilder(); - sb.append(getNextToken()); - sb.append(getNextChar()); - sb.append(getNextChar()); - sb.append(getNextChar()); - if ("true".equals(sb.toString())) { - // everything matches - } - output.append(sb.toString()); - } else { - StringBuilder sb = new StringBuilder(); - sb.append(getNextToken()); - sb.append(getNextChar()); - sb.append(getNextChar()); - sb.append(getNextChar()); - sb.append(getNextChar()); - if ("false".equals(sb.toString())) { - // everything matches - } - output.append(sb.toString()); - } - } - - protected void parseNullValue() { - StringBuilder sb = new StringBuilder(); - sb.append(getNextToken()); - sb.append(getNextChar()); - sb.append(getNextChar()); - sb.append(getNextChar()); - if ("null".equals(sb.toString())) { - // everything matches - } - output.append(sb.toString()); - } - - protected String parseString(boolean escapeDots) { - Character ch = getNextToken(); - if (ch != STRING_BEGIN_OR_END_CHAR) { - return null; - } - output.append(ch); - boolean modified = false; - StringBuilder stringContent = new StringBuilder(); - while ((ch = getNextChar()) != null) { - switch (ch) { - case STRING_ESCAPE_CHAR: - stringContent.append(ch); - output.append(ch); - ch = getNextChar(); - if (ch == STRING_UNICODE_CHAR) { - // case of Unicode escape sequence - } - output.append(ch); - stringContent.append(ch); - break; - case STRING_DOT_CHAR: - if (escapeDots && dotReplacement != null) { - output.append(dotReplacement); - modified = true; - } else { - output.append(ch); - } - stringContent.append(ch); - break; - case STRING_BEGIN_OR_END_CHAR: - output.append(ch); - if (modified) { - modifiedNames.add(StringUtils.join(currentPath, "/") + "/" + stringContent.toString()); - } - return stringContent.toString(); - default: - output.append(ch); - stringContent.append(ch); - } - } - return null; - } - - protected void parseNumber() { - StringBuilder sb = new StringBuilder(); - Character ch = peekNextToken(); - while ((ch != null) && (NUMBER_CHARS.indexOf(ch) > -1)) { - ch = getNextChar(); - sb.append(ch); - ch = peekNextToken(); - } - output.append(sb.toString()); - } - - protected void parseValue() { - char ch = peekNextToken(); - // we've got to identify the type or value first - switch (ch) { - case 't': // true - parseBooleanValue(true); - break; - case 'f': // false - parseBooleanValue(false); - break; - case 'n': // null - parseNullValue(); - break; - case BEGIN_OBJECT_CHAR: - parseObject(); - break; - case BEGIN_ARRAY_CHAR: - parseArray(); - break; - case STRING_BEGIN_OR_END_CHAR: - parseString(false); - break; - default: - parseNumber(); - } - } - - protected void parseObject() { - Character ch = getNextToken(); - if (ch != BEGIN_OBJECT_CHAR) { - return; - } - output.append(ch); - // now let's check the case of an empty object - ch = peekNextToken(); - if (ch == END_OBJECT_CHAR) { - ch = getNextToken(); - output.append(ch); - return; - } - if (parseNameValuePair()) { - return; - } - while ((ch = getNextToken()) != null) { - output.append(ch); - switch (ch) { - case VALUE_SEPARATOR_CHAR: - parseNameValuePair(); - break; - case END_OBJECT_CHAR: - return; - default: - return; - } - } - } - - protected void parseArray() { - Character ch = getNextToken(); - if (ch != BEGIN_ARRAY_CHAR) { - return; - } - output.append(ch); - // now let's check the case of an empty array - ch = peekNextToken(); - if (ch == END_ARRAY_CHAR) { - ch = getNextToken(); - output.append(ch); - return; - } - parseValue(); - while ((ch = getNextToken()) != null) { - output.append(ch); - switch (ch) { - case VALUE_SEPARATOR_CHAR: - parseValue(); - break; - case END_ARRAY_CHAR: - return; - default: - return; - } - } - } - - protected boolean parseNameValuePair() { - Character ch; - String name = parseString(true); - if (name != null) { - currentPath.addLast(name); - } - ch = getNextToken(); - if (ch != NAME_SEPARATOR_CHAR) { - return true; - } - output.append(ch); - parseValue(); - if (name != null) { - currentPath.removeLast(); - } - return false; - } - -} http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/fda0d9ce/persistence-elasticsearch/core/src/test/java/org/apache/unomi/persistence/elasticsearch/FieldDotEscapeTest.java ---------------------------------------------------------------------- diff --git a/persistence-elasticsearch/core/src/test/java/org/apache/unomi/persistence/elasticsearch/FieldDotEscapeTest.java b/persistence-elasticsearch/core/src/test/java/org/apache/unomi/persistence/elasticsearch/FieldDotEscapeTest.java deleted file mode 100644 index c4322dc..0000000 --- a/persistence-elasticsearch/core/src/test/java/org/apache/unomi/persistence/elasticsearch/FieldDotEscapeTest.java +++ /dev/null @@ -1,95 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.unomi.persistence.elasticsearch; - -import org.apache.commons.io.IOUtils; -import org.junit.Assert; -import org.junit.Test; - -import java.io.IOException; -import java.io.InputStream; -import java.util.*; - -/** - * A unit test class for the FieldDotEscaper utility methods - */ -public class FieldDotEscapeTest { - - @Test - public void testTrickyJson() { - String result = FieldDotEscaper.escapeJson("{\"tricking_the_parser\" : \"this.should.not\\\": match.either\"}"); - Assert.assertTrue("Found escaped pattern instead of untouched one", result.contains("this.should.not\\\": match.either")); - } - - @Test - public void testJustDot() { - String result = FieldDotEscaper.escapeJson("{\".\" : \"this.should.not\\\": match.either\"}"); - Assert.assertTrue("Didn't find expected escaped pattern", result.contains("\"__DOT__\"")); - Assert.assertTrue("Found escaped pattern instead of untouched one", result.contains("this.should.not\\\": match.either")); - } - - @Test - public void testComplexJson() throws IOException { - InputStream complexJsonInputStream = this.getClass().getClassLoader().getResourceAsStream("complex.json"); - String input = IOUtils.toString(complexJsonInputStream); - System.out.println("Original JSON:"); - System.out.println("================="); - System.out.println(input); - Set<String> modifiedNames = new LinkedHashSet<>(); - String result = FieldDotEscaper.escapeJson(input, modifiedNames); - System.out.println("Modified names:"); - System.out.println("================="); - for (String modifiedName : modifiedNames) { - System.out.println(modifiedName); - } - System.out.println("Transformed JSON:"); - System.out.println("================="); - System.out.println(result); - Assert.assertTrue("Didn't find expected escaped pattern", result.contains("src_terms[0]__DOT__fields__DOT__siteContent")); - Assert.assertTrue("Didn't find expected escaped pattern", result.contains("newline__DOT__test")); - Assert.assertTrue("Found escaped pattern instead of untouched one", result.contains("this.should.never:match")); - Assert.assertTrue("Found escaped pattern instead of untouched one", result.contains("this.should.not\\\": match.either")); - result = FieldDotEscaper.unescapeJson(result); - Assert.assertEquals("Round trip of escaping then unescaping should be identical", input, result); - } - - @Test - public void testString() { - String input = "this.is.a..test"; - String result = FieldDotEscaper.unescapeString(FieldDotEscaper.escapeString(input)); - Assert.assertEquals("Strings should be exactly the same", input, result); - } - - @Test - public void testMap() { - Map<String,Object> input = new HashMap<>(); - input.put("this.is.a..test", "1"); - input.put("another.test", "2"); - Map<? extends String,?> result = FieldDotEscaper.unescapeMap(FieldDotEscaper.escapeMap(input)); - Assert.assertEquals("Maps should be identical", input, result); - } - - @Test - public void testProperties() { - Properties input = new Properties(); - input.put("this.is.a..test", "1"); - input.put("another.test", "2"); - Properties result = FieldDotEscaper.unescapeProperties(FieldDotEscaper.escapeProperties(input)); - Assert.assertEquals("Properties should be identical", input, result); - } - -} http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/fda0d9ce/persistence-elasticsearch/core/src/test/java/org/apache/unomi/persistence/elasticsearch/FieldDotJsonTransformerTest.java ---------------------------------------------------------------------- diff --git a/persistence-elasticsearch/core/src/test/java/org/apache/unomi/persistence/elasticsearch/FieldDotJsonTransformerTest.java b/persistence-elasticsearch/core/src/test/java/org/apache/unomi/persistence/elasticsearch/FieldDotJsonTransformerTest.java deleted file mode 100644 index 284ac84..0000000 --- a/persistence-elasticsearch/core/src/test/java/org/apache/unomi/persistence/elasticsearch/FieldDotJsonTransformerTest.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.unomi.persistence.elasticsearch; - -import com.fasterxml.jackson.databind.ObjectMapper; -import org.apache.commons.io.IOUtils; -import org.junit.Assert; -import org.junit.Test; - -import java.io.IOException; -import java.io.InputStream; -import java.util.Map; - -/** - * A unit test class for our custom JSON parser. - */ -public class FieldDotJsonTransformerTest { - - @Test - public void testNoReplacement() throws IOException { - InputStream complexJsonInputStream = this.getClass().getClassLoader().getResourceAsStream("complex.json"); - String input = IOUtils.toString(complexJsonInputStream); - StringBuffer output = new StringBuffer(); - FieldDotJsonTransformer fieldDotJsonTransformer = new FieldDotJsonTransformer(input, output, null); - fieldDotJsonTransformer.transform(); - Assert.assertEquals("Parsed JSON and original should be identical", input, output.toString()); - InputStream profileJsonInputStream = this.getClass().getClassLoader().getResourceAsStream("profile.json"); - input = IOUtils.toString(profileJsonInputStream); - output = new StringBuffer(); - fieldDotJsonTransformer = new FieldDotJsonTransformer(input, output, null); - fieldDotJsonTransformer.transform(); - Assert.assertEquals("Parsed JSON and original should be identical", input, output.toString()); - } - - @Test - public void testWithJackson() throws IOException { - InputStream complexJsonInputStream = this.getClass().getClassLoader().getResourceAsStream("complex.json"); - String input = IOUtils.toString(complexJsonInputStream); - StringBuffer output = new StringBuffer(); - FieldDotJsonTransformer fieldDotJsonTransformer = new FieldDotJsonTransformer(input, output, FieldDotEscaper.DOT_ESCAPE_MARKER); - fieldDotJsonTransformer.transform(); - ObjectMapper objectMapper = new ObjectMapper(); - Map<String,Object> object = objectMapper.readValue(output.toString(), Map.class); - Assert.assertNotNull("Object parsed with Jackson should not be null", object); - } - -} http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/fda0d9ce/persistence-elasticsearch/core/src/test/resources/complex.json ---------------------------------------------------------------------- diff --git a/persistence-elasticsearch/core/src/test/resources/complex.json b/persistence-elasticsearch/core/src/test/resources/complex.json deleted file mode 100644 index c6ff41c..0000000 --- a/persistence-elasticsearch/core/src/test/resources/complex.json +++ /dev/null @@ -1,50 +0,0 @@ -{ - "events": [ - { - "scope": "ACMESPACE", - "eventType": "form", - "source": { - "itemType": "page", - "scope": "ACMESPACE", - "itemId": "${pageId}", - "properties": { - "pageInfo": { - "pageID": "${pageId}", - "pageName": "${pageName}", - "pagePath": "${pagePath}", - "destinationURL": "${destinationURL}", - "referringURL": "${previousURL}", - "language": "${language}" - }, - "category": {}, - "attributes": { - "colon_in_value" : "this.should.never:match", - "newline.test" - : "more.dots", - "tricking_the_parser" : "this.should.not\": match.either", - "unicode black star": "before\u2605after", - "escaping test" : "\"\n\r\t\\" - } - } - }, - "target": { - "itemType": "form", - "scope": "ACMESPACE", - "itemId": "searchForm" - }, - "properties": { - "jcrMethodToCall": "get", - "src_originSiteKey": "ACMESPACE", - "src_terms[0].term": "${word}", - "src_terms[0].applyFilter": "true", - "src_terms[0].match": "all_words", - "src_terms[0].fields.siteContent": "true", - "src_terms[0].fields.tags": "true", - "src_terms[0].fields.files": "true", - "src_sites.values": "ACMESPACE", - "src_sitesForReferences.values": "systemsite", - "src_languages.values": "en" - } - } - ] -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/fda0d9ce/persistence-elasticsearch/core/src/test/resources/profile.json ---------------------------------------------------------------------- diff --git a/persistence-elasticsearch/core/src/test/resources/profile.json b/persistence-elasticsearch/core/src/test/resources/profile.json deleted file mode 100644 index 8fc9d2c..0000000 --- a/persistence-elasticsearch/core/src/test/resources/profile.json +++ /dev/null @@ -1 +0,0 @@ -{"itemId":"6bb362d8-d094-44b4-ba2e-012163286825","itemType":"personaSession","scope":null,"profileId":"googleBot","profile":{"itemId":"googleBot","itemType":"persona","properties":{"description":"Represents a Google bot","firstName":"Google","lastName":"Bot"},"systemProperties":{},"segments":[],"scores":null,"mergedWith":null},"properties":{"operatingSystemName":"unknown","sessionCountryName":"United States","location":{"lon":-122.084058,"lat":37.422},"userAgentVersion":"2.1","sessionCountryCode":"US","deviceCategory":"Other","operatingSystemFamily":"unknown","userAgentName":"Googlebot/2.1","sessionCity":"Mountain View","remoteHost":"www.google.com","remoteAddr":"66.249.66.1"},"systemProperties":{},"timeStamp":"2014-09-18T09:18:52Z","lastEventDate":"2014-09-18T11:23:59Z","size":0,"duration":7507068} \ No newline at end of file
