Repository: metamodel Updated Branches: refs/heads/master 96f577774 -> b6fb6e04e
METAMODEL-164: Fixed and refactored ES mapping parser to use real map. Previously the parser was (by accident I think) using the toString() representation of that same map which made it very vulnerable towards parsing issues. Project: http://git-wip-us.apache.org/repos/asf/metamodel/repo Commit: http://git-wip-us.apache.org/repos/asf/metamodel/commit/6f4958f1 Tree: http://git-wip-us.apache.org/repos/asf/metamodel/tree/6f4958f1 Diff: http://git-wip-us.apache.org/repos/asf/metamodel/diff/6f4958f1 Branch: refs/heads/master Commit: 6f4958f1e916c8cf4560da8cb04cfb53ff578764 Parents: 96f5777 Author: Kasper Sørensen <[email protected]> Authored: Wed Aug 12 22:56:52 2015 +0200 Committer: Kasper Sørensen <[email protected]> Committed: Wed Aug 12 22:56:52 2015 +0200 ---------------------------------------------------------------------- .../elasticsearch/ElasticSearchDataContext.java | 24 ++++---- .../ElasticSearchMetaDataParser.java | 63 +++++++++----------- .../ElasticSearchMetaDataParserTest.java | 30 ++++++---- 3 files changed, 57 insertions(+), 60 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/metamodel/blob/6f4958f1/elasticsearch/src/main/java/org/apache/metamodel/elasticsearch/ElasticSearchDataContext.java ---------------------------------------------------------------------- diff --git a/elasticsearch/src/main/java/org/apache/metamodel/elasticsearch/ElasticSearchDataContext.java b/elasticsearch/src/main/java/org/apache/metamodel/elasticsearch/ElasticSearchDataContext.java index ce028f6..ba221d1 100644 --- a/elasticsearch/src/main/java/org/apache/metamodel/elasticsearch/ElasticSearchDataContext.java +++ b/elasticsearch/src/main/java/org/apache/metamodel/elasticsearch/ElasticSearchDataContext.java @@ -22,7 +22,6 @@ import java.lang.reflect.Method; import java.util.ArrayList; import java.util.Arrays; import java.util.Comparator; -import java.util.Iterator; import java.util.List; import java.util.Map; @@ -228,20 +227,17 @@ public class ElasticSearchDataContext extends QueryPostprocessDataContext implem throw new IllegalArgumentException("No such document type in index '" + indexName + "': " + documentType); } final Map<String, Object> mp = mappingMetaData.getSourceAsMap(); - final Iterator<Map.Entry<String, Object>> it = mp.entrySet().iterator(); - SimpleTableDef std = null; - while (it.hasNext()) { - final Map.Entry<String, Object> pair = it.next(); - if (pair.getKey().equals("properties")) { - final ElasticSearchMetaData metaData = ElasticSearchMetaDataParser.parse(pair.getValue()); - std = new SimpleTableDef(documentType, metaData.getColumnNames(), metaData.getColumnTypes()); - } - } - if (std == null) { - throw new IllegalArgumentException("No properties defined for document type '" + documentType - + "' in index: " + indexName); + final Object metadataProperties = mp.get("properties"); + if (metadataProperties != null && metadataProperties instanceof Map) { + @SuppressWarnings("unchecked") + final Map<String, ?> metadataPropertiesMap = (Map<String, ?>) metadataProperties; + final ElasticSearchMetaData metaData = ElasticSearchMetaDataParser.parse(metadataPropertiesMap); + final SimpleTableDef std = new SimpleTableDef(documentType, metaData.getColumnNames(), + metaData.getColumnTypes()); + return std; } - return std; + throw new IllegalArgumentException("No mapping properties defined for document type '" + documentType + + "' in index: " + indexName); } @Override http://git-wip-us.apache.org/repos/asf/metamodel/blob/6f4958f1/elasticsearch/src/main/java/org/apache/metamodel/elasticsearch/ElasticSearchMetaDataParser.java ---------------------------------------------------------------------- diff --git a/elasticsearch/src/main/java/org/apache/metamodel/elasticsearch/ElasticSearchMetaDataParser.java b/elasticsearch/src/main/java/org/apache/metamodel/elasticsearch/ElasticSearchMetaDataParser.java index 82953d1..41cf184 100644 --- a/elasticsearch/src/main/java/org/apache/metamodel/elasticsearch/ElasticSearchMetaDataParser.java +++ b/elasticsearch/src/main/java/org/apache/metamodel/elasticsearch/ElasticSearchMetaDataParser.java @@ -18,6 +18,9 @@ */ package org.apache.metamodel.elasticsearch; +import java.util.Map; +import java.util.Map.Entry; + import org.apache.metamodel.schema.ColumnType; /** @@ -31,51 +34,42 @@ public class ElasticSearchMetaDataParser { * object. This method makes much easier to create the ElasticSearch schema. * * @param metaDataInfo - * ElasticSearch metadata info in a json-like format + * ElasticSearch mapping metadata in Map format * @return An ElasticSearchMetaData object */ - public static ElasticSearchMetaData parse(Object metaDataInfo) { - final String plainMetaDataInfo = removeFirstAndLastCharacter(metaDataInfo.toString()); - final String metaDataWithoutDateFormats = removeDateFormats(plainMetaDataInfo); - final String[] metaDataFields = metaDataWithoutDateFormats.split("},"); - - final String[] fieldNames = new String[metaDataFields.length + 1]; - final ColumnType[] columnTypes = new ColumnType[metaDataFields.length + 1]; + public static ElasticSearchMetaData parse(Map<String, ?> metaDataInfo) { + final String[] fieldNames = new String[metaDataInfo.size() + 1]; + final ColumnType[] columnTypes = new ColumnType[metaDataInfo.size() + 1]; // add the document ID field (fixed) fieldNames[0] = ElasticSearchDataContext.FIELD_ID; columnTypes[0] = ColumnType.STRING; int i = 1; - for (String metaDataField : metaDataFields) { - // message={type=long} - fieldNames[i] = getNameFromMetaDataField(metaDataField); - columnTypes[i] = getColumnTypeFromMetaDataField(metaDataField); + for (Entry<String, ?> metaDataField : metaDataInfo.entrySet()) { + @SuppressWarnings("unchecked") + final Map<String, ?> fieldMetadata = (Map<String, ?>) metaDataField.getValue(); + + fieldNames[i] = metaDataField.getKey(); + columnTypes[i] = getColumnTypeFromMetadataField(fieldMetadata); i++; } return new ElasticSearchMetaData(fieldNames, columnTypes); } - private static String removeFirstAndLastCharacter(String metaDataInfo) { - return metaDataInfo.substring(1, metaDataInfo.length() - 1); - } - - private static String removeDateFormats(String metaDataInfo) { - return metaDataInfo.replaceAll("type=date.*?}", "type=date}"); - } + private static ColumnType getColumnTypeFromMetadataField(Map<String, ?> fieldMetadata) { + final ColumnType columnType; + final String metaDataFieldType = getMetaDataFieldTypeFromMetaDataField(fieldMetadata); - private static String getNameFromMetaDataField(String metaDataField) { - return metaDataField.substring(0, metaDataField.indexOf("=")).trim(); - } + if (metaDataFieldType == null) { + return ColumnType.STRING; + } - private static ColumnType getColumnTypeFromMetaDataField(String metaDataField) { - final ColumnType columnType; - final String metaDataFieldType = getMetaDataFieldTypeFromMetaDataField(metaDataField); - if (metaDataFieldType.equals("long")) { - columnType = ColumnType.BIGINT; - } else if (metaDataFieldType.equals("date")) { + if (metaDataFieldType.startsWith("date")) { columnType = ColumnType.DATE; + } else if (metaDataFieldType.equals("long")) { + columnType = ColumnType.BIGINT; } else if (metaDataFieldType.equals("string")) { columnType = ColumnType.STRING; } else if (metaDataFieldType.equals("float")) { @@ -90,15 +84,12 @@ public class ElasticSearchMetaDataParser { return columnType; } - private static String getMetaDataFieldTypeFromMetaDataField(String metaDataField) { - String type = metaDataField.substring(metaDataField.indexOf("type=") + 5); - if (type.indexOf(",") > 0) { - type = type.substring(0, type.indexOf(",")); - } - if (type.indexOf("}") > 0) { - type = type.substring(0, type.indexOf("}")); + private static String getMetaDataFieldTypeFromMetaDataField(Map<String, ?> metaDataField) { + final Object type = metaDataField.get("type"); + if (type == null) { + return null; } - return type; + return type.toString(); } } http://git-wip-us.apache.org/repos/asf/metamodel/blob/6f4958f1/elasticsearch/src/test/java/org/apache/metamodel/elasticsearch/ElasticSearchMetaDataParserTest.java ---------------------------------------------------------------------- diff --git a/elasticsearch/src/test/java/org/apache/metamodel/elasticsearch/ElasticSearchMetaDataParserTest.java b/elasticsearch/src/test/java/org/apache/metamodel/elasticsearch/ElasticSearchMetaDataParserTest.java index 48a1f06..4d8bef3 100644 --- a/elasticsearch/src/test/java/org/apache/metamodel/elasticsearch/ElasticSearchMetaDataParserTest.java +++ b/elasticsearch/src/test/java/org/apache/metamodel/elasticsearch/ElasticSearchMetaDataParserTest.java @@ -18,24 +18,31 @@ */ package org.apache.metamodel.elasticsearch; +import java.util.LinkedHashMap; +import java.util.Map; + import junit.framework.TestCase; + import org.apache.metamodel.schema.ColumnType; +import org.elasticsearch.common.collect.MapBuilder; public class ElasticSearchMetaDataParserTest extends TestCase { public void testParseMetadataInfo() throws Exception { - String metaDataInfo = "{message={type=long}, " + - "postDate={type=date, format=dateOptionalTime}, " + - "anotherDate={type=date, format=dateOptionalTime}, " + - "user={type=string}, " + - "critical={type=boolean}, " + - "income={type=double}}"; - - ElasticSearchMetaData metaData = ElasticSearchMetaDataParser.parse(metaDataInfo); + Map<String, Object> metadata = new LinkedHashMap<String, Object>(); + metadata.put("message", MapBuilder.newMapBuilder().put("type", "long").immutableMap()); + metadata.put("postDate", MapBuilder.newMapBuilder().put("type", "date").put("format", "dateOptionalTime").immutableMap()); + metadata.put("anotherDate", MapBuilder.newMapBuilder().put("type", "date").put("format", "dateOptionalTime").immutableMap()); + metadata.put("user", MapBuilder.newMapBuilder().put("type", "string").immutableMap()); + metadata.put("critical", MapBuilder.newMapBuilder().put("type", "boolean").immutableMap()); + metadata.put("income", MapBuilder.newMapBuilder().put("type", "double").immutableMap()); + metadata.put("untypedthingie", MapBuilder.newMapBuilder().put("foo", "bar").immutableMap()); + + ElasticSearchMetaData metaData = ElasticSearchMetaDataParser.parse(metadata); String[] columnNames = metaData.getColumnNames(); ColumnType[] columnTypes = metaData.getColumnTypes(); - assertTrue(columnNames.length == 7); + assertTrue(columnNames.length == 8); assertEquals(columnNames[0], "_id"); assertEquals(columnNames[1], "message"); assertEquals(columnNames[2], "postDate"); @@ -43,7 +50,9 @@ public class ElasticSearchMetaDataParserTest extends TestCase { assertEquals(columnNames[4], "user"); assertEquals(columnNames[5], "critical"); assertEquals(columnNames[6], "income"); - assertTrue(columnTypes.length == 7); + assertEquals(columnNames[7], "untypedthingie"); + + assertTrue(columnTypes.length == 8); assertEquals(columnTypes[0], ColumnType.STRING); assertEquals(columnTypes[1], ColumnType.BIGINT); assertEquals(columnTypes[2], ColumnType.DATE); @@ -51,5 +60,6 @@ public class ElasticSearchMetaDataParserTest extends TestCase { assertEquals(columnTypes[4], ColumnType.STRING); assertEquals(columnTypes[5], ColumnType.BOOLEAN); assertEquals(columnTypes[6], ColumnType.DOUBLE); + assertEquals(columnTypes[7], ColumnType.STRING); } }
