Repository: incubator-usergrid Updated Branches: refs/heads/USERGRID-669 [created] c3c248049
Changes parsing to use regex Changes fields to be more descriptive Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/c3c24804 Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/c3c24804 Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/c3c24804 Branch: refs/heads/USERGRID-669 Commit: c3c248049d3d3b3699703bf053402115761cd57b Parents: 53563e8 Author: Todd Nine <tn...@apigee.com> Authored: Wed May 27 10:25:47 2015 -0600 Committer: Todd Nine <tn...@apigee.com> Committed: Thu May 28 06:30:46 2015 -0600 ---------------------------------------------------------------------- .../index/impl/EntityToMapConverter.java | 11 +- .../impl/EsApplicationEntityIndexImpl.java | 3 +- .../persistence/index/impl/IndexOperation.java | 1 - .../persistence/index/impl/IndexingUtils.java | 153 ++++++++++++++----- .../index/migration/LegacyIndexIdentifier.java | 78 ---------- .../index/impl/EntityToMapConverterTest.java | 6 +- .../index/impl/IndexingUtilsTest.java | 109 +++++++++++++ 7 files changed, 233 insertions(+), 128 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/c3c24804/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EntityToMapConverter.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EntityToMapConverter.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EntityToMapConverter.java index ff81672..cced80f 100644 --- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EntityToMapConverter.java +++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EntityToMapConverter.java @@ -37,8 +37,11 @@ import static org.apache.usergrid.persistence.index.impl.IndexingUtils.ENTITY_FI import static org.apache.usergrid.persistence.index.impl.IndexingUtils.ENTITY_ID_FIELDNAME; import static org.apache.usergrid.persistence.index.impl.IndexingUtils.ENTITY_TYPE_FIELDNAME; import static org.apache.usergrid.persistence.index.impl.IndexingUtils.ENTITY_VERSION_FIELDNAME; +import static org.apache.usergrid.persistence.index.impl.IndexingUtils.applicationId; +import static org.apache.usergrid.persistence.index.impl.IndexingUtils.entityId; import static org.apache.usergrid.persistence.index.impl.IndexingUtils.getType; -import static org.apache.usergrid.persistence.index.impl.IndexingUtils.idString; +import static org.apache.usergrid.persistence.index.impl.IndexingUtils.nodeId; + /** * Convert a CP entity to an elasticsearch document @@ -64,16 +67,16 @@ public class EntityToMapConverter { * Add our static fields for easier admin/debugging/reporting ****/ - outputEntity.put( ENTITY_ID_FIELDNAME, idString(entityId) ); + outputEntity.put( ENTITY_ID_FIELDNAME, entityId( entityId ) ); outputEntity.put( ENTITY_VERSION_FIELDNAME, entity.getVersion() ); outputEntity.put( ENTITY_TYPE_FIELDNAME, getType( applicationScope, entityId ) ); - outputEntity.put( APPLICATION_ID_FIELDNAME, idString( applicationScope.getApplication() ) ); + outputEntity.put( APPLICATION_ID_FIELDNAME, applicationId( applicationScope.getApplication() ) ); - outputEntity.put( EDGE_NODE_ID_FIELDNAME, idString( indexEdge.getNodeId() ) ); + outputEntity.put( EDGE_NODE_ID_FIELDNAME, nodeId( indexEdge.getNodeId() ) ); outputEntity.put( EDGE_NODE_TYPE_FIELDNAME, indexEdge.getNodeType() ); http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/c3c24804/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java index 5b67060..34b4e9a 100644 --- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java +++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java @@ -65,6 +65,7 @@ import com.google.inject.Inject; import rx.Observable; import static org.apache.usergrid.persistence.index.impl.IndexingUtils.APPLICATION_ID_FIELDNAME; +import static org.apache.usergrid.persistence.index.impl.IndexingUtils.applicationId; import static org.apache.usergrid.persistence.index.impl.IndexingUtils.parseIndexDocId; @@ -178,7 +179,7 @@ public class EsApplicationEntityIndexImpl implements ApplicationEntityIndex { */ public Observable deleteApplication() { deleteApplicationMeter.mark(); - String idString = IndexingUtils.idString( applicationScope.getApplication() ); + String idString = applicationId( applicationScope.getApplication() ); final TermQueryBuilder tqb = QueryBuilders.termQuery( APPLICATION_ID_FIELDNAME, idString ); final String[] indexes = entityIndex.getUniqueIndexes(); Timer.Context timer = deleteApplicationTimer.time(); http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/c3c24804/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexOperation.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexOperation.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexOperation.java index 2e2f188..fae809f 100644 --- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexOperation.java +++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexOperation.java @@ -30,7 +30,6 @@ import org.elasticsearch.action.bulk.BulkRequestBuilder; import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.client.Client; -import static org.apache.usergrid.persistence.index.impl.IndexingUtils.idString; /** http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/c3c24804/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java index bc15149..94038b1 100644 --- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java +++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java @@ -19,6 +19,8 @@ package org.apache.usergrid.persistence.index.impl;/* import java.util.UUID; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import org.apache.usergrid.persistence.core.scope.ApplicationScope; import org.apache.usergrid.persistence.index.CandidateResult; @@ -28,14 +30,46 @@ import org.apache.usergrid.persistence.model.entity.Entity; import org.apache.usergrid.persistence.model.entity.Id; import org.apache.usergrid.persistence.model.entity.SimpleId; +import com.google.common.base.Preconditions; + public class IndexingUtils { + /** + * Regular expression for uuids + */ + public static final String UUID_REX = + "([A-Fa-f0-9]{8}-[A-Fa-f0-9]{4}-[A-Fa-f0-9]{4}-[A-Fa-f0-9]{4}-[A-Fa-f0-9]{12})"; + + public static final String TYPE_REX = "(\\w+)"; + + + private static final String APPID_NAME = "appId"; + + private static final String ENTITY_NAME = "entityId"; + + private static final String NODEID_NAME = "nodeId"; + + private static final String VERSION_NAME = "version"; + private static final String EDGE_NAME = "edgeName"; + + private static final String NODE_TYPE_NAME = "nodeType"; + private static final String ENTITY_TYPE_NAME = "entityType"; + + + //the document Id will have 9 groups + private static final String DOCUMENT_ID_REGEX = + "appId\\(" + UUID_REX + "," + TYPE_REX + "\\)\\.entityId\\(" + UUID_REX + "," + TYPE_REX + "\\)\\.version\\(" + UUID_REX + + "\\)\\.nodeId\\(" + UUID_REX + "," + TYPE_REX + "\\)\\.edgeName\\(" + TYPE_REX + "\\)\\.nodeType\\(" + TYPE_REX + "\\)"; + + + private static final Pattern DOCUMENT_PATTERN = Pattern.compile( DOCUMENT_ID_REGEX ); + // These are not allowed in document type names: _ . , | # - public static final String FIELD_SEPERATOR = "__"; + public static final String FIELD_SEPERATOR = "."; - public static final String ID_SEPERATOR = "::"; + public static final String ID_SEPERATOR = ","; /** @@ -93,10 +127,6 @@ public class IndexingUtils { public static final String FIELD_STRING_NESTED_UNANALYZED = FIELD_STRING_NESTED + ".exact"; - - - - /** * Create our sub scope. This is the ownerUUID + type * @@ -104,29 +134,11 @@ public class IndexingUtils { */ public static String createContextName( final ApplicationScope applicationScope, final SearchEdge scope ) { StringBuilder sb = new StringBuilder(); - idString( sb, applicationScope.getApplication() ); + idString( sb, APPID_NAME, applicationScope.getApplication() ); sb.append( FIELD_SEPERATOR ); - idString( sb, scope.getNodeId() ); + idString( sb, NODEID_NAME, scope.getNodeId() ); sb.append( FIELD_SEPERATOR ); - sb.append( scope.getEdgeName() ); - return sb.toString(); - } - - - /** - * Append the id to the string - */ - public static final void idString( final StringBuilder builder, final Id id ) { - builder.append( id.getUuid() ).append( ID_SEPERATOR ).append( id.getType().toLowerCase() ); - } - - - /** - * Turn the id into a string - */ - public static final String idString( final Id id ) { - final StringBuilder sb = new StringBuilder(); - idString( sb, id ); + appendField( sb, EDGE_NAME, scope.getEdgeName() ); return sb.toString(); } @@ -147,46 +159,105 @@ public class IndexingUtils { final UUID version, final SearchEdge searchEdge ) { StringBuilder sb = new StringBuilder(); - idString( sb, applicationScope.getApplication() ); + idString( sb, APPID_NAME, applicationScope.getApplication() ); sb.append( FIELD_SEPERATOR ); - idString( sb, entityId ); + idString( sb, ENTITY_ID_FIELDNAME, entityId ); sb.append( FIELD_SEPERATOR ); - sb.append( version.toString() ); - + appendField( sb, VERSION_NAME, version.toString() ); sb.append( FIELD_SEPERATOR ); - idString( sb, searchEdge.getNodeId() ); + idString( sb, NODEID_NAME, searchEdge.getNodeId() ); sb.append( FIELD_SEPERATOR ); - sb.append( searchEdge.getEdgeName() ); + appendField( sb, EDGE_NAME, searchEdge.getEdgeName() ); sb.append( FIELD_SEPERATOR ); - sb.append( searchEdge.getNodeType() ); + appendField( sb, NODE_TYPE_NAME, searchEdge.getNodeType().name() ); return sb.toString(); } + public static final String entityId( final Id id ) { + return idString( ENTITY_NAME, id ); + } + + + public static final String applicationId( final Id id ) { + return idString( APPID_NAME, id ); + } + + + public static final String nodeId( final Id id ) { + return idString( NODEID_NAME, id ); + } + + + /** + * Construct and Id string with the specified type for the id provided. + */ + private static final String idString( final String type, final Id id ) { + final StringBuilder stringBuilder = new StringBuilder(); + + idString( stringBuilder, type, id ); + + return stringBuilder.toString(); + } + + + /** + * Append the id to the string + */ + private static final void idString( final StringBuilder builder, final String type, final Id id ) { + builder.append( type ).append( "(" ).append( id.getUuid() ).append( ID_SEPERATOR ) + .append( id.getType().toLowerCase() ).append( ")" ); + } + + + /** + * Append a field + */ + private static void appendField( final StringBuilder builder, final String type, final String value ) { + builder.append( type ).append( "(" ).append( value ).append( ")" ); + } + + /** * Parse the document id into a candidate result */ public static CandidateResult parseIndexDocId( final String documentId ) { - String[] idparts = documentId.split( FIELD_SEPERATOR ); - String entityIdString = idparts[1]; - String version = idparts[2]; - final String[] entityIdParts = entityIdString.split( ID_SEPERATOR ); + final Matcher matcher = DOCUMENT_PATTERN.matcher( documentId ); + + Preconditions.checkArgument( matcher.matches(), "Pattern for document id did not match expected format" ); + Preconditions.checkArgument( matcher.groupCount() == 9, "9 groups expected in the pattern" ); + + //Other fields can be parsed using groups. The groups start at value 1, group 0 is the entire match + final String entityUUID = matcher.group( 3 ); + final String entityType = matcher.group( 4 ); + + final String versionUUID = matcher.group( 5 ); - Id entityId = new SimpleId( UUID.fromString( entityIdParts[0] ), entityIdParts[1] ); - return new CandidateResult( entityId, UUID.fromString( version ) ); + Id entityId = new SimpleId( UUID.fromString( entityUUID ), entityType ); + + return new CandidateResult( entityId, UUID.fromString( versionUUID ) ); } + /** + * Get the entity type + */ public static String getType( ApplicationScope applicationScope, Id entityId ) { return getType( applicationScope, entityId.getType() ); } public static String getType( ApplicationScope applicationScope, String type ) { - return idString( applicationScope.getApplication() ) + FIELD_SEPERATOR + type; + + StringBuilder sb = new StringBuilder(); + + idString( sb, APPID_NAME, applicationScope.getApplication() ); + sb.append( FIELD_SEPERATOR ); + sb.append( ENTITY_TYPE_NAME).append("(" ).append( type ).append( ")" ); + return sb.toString(); } } http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/c3c24804/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/LegacyIndexIdentifier.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/LegacyIndexIdentifier.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/LegacyIndexIdentifier.java deleted file mode 100644 index c93fd86..0000000 --- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/LegacyIndexIdentifier.java +++ /dev/null @@ -1,78 +0,0 @@ -/* - * - * * Licensed to the Apache Software Foundation (ASF) under one or more - * * contributor license agreements. The ASF licenses this file to You - * * under the Apache License, Version 2.0 (the "License"); you may not - * * use this file except in compliance with the License. - * * You may obtain a copy of the License at - * * - * * http://www.apache.org/licenses/LICENSE-2.0 - * * - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, - * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * * See the License for the specific language governing permissions and - * * limitations under the License. For additional information regarding - * * copyright in this work, please see the NOTICE file in the top level - * * directory of this distribution. - * - */ -package org.apache.usergrid.persistence.index.migration; - -import org.apache.usergrid.persistence.core.scope.ApplicationScope; -import org.apache.usergrid.persistence.index.impl.IndexAlias; -import org.apache.usergrid.persistence.index.IndexFig; -import org.apache.usergrid.persistence.index.impl.IndexIdentifier; -import org.apache.usergrid.persistence.index.impl.IndexingUtils; - -/** - * Class is used to generate an index name and alias name the old way via app name - */ -public class LegacyIndexIdentifier implements IndexIdentifier { - private final IndexFig config; - private final ApplicationScope applicationScope; - - public LegacyIndexIdentifier(IndexFig config, ApplicationScope applicationScope) { - this.config = config; - this.applicationScope = applicationScope; - } - - /** - * Get the alias name - * @return - */ - public IndexAlias getAlias() { - return new IndexAlias(config,getIndexBase()); - } - - /** - * Get index name, send in additional parameter to add incremental indexes - * @param suffix - * @return - */ - public String getIndex(String suffix) { - if (suffix != null) { - return getIndexBase() + "_" + suffix; - } else { - return getIndexBase(); - } - } - - /** - * returns the base name for index which will be used to add an alias and index - * @return - */ - private String getIndexBase() { - StringBuilder sb = new StringBuilder(); - sb.append(config.getIndexPrefix()).append(IndexingUtils.FIELD_SEPERATOR ); - IndexingUtils.idString(sb, applicationScope.getApplication()); - return sb.toString(); - } - - - - public String toString() { - return "application: " + applicationScope.getApplication().getUuid(); - } - -} http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/c3c24804/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityToMapConverterTest.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityToMapConverterTest.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityToMapConverterTest.java index 7fed42c..96f9169 100644 --- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityToMapConverterTest.java +++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityToMapConverterTest.java @@ -86,11 +86,11 @@ public class EntityToMapConverterTest { final String applicationId = entityMap.get( IndexingUtils.APPLICATION_ID_FIELDNAME ).toString(); - assertEquals( IndexingUtils.idString( scope.getApplication() ), applicationId ); + assertEquals( IndexingUtils.applicationId( scope.getApplication() ), applicationId ); final String entityIdString = entityMap.get( IndexingUtils.ENTITY_ID_FIELDNAME ).toString(); - assertEquals( IndexingUtils.idString( entity.getId() ), entityIdString ); + assertEquals( IndexingUtils.entityId( entity.getId() ), entityIdString ); final String versionString = entityMap.get( IndexingUtils.ENTITY_VERSION_FIELDNAME ).toString(); @@ -103,7 +103,7 @@ public class EntityToMapConverterTest { final String nodeIdString = entityMap.get( IndexingUtils.EDGE_NODE_ID_FIELDNAME ).toString(); - assertEquals( IndexingUtils.idString( indexEdge.getNodeId() ), nodeIdString ); + assertEquals( IndexingUtils.nodeId( indexEdge.getNodeId() ), nodeIdString ); final String edgeName = entityMap.get( IndexingUtils.EDGE_NAME_FIELDNAME ).toString(); http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/c3c24804/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/IndexingUtilsTest.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/IndexingUtilsTest.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/IndexingUtilsTest.java new file mode 100644 index 0000000..cc66854 --- /dev/null +++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/IndexingUtilsTest.java @@ -0,0 +1,109 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.usergrid.persistence.index.impl; + + +import java.util.UUID; + +import org.junit.Test; + +import org.apache.usergrid.persistence.core.scope.ApplicationScopeImpl; +import org.apache.usergrid.persistence.index.CandidateResult; +import org.apache.usergrid.persistence.index.SearchEdge; +import org.apache.usergrid.persistence.model.entity.Id; +import org.apache.usergrid.persistence.model.entity.SimpleId; +import org.apache.usergrid.persistence.model.util.UUIDGenerator; + +import static org.apache.usergrid.persistence.index.impl.IndexingUtils.parseIndexDocId; +import static org.junit.Assert.assertEquals; + + +public class IndexingUtilsTest { + + @Test + public void testCreateContextName() throws Exception { + + final ApplicationScopeImpl applicationScope = new ApplicationScopeImpl( new SimpleId( "application" ) ); + + final SearchEdgeImpl searchEdge = + new SearchEdgeImpl( new SimpleId( "source" ), "users", SearchEdge.NodeType.TARGET ); + + final String output = IndexingUtils.createContextName( applicationScope, searchEdge ); + + + final String expected = + "appId(" + applicationScope.getApplication().getUuid() + ",application).nodeId(" + searchEdge.getNodeId() + .getUuid() + + "," + searchEdge.getNodeId().getType() + ").edgeName(users)"; + + + assertEquals( output, expected ); + } + + + @Test + public void testDocumentId() { + + final ApplicationScopeImpl applicationScope = new ApplicationScopeImpl( new SimpleId( "application" ) ); + + final Id id = new SimpleId( "id" ); + final UUID version = UUIDGenerator.newTimeUUID(); + + final SearchEdgeImpl searchEdge = + new SearchEdgeImpl( new SimpleId( "source" ), "users", SearchEdge.NodeType.TARGET ); + + final String output = IndexingUtils.createIndexDocId( applicationScope, id, version, searchEdge ); + + + final String expected = + "appId(" + applicationScope.getApplication().getUuid() + ",application).entityId(" + id.getUuid() + "," + id + .getType() + ").version(" + version + ").nodeId(" + searchEdge.getNodeId().getUuid() + "," + searchEdge + .getNodeId().getType() + ").edgeName(users).nodeType(TARGET)"; + + + assertEquals( output, expected ); + + + //now parse it + + final CandidateResult parsedId = parseIndexDocId( output ); + + assertEquals(version, parsedId.getVersion()); + assertEquals(id, parsedId.getId()); + } + + + @Test + public void testEntityType() { + + final ApplicationScopeImpl applicationScope = new ApplicationScopeImpl( new SimpleId( "application" ) ); + + final Id id = new SimpleId( "id" ); + + final String output = IndexingUtils.getType( applicationScope, id ); + + + final String expected = + "appId(" + applicationScope.getApplication().getUuid() + ",application).entityType(" + id.getType() + ")"; + + + assertEquals( output, expected ); + } +}
