Add tool that will iterator a collection and print UUIDs.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/a7639d55 Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/a7639d55 Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/a7639d55 Branch: refs/heads/master Commit: a7639d55336dd696ea804f166811d6db90225b86 Parents: fe6bea2 Author: Michael Russo <[email protected]> Authored: Thu Aug 25 14:31:24 2016 -0700 Committer: Michael Russo <[email protected]> Committed: Thu Aug 25 14:31:24 2016 -0700 ---------------------------------------------------------------------- .../usergrid/tools/CollectionIterator.java | 189 +++++++++++++++++++ .../org/apache/usergrid/tools/ExportApp.java | 118 +----------- .../org/apache/usergrid/tools/ToolBase.java | 2 + .../usergrid/tools/export/ExportConnection.java | 74 ++++++++ .../usergrid/tools/export/ExportEntity.java | 66 +++++++ 5 files changed, 337 insertions(+), 112 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/usergrid/blob/a7639d55/stack/tools/src/main/java/org/apache/usergrid/tools/CollectionIterator.java ---------------------------------------------------------------------- diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/CollectionIterator.java b/stack/tools/src/main/java/org/apache/usergrid/tools/CollectionIterator.java new file mode 100644 index 0000000..02188cb --- /dev/null +++ b/stack/tools/src/main/java/org/apache/usergrid/tools/CollectionIterator.java @@ -0,0 +1,189 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.usergrid.tools; + + +import java.text.DateFormat; +import java.text.SimpleDateFormat; +import java.util.*; +import java.util.concurrent.atomic.AtomicInteger; + +import com.google.common.base.*; +import com.google.common.base.Optional; +import com.netflix.astyanax.connectionpool.exceptions.ConnectionException; +import com.netflix.astyanax.model.Column; +import com.netflix.astyanax.model.ConsistencyLevel; +import com.netflix.astyanax.util.RangeBuilder; +import org.apache.usergrid.corepersistence.pipeline.read.ResultsPage; +import org.apache.usergrid.corepersistence.results.EntityQueryExecutor; +import org.apache.usergrid.corepersistence.results.IdQueryExecutor; +import org.apache.usergrid.corepersistence.service.CollectionSearch; +import org.apache.usergrid.corepersistence.service.CollectionService; +import org.apache.usergrid.persistence.*; +import org.apache.usergrid.persistence.collection.serialization.MvccEntitySerializationStrategy; +import org.apache.usergrid.persistence.collection.serialization.UniqueValueSerializationStrategy; +import org.apache.usergrid.persistence.collection.serialization.UniqueValueSet; +import org.apache.usergrid.persistence.collection.serialization.impl.*; +import org.apache.usergrid.persistence.core.scope.ApplicationScopeImpl; +import org.apache.usergrid.persistence.index.utils.UUIDUtils; +import org.apache.usergrid.persistence.model.entity.*; +import org.apache.usergrid.persistence.model.entity.Entity; +import org.apache.usergrid.persistence.model.field.StringField; +import org.apache.usergrid.persistence.schema.CollectionInfo; +import org.apache.usergrid.utils.InflectionUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.OptionBuilder; +import org.apache.commons.cli.Options; + + +import org.apache.usergrid.persistence.core.astyanax.MultiTenantColumnFamily; +import org.apache.usergrid.persistence.core.astyanax.ScopedRowKey; +import org.apache.usergrid.persistence.core.astyanax.ScopedRowKeySerializer; +import rx.*; + +import static org.apache.usergrid.persistence.Schema.getDefaultSchema; + + +public class CollectionIterator extends ToolBase { + + private static final Logger logger = LoggerFactory.getLogger( CollectionIterator.class ); + + private static final String APPLICATION_ARG = "app"; + + private static final String ENTITY_TYPE_ARG = "entityType"; + + private EntityManager em; + + @Override + @SuppressWarnings( "static-access" ) + public Options createOptions() { + + + Options options = super.createOptions(); + + + Option appOption = OptionBuilder.withArgName( APPLICATION_ARG ).hasArg().isRequired( true ) + .withDescription( "application id" ).create( APPLICATION_ARG ); + + + options.addOption( appOption ); + + Option collectionOption = + OptionBuilder.withArgName(ENTITY_TYPE_ARG).hasArg().isRequired( true ).withDescription( "collection name" ) + .create(ENTITY_TYPE_ARG); + + options.addOption( collectionOption ); + + + return options; + } + + + /* + * (non-Javadoc) + * + * @see + * org.apache.usergrid.tools.ToolBase#runTool(org.apache.commons.cli.CommandLine) + */ + @Override + public void runTool( CommandLine line ) throws Exception { + + startSpring(); + + UUID appToFilter = null; + if (!line.getOptionValue(APPLICATION_ARG).isEmpty()) { + appToFilter = UUID.fromString(line.getOptionValue(APPLICATION_ARG)); + } + + String entityType = line.getOptionValue(ENTITY_TYPE_ARG); + + logger.info("Staring Tool: CollectionIterator"); + logger.info("Using Cassandra consistency level: {}", System.getProperty("usergrid.read.cl", "CL_LOCAL_QUORUM")); + + em = emf.getEntityManager( appToFilter ); + EntityRef headEntity = new SimpleEntityRef("application", appToFilter); + + CollectionService collectionService = injector.getInstance(CollectionService.class); + String collectionName = InflectionUtils.pluralize(entityType); + + Query query = new Query(); + query.setCollection(collectionName); + query.setLimit(1000); + + com.google.common.base.Optional<String> queryString = com.google.common.base.Optional.absent(); + + CollectionInfo collection = getDefaultSchema().getCollection(headEntity.getType(), collectionName); + + final UUID app = appToFilter; + + + IdQueryExecutor idQueryExecutor = new IdQueryExecutor(query.getCursor()) { + @Override + protected rx.Observable<ResultsPage<Id>> buildNewResultsPage( + final Optional<String> cursor) { + + final CollectionSearch search = + new CollectionSearch(new ApplicationScopeImpl(new SimpleId(app, "application")), new SimpleId(app, "application"), collectionName, collection.getType(), query.getLimit(), + queryString, cursor); + + return collectionService.searchCollectionIds(search); + } + }; + + while ( idQueryExecutor.hasNext() ){ + + Results results = idQueryExecutor.next(); + + List<UUID> ids = results.getIds(); + + ids.forEach( uuid -> { + + try { + org.apache.usergrid.persistence.Entity retrieved = em.get(new SimpleEntityRef(entityType, uuid)); + + long timestamp = 0; + String dateString = "NOT TIME-BASED"; + if (UUIDUtils.isTimeBased(uuid)){ + timestamp = UUIDUtils.getTimestampInMillis(uuid); + Date uuidDate = new Date(timestamp); + DateFormat df = new SimpleDateFormat(); + df.setTimeZone(TimeZone.getTimeZone("GMT")); + dateString = df.format(uuidDate) + " GMT"; + } + + if ( retrieved != null ){ + + logger.info("{} - {} - entity data found", uuid, dateString); + }else{ + logger.info("{} - {} - entity data NOT found", uuid, dateString); + } + } catch (Exception e) { + logger.error("{} - exception while trying to load entity data, {} ", uuid, e.getMessage()); + } + + + }); + + } + + + } +} http://git-wip-us.apache.org/repos/asf/usergrid/blob/a7639d55/stack/tools/src/main/java/org/apache/usergrid/tools/ExportApp.java ---------------------------------------------------------------------- diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/ExportApp.java b/stack/tools/src/main/java/org/apache/usergrid/tools/ExportApp.java index 985c5f0..db59521 100644 --- a/stack/tools/src/main/java/org/apache/usergrid/tools/ExportApp.java +++ b/stack/tools/src/main/java/org/apache/usergrid/tools/ExportApp.java @@ -24,6 +24,8 @@ import org.apache.usergrid.persistence.Entity; import org.apache.usergrid.persistence.EntityManager; import org.apache.usergrid.persistence.Query; import org.apache.usergrid.persistence.Results; +import org.apache.usergrid.tools.export.ExportConnection; +import org.apache.usergrid.tools.export.ExportEntity; import org.apache.usergrid.utils.StringUtils; import org.codehaus.jackson.JsonGenerator; import org.codehaus.jackson.map.ObjectMapper; @@ -195,7 +197,7 @@ public class ExportApp extends ExportingToolBase { /** * Emits entities of collection. */ - class EntityObservable implements rx.Observable.OnSubscribe<ExportEntity> { + private class EntityObservable implements rx.Observable.OnSubscribe<ExportEntity> { EntityManager em; String collection; @@ -264,7 +266,7 @@ public class ExportApp extends ExportingToolBase { /** * Emits connections of an entity. */ - class ConnectionsObservable implements rx.Observable.OnSubscribe<ExportConnection> { + private class ConnectionsObservable implements rx.Observable.OnSubscribe<ExportConnection> { EntityManager em; ExportEntity exportEntity; @@ -330,7 +332,7 @@ public class ExportApp extends ExportingToolBase { /** * Writes entities to JSON file. */ - class EntityWriteAction implements Action1<ExportEntity> { + private class EntityWriteAction implements Action1<ExportEntity> { public void call(ExportEntity entity) { @@ -368,7 +370,7 @@ public class ExportApp extends ExportingToolBase { /** * Writes connection to JSON file. */ - class ConnectionWriteAction implements Action1<ExportConnection> { + private class ConnectionWriteAction implements Action1<ExportConnection> { public void call(ExportConnection conn) { @@ -432,111 +434,3 @@ public class ExportApp extends ExportingToolBase { } } } - - -/** - * Represents entity data to be serialized to JSON. - */ -class ExportEntity { - private String organization; - private String application; - private Entity entity; - private Map<String, Object> dictionaries; - public ExportEntity( String organization, String application, Entity entity, Map<String, Object> dictionaries ) { - this.organization = organization; - this.application = application; - this.entity = entity; - this.dictionaries = dictionaries; - } - - public String getApplication() { - return application; - } - - public void setApplication(String application) { - this.application = application; - } - - public Entity getEntity() { - return entity; - } - - public void setEntity(Entity entity) { - this.entity = entity; - } - - public Map<String, Object> getDictionaries() { - return dictionaries; - } - - public void setDictionaries(Map<String, Object> dictionaries) { - this.dictionaries = dictionaries; - } - - public String getOrganization() { - return organization; - } - - public void setOrganization(String organization) { - this.organization = organization; - } -} - - -/** - * Represents connection data to be serialized to JSON. - */ -class ExportConnection { - private String organization; - private String application; - private String connectionType; - private UUID sourceUuid; - private UUID targetUuid; - public ExportConnection(String organization, String application, String connectionType, UUID sourceUuid, UUID targetUuid) { - this.organization= organization; - this.application = application; - this.connectionType = connectionType; - this.sourceUuid = sourceUuid; - this.targetUuid = targetUuid; - } - - public String getApplication() { - return application; - } - - public void setApplication(String application) { - this.application = application; - } - - public String getConnectionType() { - return connectionType; - } - - public void setConnectionType(String connectionType) { - this.connectionType = connectionType; - } - - public UUID getSourceUuid() { - return sourceUuid; - } - - public void setSourceUuid(UUID sourceUuid) { - this.sourceUuid = sourceUuid; - } - - public UUID getTargetUuid() { - return targetUuid; - } - - public void setTargetUuid(UUID targetUuid) { - this.targetUuid = targetUuid; - } - - public String getOrganization() { - return organization; - } - - public void setOrganization(String organization) { - this.organization = organization; - } -} http://git-wip-us.apache.org/repos/asf/usergrid/blob/a7639d55/stack/tools/src/main/java/org/apache/usergrid/tools/ToolBase.java ---------------------------------------------------------------------- diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/ToolBase.java b/stack/tools/src/main/java/org/apache/usergrid/tools/ToolBase.java index e82e5f8..c33c5f5 100644 --- a/stack/tools/src/main/java/org/apache/usergrid/tools/ToolBase.java +++ b/stack/tools/src/main/java/org/apache/usergrid/tools/ToolBase.java @@ -99,6 +99,8 @@ public abstract class ToolBase { //job scheduler also not needed for tools System.setProperty("usergrid.scheduler.enabled", "false"); + System.setProperty("usergrid.cluster.enabled", "false"); + http://git-wip-us.apache.org/repos/asf/usergrid/blob/a7639d55/stack/tools/src/main/java/org/apache/usergrid/tools/export/ExportConnection.java ---------------------------------------------------------------------- diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/export/ExportConnection.java b/stack/tools/src/main/java/org/apache/usergrid/tools/export/ExportConnection.java new file mode 100644 index 0000000..18b4a26 --- /dev/null +++ b/stack/tools/src/main/java/org/apache/usergrid/tools/export/ExportConnection.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.usergrid.tools.export; + +import java.util.UUID; + +public class ExportConnection { + private String organization; + private String application; + private String connectionType; + private UUID sourceUuid; + private UUID targetUuid; + public ExportConnection(String organization, String application, String connectionType, UUID sourceUuid, UUID targetUuid) { + this.organization= organization; + this.application = application; + this.connectionType = connectionType; + this.sourceUuid = sourceUuid; + this.targetUuid = targetUuid; + } + + public String getApplication() { + return application; + } + + public void setApplication(String application) { + this.application = application; + } + + public String getConnectionType() { + return connectionType; + } + + public void setConnectionType(String connectionType) { + this.connectionType = connectionType; + } + + public UUID getSourceUuid() { + return sourceUuid; + } + + public void setSourceUuid(UUID sourceUuid) { + this.sourceUuid = sourceUuid; + } + + public UUID getTargetUuid() { + return targetUuid; + } + + public void setTargetUuid(UUID targetUuid) { + this.targetUuid = targetUuid; + } + + public String getOrganization() { + return organization; + } + + public void setOrganization(String organization) { + this.organization = organization; + } +} http://git-wip-us.apache.org/repos/asf/usergrid/blob/a7639d55/stack/tools/src/main/java/org/apache/usergrid/tools/export/ExportEntity.java ---------------------------------------------------------------------- diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/export/ExportEntity.java b/stack/tools/src/main/java/org/apache/usergrid/tools/export/ExportEntity.java new file mode 100644 index 0000000..800287c --- /dev/null +++ b/stack/tools/src/main/java/org/apache/usergrid/tools/export/ExportEntity.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.usergrid.tools.export; + +import org.apache.usergrid.persistence.Entity; + +import java.util.Map; + +public class ExportEntity { + private String organization; + private String application; + private Entity entity; + private Map<String, Object> dictionaries; + public ExportEntity( String organization, String application, Entity entity, Map<String, Object> dictionaries ) { + this.organization = organization; + this.application = application; + this.entity = entity; + this.dictionaries = dictionaries; + } + + public String getApplication() { + return application; + } + + public void setApplication(String application) { + this.application = application; + } + + public Entity getEntity() { + return entity; + } + + public void setEntity(Entity entity) { + this.entity = entity; + } + + public Map<String, Object> getDictionaries() { + return dictionaries; + } + + public void setDictionaries(Map<String, Object> dictionaries) { + this.dictionaries = dictionaries; + } + + public String getOrganization() { + return organization; + } + + public void setOrganization(String organization) { + this.organization = organization; + } +}
