Fix bootstrapping issue and limit the amount of cache used for Graph Node Shards.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/ceb50ff4 Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/ceb50ff4 Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/ceb50ff4 Branch: refs/heads/asf-site Commit: ceb50ff458f9e2cee8e6cbc6a3a4eed1f95bf638 Parents: b1d3e25 Author: Michael Russo <[email protected]> Authored: Thu Aug 18 17:49:38 2016 -0700 Committer: Michael Russo <[email protected]> Committed: Thu Aug 18 17:49:38 2016 -0700 ---------------------------------------------------------------------- .../exception/CollectionRuntimeException.java | 2 +- .../data/MigrationInfoSerializationImpl.java | 4 +- .../core/migration/util/AstyanaxUtils.java | 50 ++++++++++++++++++++ .../usergrid/persistence/graph/GraphFig.java | 2 +- .../impl/shard/impl/NodeShardCacheImpl.java | 28 ++--------- 5 files changed, 57 insertions(+), 29 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/usergrid/blob/ceb50ff4/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/exception/CollectionRuntimeException.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/exception/CollectionRuntimeException.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/exception/CollectionRuntimeException.java index 8aa2a7a..a6457cf 100644 --- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/exception/CollectionRuntimeException.java +++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/exception/CollectionRuntimeException.java @@ -61,7 +61,7 @@ public class CollectionRuntimeException extends RuntimeException { if ( getCause() instanceof BadRequestException ) { BadRequestException bre = (BadRequestException)getCause(); String msg = bre.getMessage(); - if ( msg.contains("Keyspace") && msg.contains( "does not exist" ) ) { + if ( (msg.contains("Keyspace") && msg.contains( "does not exist" )) || msg.contains("unconfigured columnfamily")) { return true; } } http://git-wip-us.apache.org/repos/asf/usergrid/blob/ceb50ff4/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/MigrationInfoSerializationImpl.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/MigrationInfoSerializationImpl.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/MigrationInfoSerializationImpl.java index 18427f7..ee04293 100644 --- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/MigrationInfoSerializationImpl.java +++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/MigrationInfoSerializationImpl.java @@ -28,7 +28,7 @@ import org.apache.cassandra.db.marshal.UTF8Type; import org.apache.usergrid.persistence.core.astyanax.*; import org.apache.usergrid.persistence.core.datastax.TableDefinition; -import org.apache.usergrid.persistence.core.migration.util.AstayanxUtils; +import org.apache.usergrid.persistence.core.migration.util.AstyanaxUtils; import org.apache.usergrid.persistence.model.entity.Id; import org.apache.usergrid.persistence.model.entity.SimpleId; @@ -143,7 +143,7 @@ public class MigrationInfoSerializationImpl implements MigrationInfoSerializatio return 0; } catch ( ConnectionException e ) { - AstayanxUtils.isKeyspaceMissing("Unable to connect to cassandra to retrieve status", e); + AstyanaxUtils.isSchemaMissing("Unable to connect to cassandra to retrieve status", e); return 0; } } http://git-wip-us.apache.org/repos/asf/usergrid/blob/ceb50ff4/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/util/AstyanaxUtils.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/util/AstyanaxUtils.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/util/AstyanaxUtils.java new file mode 100644 index 0000000..e83fedf --- /dev/null +++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/util/AstyanaxUtils.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.usergrid.persistence.core.migration.util; + + +import com.netflix.astyanax.connectionpool.exceptions.BadRequestException; + + +public class AstyanaxUtils { + + /** + * Return true if the exception is an instance of a missing keysapce + * @param rethrowMessage The message to add to the exception if rethrown + * @param cassandraException The exception from cassandar + * @return + */ + public static void isSchemaMissing(final String rethrowMessage, final Exception cassandraException ) { + + if ( cassandraException instanceof BadRequestException ) { + + //check if it's b/c the keyspace is missing, if so + final String message = cassandraException.getMessage(); + + //no op, just swallow + if( (message.contains( "why:Keyspace" ) && message.contains( "does not exist" )) + || message.contains("why:unconfigured columnfamily")){ + return; + }; + } + + throw new RuntimeException( rethrowMessage, cassandraException ); + } +} http://git-wip-us.apache.org/repos/asf/usergrid/blob/ceb50ff4/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/GraphFig.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/GraphFig.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/GraphFig.java index efd94ed..46f6f0c 100644 --- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/GraphFig.java +++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/GraphFig.java @@ -134,7 +134,7 @@ public interface GraphFig extends GuicyFig { long getShardMinDelta(); - @Default("250000") + @Default("100000") @Key(SHARD_CACHE_SIZE) long getShardCacheSize(); http://git-wip-us.apache.org/repos/asf/usergrid/blob/ceb50ff4/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/NodeShardCacheImpl.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/NodeShardCacheImpl.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/NodeShardCacheImpl.java index 545ac37..ee9602c 100644 --- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/NodeShardCacheImpl.java +++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/NodeShardCacheImpl.java @@ -72,18 +72,8 @@ import com.google.inject.Inject; @Singleton public class NodeShardCacheImpl implements NodeShardCache { - /** - * Only cache shards that have < 10k groups. This is an arbitrary amount, and may change with profiling and - * testing - */ - private static final int MAX_WEIGHT_PER_ELEMENT = 10000; - - private final NodeShardAllocation nodeShardAllocation; private final GraphFig graphFig; - - - private ListeningScheduledExecutorService refreshExecutors; private LoadingCache<CacheKey, CacheEntry> graphs; @@ -215,10 +205,10 @@ public class NodeShardCacheImpl implements NodeShardCache { //wait for a trip to cassandra .refreshAfterWrite( graphFig.getShardCacheTimeout(), TimeUnit.MILLISECONDS ) - //set our weight function, since not all shards are equal - .maximumWeight(MAX_WEIGHT_PER_ELEMENT * graphFig.getShardCacheSize() ).weigher( new ShardWeigher() ) + //set a static cache entry size here + .maximumSize(graphFig.getShardCacheSize()) - //set our shard loader + //set our shard loader .build( new ShardCacheLoader() ); } @@ -363,16 +353,4 @@ public class NodeShardCacheImpl implements NodeShardCache { //TODO, use RX for sliding window buffering and duplicate removal } - - - /** - * Calculates the weight of the entry by geting the size of the cache - */ - final class ShardWeigher implements Weigher<CacheKey, CacheEntry> { - - @Override - public int weigh( final CacheKey key, final CacheEntry value ) { - return value.getCacheSize(); - } - } }
