Adds DataCorruptionException to detect parse failure and return an empty entity, since it can no longer be read and should be deleted.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/b1aa75e8 Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/b1aa75e8 Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/b1aa75e8 Branch: refs/heads/two-dot-o-events Commit: b1aa75e8d40900f4616c55735598aa6ce772c324 Parents: 1f0c8d7 Author: Todd Nine <[email protected]> Authored: Mon Nov 17 19:06:32 2014 -0700 Committer: Todd Nine <[email protected]> Committed: Mon Nov 17 19:06:32 2014 -0700 ---------------------------------------------------------------------- .../persistence/PerformanceEntityWriteTest.java | 2 +- .../exception/DataCorruptionException.java | 32 +++++++++++++++ .../MvccEntitySerializationStrategyImpl.java | 43 ++++++++++++++++---- .../graph/GraphManagerShardConsistencyIT.java | 2 +- 4 files changed, 68 insertions(+), 11 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b1aa75e8/stack/core/src/test/java/org/apache/usergrid/persistence/PerformanceEntityWriteTest.java ---------------------------------------------------------------------- diff --git a/stack/core/src/test/java/org/apache/usergrid/persistence/PerformanceEntityWriteTest.java b/stack/core/src/test/java/org/apache/usergrid/persistence/PerformanceEntityWriteTest.java index 9dfa3c4..64342a1 100644 --- a/stack/core/src/test/java/org/apache/usergrid/persistence/PerformanceEntityWriteTest.java +++ b/stack/core/src/test/java/org/apache/usergrid/persistence/PerformanceEntityWriteTest.java @@ -147,7 +147,7 @@ public class PerformanceEntityWriteTest extends AbstractCoreIT { addToCollectionEntity.put( "key2", 2000 ); addToCollectionEntity.put( "key3", "Some value" ); - final List<EntityRef> owners = Arrays.asList( ( EntityRef ) owner1, ( EntityRef ) owner2 ); + final List<EntityRef> owners = Arrays.<EntityRef>asList( owner1, owner2 ); int i = 0; http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b1aa75e8/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/exception/DataCorruptionException.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/exception/DataCorruptionException.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/exception/DataCorruptionException.java new file mode 100644 index 0000000..3089e38 --- /dev/null +++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/exception/DataCorruptionException.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.usergrid.persistence.collection.exception; + + +/** + * Exception thrown when we are unable to parse data. This seems to be a data corruption issue with + * Abstract composite creation + */ +public class DataCorruptionException extends RuntimeException{ + + public DataCorruptionException( final String message, final Throwable cause ) { + super( message, cause ); + } +} http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b1aa75e8/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyImpl.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyImpl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyImpl.java index dc12087..b97429b 100644 --- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyImpl.java +++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyImpl.java @@ -38,6 +38,7 @@ import org.apache.usergrid.persistence.collection.CollectionScope; import org.apache.usergrid.persistence.collection.EntitySet; import org.apache.usergrid.persistence.collection.MvccEntity; import org.apache.usergrid.persistence.collection.exception.CollectionRuntimeException; +import org.apache.usergrid.persistence.collection.exception.DataCorruptionException; import org.apache.usergrid.persistence.collection.mvcc.MvccEntitySerializationStrategy; import org.apache.usergrid.persistence.collection.mvcc.entity.impl.MvccEntityImpl; import org.apache.usergrid.persistence.collection.serialization.EntityRepair; @@ -394,21 +395,34 @@ public class MvccEntitySerializationStrategyImpl implements MvccEntitySerializat @Override public MvccEntity parseColumn( Column<UUID> column ) { - final EntityWrapper deSerialized = column.getValue( ENTITY_JSON_SER ); + final EntityWrapper deSerialized; + final UUID version = column.getName(); + + try { + deSerialized = column.getValue( ENTITY_JSON_SER ); + } + catch ( DataCorruptionException e ) { + log.error( + "DATA CORRUPTION DETECTED when de-serializing entity with Id {} and version {}. This means the" + + " write was truncated.", + id, version ); + //return an empty entity, we can never load this one, and we don't want it to bring the system + //to a grinding halt + return new MvccEntityImpl( id, version, MvccEntity.Status.DELETED, Optional.<Entity>absent() ); + } //Inject the id into it. if ( deSerialized.entity.isPresent() ) { EntityUtils.setId( deSerialized.entity.get(), id ); } - return new MvccEntityImpl( id, column.getName(), deSerialized.status, deSerialized.entity ); + return new MvccEntityImpl( id, version, deSerialized.status, deSerialized.entity ); } } public static class EntitySerializer extends AbstractSerializer<EntityWrapper> { - public static final EntitySerializer INSTANCE = new EntitySerializer(); public static final SmileFactory f = new SmileFactory(); @@ -421,9 +435,6 @@ public class MvccEntitySerializationStrategyImpl implements MvccEntitySerializat private static byte[] VERSION = new byte[] { 0 }; - //the marker for when we're passed a "null" value - private static final byte[] EMPTY = new byte[] { 0x0 }; - public EntitySerializer() { try { @@ -479,7 +490,21 @@ public class MvccEntitySerializationStrategyImpl implements MvccEntitySerializat @Override public EntityWrapper fromByteBuffer( final ByteBuffer byteBuffer ) { - CompositeParser parser = Composites.newCompositeParser( byteBuffer ); + + /** + * We intentionally turn data corruption exceptions when we're unable to de-serialize + * the data in cassandra. If this occurs, we'll never be able to de-serialize it + * and it should be considered lost. This is an error that is occuring due to a bug + * in serializing the entity. This is a lazy recognition + repair signal for deployment with + * existing systems. + */ + CompositeParser parser; + try { + parser = Composites.newCompositeParser( byteBuffer ); + } + catch ( Exception e ) { + throw new DataCorruptionException("Unable to de-serialze entity", e); + } byte[] version = parser.read( BYTES_ARRAY_SERIALIZER ); @@ -495,7 +520,7 @@ public class MvccEntitySerializationStrategyImpl implements MvccEntitySerializat return new EntityWrapper( MvccEntity.Status.COMPLETE, Optional.<Entity>absent() ); } - Entity storedEntity = null; + Entity storedEntity; ByteBuffer jsonBytes = parser.read( BUFFER_SERIALIZER ); byte[] array = jsonBytes.array(); @@ -506,7 +531,7 @@ public class MvccEntitySerializationStrategyImpl implements MvccEntitySerializat storedEntity = mapper.readValue( array, start, length, Entity.class ); } catch ( Exception e ) { - throw new RuntimeException( "Unable to read entity data", e ); + throw new DataCorruptionException( "Unable to read entity data", e ); } final Optional<Entity> entity = Optional.of( storedEntity ); http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b1aa75e8/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerShardConsistencyIT.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerShardConsistencyIT.java b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerShardConsistencyIT.java index 0f25537..095f855 100644 --- a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerShardConsistencyIT.java +++ b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerShardConsistencyIT.java @@ -79,7 +79,7 @@ import static org.junit.Assert.assertNotNull; import static org.junit.Assert.fail; -@Ignore( "Kills cassandra, needs to be part of functional testing" ) +//@Ignore( "Kills cassandra, needs to be part of functional testing" ) public class GraphManagerShardConsistencyIT { private static final Logger log = LoggerFactory.getLogger( GraphManagerShardConsistencyIT.class );
