This is an automated email from the ASF dual-hosted git repository.

kturner pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/accumulo.git


The following commit(s) were added to refs/heads/master by this push:
     new d5a371a  ACCUMULO-4709 sanity check in Mutation (#373)
d5a371a is described below

commit d5a371a8a2dfe2235a894b8e55a40aa71b39ad46
Author: ghajos <30797379+gha...@users.noreply.github.com>
AuthorDate: Mon Feb 12 22:56:02 2018 +0100

    ACCUMULO-4709 sanity check in Mutation (#373)
---
 .../java/org/apache/accumulo/core/data/Mutation.java  | 19 +++++++++++++++++++
 .../accumulo/core/util/UnsynchronizedBuffer.java      |  4 ++++
 .../org/apache/accumulo/core/data/MutationTest.java   |  8 ++++++++
 3 files changed, 31 insertions(+)

diff --git a/core/src/main/java/org/apache/accumulo/core/data/Mutation.java 
b/core/src/main/java/org/apache/accumulo/core/data/Mutation.java
index ebc72f5..5e1a7ba 100644
--- a/core/src/main/java/org/apache/accumulo/core/data/Mutation.java
+++ b/core/src/main/java/org/apache/accumulo/core/data/Mutation.java
@@ -37,6 +37,9 @@ import org.apache.hadoop.io.Text;
 import org.apache.hadoop.io.Writable;
 import org.apache.hadoop.io.WritableUtils;
 
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+
 /**
  * Mutation represents an action that manipulates a row in a table. A mutation 
holds a list of column/value pairs that represent an atomic set of modifications
  * to make to a row.
@@ -67,6 +70,13 @@ public class Mutation implements Writable {
   static final int VALUE_SIZE_COPY_CUTOFF = 1 << 15;
 
   /**
+   * Maximum size of a mutation (2GB).
+   */
+  static final long MAX_MUTATION_SIZE = (1L << 31);
+
+  static final long SERIALIZATION_OVERHEAD = 5;
+
+  /**
    * Formats available for serializing Mutations. The formats are described in 
a <a href="doc-files/mutation-serialization.html">separate document</a>.
    */
   public static enum SERIALIZED_FORMAT {
@@ -79,6 +89,10 @@ public class Mutation implements Writable {
   private int entries;
   private List<byte[]> values;
 
+  // tracks estimated size of row.length + largeValues.length
+  @VisibleForTesting
+  long estRowAndLargeValSize = 0;
+
   private UnsynchronizedBuffer.Writer buffer;
 
   private List<ColumnUpdate> updates;
@@ -169,6 +183,7 @@ public class Mutation implements Writable {
     this.row = new byte[length];
     System.arraycopy(row, start, this.row, 0, length);
     buffer = new UnsynchronizedBuffer.Writer(initialBufferSize);
+    estRowAndLargeValSize = length + SERIALIZATION_OVERHEAD;
   }
 
   /**
@@ -306,6 +321,9 @@ public class Mutation implements Writable {
     if (buffer == null) {
       throw new IllegalStateException("Can not add to mutation after 
serializing it");
     }
+    long estimatedSizeAfterPut = estRowAndLargeValSize + buffer.size() + 
cfLength + cqLength + cv.length + (hasts ? 8 : 0) + valLength + 2
+        + 4 * SERIALIZATION_OVERHEAD;
+    Preconditions.checkArgument(estimatedSizeAfterPut < MAX_MUTATION_SIZE && 
estimatedSizeAfterPut >= 0, "Maximum mutation size must be less than 2GB ");
     put(cf, cfLength);
     put(cq, cqLength);
     put(cv);
@@ -325,6 +343,7 @@ public class Mutation implements Writable {
       System.arraycopy(val, 0, copy, 0, valLength);
       values.add(copy);
       put(-1 * values.size());
+      estRowAndLargeValSize += valLength + SERIALIZATION_OVERHEAD;
     }
 
     entries++;
diff --git 
a/core/src/main/java/org/apache/accumulo/core/util/UnsynchronizedBuffer.java 
b/core/src/main/java/org/apache/accumulo/core/util/UnsynchronizedBuffer.java
index f58737b..89671d1 100644
--- a/core/src/main/java/org/apache/accumulo/core/util/UnsynchronizedBuffer.java
+++ b/core/src/main/java/org/apache/accumulo/core/util/UnsynchronizedBuffer.java
@@ -162,6 +162,10 @@ public class UnsynchronizedBuffer {
         data[offset++] = (byte) ((i & mask) >> shiftbits);
       }
     }
+
+    public int size() {
+      return offset;
+    }
   }
 
   /**
diff --git a/core/src/test/java/org/apache/accumulo/core/data/MutationTest.java 
b/core/src/test/java/org/apache/accumulo/core/data/MutationTest.java
index 7d247f5..7f4e2fd 100644
--- a/core/src/test/java/org/apache/accumulo/core/data/MutationTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/data/MutationTest.java
@@ -680,4 +680,12 @@ public class MutationTest {
       fail("Calling Mutation#equals then Mutation#put should not result in an 
IllegalStateException.");
     }
   }
+
+  @Test(expected = IllegalArgumentException.class)
+  public void testSanityCheck() {
+    Mutation m = new Mutation("too big mutation");
+    m.put("cf", "cq1", "v");
+    m.estRowAndLargeValSize += (Long.MAX_VALUE / 2);
+    m.put("cf", "cq2", "v");
+  }
 }

-- 
To stop receiving notification emails like this one, please contact
ktur...@apache.org.

Reply via email to