hbase git commit: HBASE-16645 Wrong range of Cells is caused by CellFlatMap#tailMap, headMap, and SubMap (ChiaPing Tsai)

2016-09-25 Thread tedyu 
Repository: hbase
Updated Branches:
  refs/heads/master 3896d9ed0 -> b7e0e1578


HBASE-16645 Wrong range of Cells is caused by CellFlatMap#tailMap, headMap, and 
SubMap (ChiaPing Tsai)


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/b7e0e157
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/b7e0e157
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/b7e0e157

Branch: refs/heads/master
Commit: b7e0e1578717709fc564832e95fac64a325da6aa
Parents: 3896d9e
Author: tedyu 
Authored: Sun Sep 25 06:42:32 2016 -0700
Committer: tedyu 
Committed: Sun Sep 25 06:42:32 2016 -0700

--
 .../hadoop/hbase/regionserver/CellFlatMap.java  | 100 +++---
 .../hadoop/hbase/regionserver/CellSet.java  |   6 +
 .../hbase/regionserver/TestCellFlatSet.java | 130 ++-
 3 files changed, 154 insertions(+), 82 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/b7e0e157/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellFlatMap.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellFlatMap.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellFlatMap.java
index 6d26785..c83a382 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellFlatMap.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellFlatMap.java
@@ -28,6 +28,8 @@ import java.util.Map;
 import java.util.NavigableSet;
 import java.util.NavigableMap;
 import java.util.Set;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
 
 
 /**
@@ -42,7 +44,7 @@ import java.util.Set;
  */
 @InterfaceAudience.Private
 public abstract class CellFlatMap implements NavigableMap {
-
+  private static final Log LOG = LogFactory.getLog(CellFlatMap.class);
   private final Comparator comparator;
   protected int minCellIdx   = 0;   // the index of the minimal cell (for 
sub-sets)
   protected int maxCellIdx   = 0;   // the index of the cell after the maximal 
cell (for sub-sets)
@@ -88,8 +90,9 @@ public abstract class CellFlatMap implements 
NavigableMap {
   if (compareRes == 0) {
 return mid;  // 0 means equals. We found the key
   }
-
-  if (compareRes < 0) {
+  // Key not found. Check the comparison results; reverse the meaning of
+  // the comparison in case the order is descending (using XOR)
+  if ((compareRes < 0) ^ descending) {
 // midCell is less than needle so we need to look at farther up
 begin = mid + 1;
   } else {
@@ -101,37 +104,37 @@ public abstract class CellFlatMap implements 
NavigableMap {
 return (-1 * begin)-1;
   }
 
-  /* Get the index of the given anchor key for creating subsequent set.
-  ** It doesn't matter whether the given key exists in the set or not.
-  **
-  ** taking into consideration whether
-  ** the key should be inclusive or exclusive */
+  /**
+   * Get the index of the given anchor key for creating subsequent set.
+   * It doesn't matter whether the given key exists in the set or not.
+   * taking into consideration whether
+   * the key should be inclusive or exclusive.
+   */
   private int getValidIndex(Cell key, boolean inclusive, boolean tail) {
-int index = find(key);
-int result = -1;
-
-// if the key is found and to be included, for all possibilities, the 
answer is the found index
-if (index >= 0 && inclusive) result = index;
-
-// The compliment Operator (~) converts the returned insertion point to 
the real one
-if (index<0) result = ~index;
-
-if (tail && result==-1) {
-  if (index >= 0 && !inclusive)
-result = (descending) ? index - 1 : index + 1;
-} else if (result==-1) {
-  if (index >= 0 && !inclusive)
-result = (descending) ? index + 1 : index - 1;
-}
-
-if (result < minCellIdx || result > maxCellIdx) {
-  throw new IllegalArgumentException("Index " + result + " (initial index 
" + index + ") "
-  + " out of boundary, when looking for key " + key + ". The 
minCellIdx is " + minCellIdx
-  + " and the maxCellIdx is " + maxCellIdx + ". Finally, descending? " 
+ descending
-  + " and was the key requested inclusively? " + inclusive);
+final int index = find(key);
+// get the valid (positive) insertion point from the output of the find() 
method
+int insertionPoint = index < 0 ? ~index : index;
+
+// correct the insertion point in case the given anchor key DOES EXIST in 
the set
+if (index >= 0) {
+  if ( descending && !(tail ^ inclusive)) {
+// for the descending case
+// if anchor for head set

hbase git commit: HBASE-16692 Make ByteBufferUtils#equals safer and correct (binlijin)

2016-09-25 Thread tedyu 
Repository: hbase
Updated Branches:
  refs/heads/master 21969f515 -> 3896d9ed0


HBASE-16692 Make ByteBufferUtils#equals safer and correct (binlijin)


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/3896d9ed
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/3896d9ed
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/3896d9ed

Branch: refs/heads/master
Commit: 3896d9ed0a87c77330f3f2c998a6fdafe272e2d6
Parents: 21969f5
Author: tedyu 
Authored: Sun Sep 25 06:37:40 2016 -0700
Committer: tedyu 
Committed: Sun Sep 25 06:37:40 2016 -0700

--
 .../hadoop/hbase/util/ByteBufferUtils.java  |   9 +
 .../hadoop/hbase/util/TestByteBufferUtils.java  | 441 +++
 .../hadoop/hbase/util/TestByteBufferUtils.java  | 412 -
 3 files changed, 450 insertions(+), 412 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/3896d9ed/hbase-common/src/main/java/org/apache/hadoop/hbase/util/ByteBufferUtils.java
--
diff --git 
a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/ByteBufferUtils.java 
b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/ByteBufferUtils.java
index d788c70..c491fe1 100644
--- 
a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/ByteBufferUtils.java
+++ 
b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/ByteBufferUtils.java
@@ -17,6 +17,7 @@
 package org.apache.hadoop.hbase.util;
 
 import java.io.ByteArrayOutputStream;
+import java.io.DataInput;
 import java.io.DataInputStream;
 import java.io.IOException;
 import java.io.InputStream;
@@ -573,6 +574,10 @@ public final class ByteBufferUtils {
   }
 
   public static boolean equals(ByteBuffer buf1, int o1, int l1, ByteBuffer 
buf2, int o2, int l2) {
+if ((l1 == 0) || (l2 == 0)) {
+  // both 0 length, return true, or else false
+  return l1 == l2;
+}
 // Since we're often comparing adjacent sorted data,
 // it's usual to have equal arrays except for the very last byte
 // so check that first
@@ -627,6 +632,10 @@ public final class ByteBufferUtils {
   }
 
   public static boolean equals(ByteBuffer buf1, int o1, int l1, byte[] buf2, 
int o2, int l2) {
+if ((l1 == 0) || (l2 == 0)) {
+  // both 0 length, return true, or else false
+  return l1 == l2;
+}
 // Since we're often comparing adjacent sorted data,
 // it's usual to have equal arrays except for the very last byte
 // so check that first

http://git-wip-us.apache.org/repos/asf/hbase/blob/3896d9ed/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestByteBufferUtils.java
--
diff --git 
a/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestByteBufferUtils.java
 
b/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestByteBufferUtils.java
new file mode 100644
index 000..dfbc015
--- /dev/null
+++ 
b/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestByteBufferUtils.java
@@ -0,0 +1,441 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hadoop.hbase.util;
+
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Set;
+import java.util.SortedSet;
+import java.util.TreeSet;
+
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.testclassification.MiscTests;
+import org.apache.hadoop.hbase.testclassification.SmallTests;
+import org.apache.hadoop.io.WritableUtils;
+import org.junit.Before;
+import org.junit.Test;
+import

hbase git commit: HBASE-16665 Check whether KeyValueUtil.createXXX could be replaced by CellUtil without copy

2016-09-25 Thread chenheng 
Repository: hbase
Updated Branches:
  refs/heads/master f7bb6fbf2 -> 21969f515


HBASE-16665 Check whether KeyValueUtil.createXXX could be replaced by CellUtil 
without copy


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/21969f51
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/21969f51
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/21969f51

Branch: refs/heads/master
Commit: 21969f5159e6e8f93a7b8f9c7cfe2f359f11dd27
Parents: f7bb6fb
Author: chenheng 
Authored: Sun Sep 25 14:06:55 2016 +0800
Committer: chenheng 
Committed: Sun Sep 25 14:06:55 2016 +0800

--
 .../org/apache/hadoop/hbase/client/Result.java  | 16 --
 .../java/org/apache/hadoop/hbase/CellUtil.java  | 22 
 .../hbase/io/hfile/HFilePrettyPrinter.java  |  2 +-
 .../hbase/mob/mapreduce/MemStoreWrapper.java|  3 ++-
 .../hbase/mob/mapreduce/SweepReducer.java   |  3 ++-
 .../hbase/regionserver/AbstractMemStore.java|  2 +-
 .../hbase/regionserver/HRegionFileSystem.java   |  5 +++--
 .../hbase/regionserver/StoreFileReader.java |  4 +---
 8 files changed, 46 insertions(+), 11 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/21969f51/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Result.java
--
diff --git 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Result.java 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Result.java
index f1e7cc4..98792e7 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Result.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Result.java
@@ -275,12 +275,24 @@ public class Result implements CellScannable, CellScanner 
{
 return result;
   }
 
+  private byte[] notNullBytes(final byte[] bytes) {
+if (bytes == null) {
+  return HConstants.EMPTY_BYTE_ARRAY;
+} else {
+  return bytes;
+}
+  }
+
   protected int binarySearch(final Cell [] kvs,
  final byte [] family,
  final byte [] qualifier) {
+byte[] familyNotNull = notNullBytes(family);
+byte[] qualifierNotNull = notNullBytes(qualifier);
 Cell searchTerm =
-KeyValueUtil.createFirstOnRow(CellUtil.cloneRow(kvs[0]),
-family, qualifier);
+CellUtil.createFirstOnRow(kvs[0].getRowArray(),
+kvs[0].getRowOffset(), kvs[0].getRowLength(),
+familyNotNull, 0, (byte)familyNotNull.length,
+qualifierNotNull, 0, qualifierNotNull.length);
 
 // pos === ( -(insertion point) - 1)
 int pos = Arrays.binarySearch(kvs, searchTerm, CellComparator.COMPARATOR);

http://git-wip-us.apache.org/repos/asf/hbase/blob/21969f51/hbase-common/src/main/java/org/apache/hadoop/hbase/CellUtil.java
--
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/CellUtil.java 
b/hbase-common/src/main/java/org/apache/hadoop/hbase/CellUtil.java
index 94c7189..2da71fb 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/CellUtil.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/CellUtil.java
@@ -1735,6 +1735,24 @@ public final class CellUtil {
 return new FirstOnRowCell(row, roffset, rlength);
   }
 
+  public static Cell createFirstOnRow(final byte[] row, final byte[] family, 
final byte[] col) {
+return createFirstOnRow(row, 0, (short)row.length,
+family, 0, (byte)family.length,
+col, 0, col.length);
+  }
+
+  public static Cell createFirstOnRow(final byte[] row, int roffset, short 
rlength,
+  final byte[] family, int foffset, byte 
flength,
+  final byte[] col, int coffset, int 
clength) {
+return new FirstOnRowColCell(row, roffset, rlength,
+family, foffset, flength,
+col, coffset, clength);
+  }
+
+  public static Cell createFirstOnRow(final byte[] row) {
+return createFirstOnRow(row, 0, (short)row.length);
+  }
+
   /**
* Create a Cell that is smaller than all other possible Cells for the given 
Cell's row.
* The family length is considered to be 0
@@ -1824,6 +1842,10 @@ public final class CellUtil {
 return new LastOnRowCell(cell.getRowArray(), cell.getRowOffset(), 
cell.getRowLength());
   }
 
+  public static Cell createLastOnRow(final byte[] row) {
+return new LastOnRowCell(row, 0, (short)row.length);
+  }
+
   /**
* Create a Cell that is larger than all other possible Cells for the given 
Cell's rk:cf:q. Used
* in creating "fake keys" for the multi-column Bloom filter optimization to 
skip the row/column