Author: garyh
Date: Fri Sep 9 23:55:06 2011
New Revision: 1167407
URL: http://svn.apache.org/viewvc?rev=1167407&view=rev
Log:
HBASE-4331 Bypassing default actions in prePut fails sometimes with HTable
client
Added:
hbase/trunk/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionObserverBypass.java
Modified:
hbase/trunk/CHANGES.txt
hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
Modified: hbase/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hbase/trunk/CHANGES.txt?rev=1167407&r1=1167406&r2=1167407&view=diff
==============================================================================
--- hbase/trunk/CHANGES.txt (original)
+++ hbase/trunk/CHANGES.txt Fri Sep 9 23:55:06 2011
@@ -256,6 +256,8 @@ Release 0.91.0 - Unreleased
HBASE-4007 distributed log splitting can get indefinitely stuck
(Prakash Khemani)
HBASE-4301 META migration from 0.90 to trunk fails (Subbu Iyer)
+ HBASE-4331 Bypassing default actions in prePut fails sometimes with
+ HTable client (Lars Hofhansl via garyh)
IMPROVEMENTS
Modified:
hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
URL:
http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java?rev=1167407&r1=1167406&r2=1167407&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
(original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
Fri Sep 9 23:55:06 2011
@@ -1641,23 +1641,17 @@ public class HRegion implements HeapSize
BatchOperationInProgress<Pair<Put, Integer>> batchOp) throws IOException
{
/* Run coprocessor pre hook outside of locks to avoid deadlock */
if (coprocessorHost != null) {
- List<Pair<Put, Integer>> ops =
- new ArrayList<Pair<Put, Integer>>(batchOp.operations.length);
for (int i = 0; i < batchOp.operations.length; i++) {
Pair<Put, Integer> nextPair = batchOp.operations[i];
Put put = nextPair.getFirst();
Map<byte[], List<KeyValue>> familyMap = put.getFamilyMap();
if (coprocessorHost.prePut(familyMap, put.getWriteToWAL())) {
// pre hook says skip this Put
- // adjust nextIndexToProcess if we skipped before it
- if (batchOp.nextIndexToProcess > i) {
- batchOp.nextIndexToProcess--;
- }
- continue;
+ // mark as success and skip below
+ batchOp.retCodeDetails[i] = new OperationStatus(
+ OperationStatusCode.SUCCESS);
}
- ops.add(nextPair);
}
- batchOp.operations = ops.toArray(new Pair[ops.size()]);
}
long now = EnvironmentEdgeManager.currentTimeMillis();
@@ -1687,6 +1681,13 @@ public class HRegion implements HeapSize
// store the family map reference to allow for mutations
familyMaps[lastIndexExclusive] = familyMap;
+ // skip anything that "ran" already
+ if (batchOp.retCodeDetails[lastIndexExclusive].getOperationStatusCode()
+ != OperationStatusCode.NOT_RUN) {
+ lastIndexExclusive++;
+ continue;
+ }
+
// Check the families in the put. If bad, skip this one.
try {
checkFamilies(familyMap.keySet());
Added:
hbase/trunk/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionObserverBypass.java
URL:
http://svn.apache.org/viewvc/hbase/trunk/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionObserverBypass.java?rev=1167407&view=auto
==============================================================================
---
hbase/trunk/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionObserverBypass.java
(added)
+++
hbase/trunk/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionObserverBypass.java
Fri Sep 9 23:55:06 2011
@@ -0,0 +1,191 @@
+/*
+ * Copyright 2011 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.coprocessor;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.client.Delete;
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.coprocessor.BaseRegionObserver;
+import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
+import org.apache.hadoop.hbase.coprocessor.ObserverContext;
+import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+
+public class TestRegionObserverBypass {
+ private static HBaseTestingUtility util;
+ private static final byte[] tableName = Bytes.toBytes("test");
+ private static final byte[] dummy = Bytes.toBytes("dummy");
+ private static final byte[] row1 = Bytes.toBytes("r1");
+ private static final byte[] row2 = Bytes.toBytes("r2");
+ private static final byte[] row3 = Bytes.toBytes("r3");
+ private static final byte[] test = Bytes.toBytes("test");
+
+ @BeforeClass
+ public static void setUpBeforeClass() throws Exception {
+ Configuration conf = HBaseConfiguration.create();
+ conf.setStrings(CoprocessorHost.USER_REGION_COPROCESSOR_CONF_KEY,
+ TestCoprocessor.class.getName());
+ util = new HBaseTestingUtility(conf);
+ util.startMiniCluster();
+ util.createTable(tableName, new byte[][] {dummy, test});
+ }
+
+ @AfterClass
+ public static void tearDownAfterClass() throws Exception {
+ util.shutdownMiniCluster();
+ }
+
+ /**
+ * do a single put that is bypassed by a RegionObserver
+ * @throws Exception
+ */
+ @Test
+ public void testSimple() throws Exception {
+ HTable t = new HTable(util.getConfiguration(), tableName);
+ Put p = new Put(row1);
+ p.add(test,dummy,dummy);
+ // before HBASE-4331, this would throw an exception
+ t.put(p);
+ checkRowAndDelete(t,row1,0);
+ }
+
+ /**
+ * Test various multiput operations.
+ * @throws Exception
+ */
+ @Test
+ public void testMulti() throws Exception {
+ HTable t = new HTable(util.getConfiguration(), tableName);
+ List<Put> puts = new ArrayList<Put>();
+ Put p = new Put(row1);
+ p.add(dummy,dummy,dummy);
+ puts.add(p);
+ p = new Put(row2);
+ p.add(test,dummy,dummy);
+ puts.add(p);
+ p = new Put(row3);
+ p.add(test,dummy,dummy);
+ puts.add(p);
+ // before HBASE-4331, this would throw an exception
+ t.put(puts);
+ checkRowAndDelete(t,row1,1);
+ checkRowAndDelete(t,row2,0);
+ checkRowAndDelete(t,row3,0);
+
+ puts.clear();
+ p = new Put(row1);
+ p.add(test,dummy,dummy);
+ puts.add(p);
+ p = new Put(row2);
+ p.add(test,dummy,dummy);
+ puts.add(p);
+ p = new Put(row3);
+ p.add(test,dummy,dummy);
+ puts.add(p);
+ // before HBASE-4331, this would throw an exception
+ t.put(puts);
+ checkRowAndDelete(t,row1,0);
+ checkRowAndDelete(t,row2,0);
+ checkRowAndDelete(t,row3,0);
+
+ puts.clear();
+ p = new Put(row1);
+ p.add(test,dummy,dummy);
+ puts.add(p);
+ p = new Put(row2);
+ p.add(test,dummy,dummy);
+ puts.add(p);
+ p = new Put(row3);
+ p.add(dummy,dummy,dummy);
+ puts.add(p);
+ // this worked fine even before HBASE-4331
+ t.put(puts);
+ checkRowAndDelete(t,row1,0);
+ checkRowAndDelete(t,row2,0);
+ checkRowAndDelete(t,row3,1);
+
+ puts.clear();
+ p = new Put(row1);
+ p.add(dummy,dummy,dummy);
+ puts.add(p);
+ p = new Put(row2);
+ p.add(test,dummy,dummy);
+ puts.add(p);
+ p = new Put(row3);
+ p.add(dummy,dummy,dummy);
+ puts.add(p);
+ // this worked fine even before HBASE-4331
+ t.put(puts);
+ checkRowAndDelete(t,row1,1);
+ checkRowAndDelete(t,row2,0);
+ checkRowAndDelete(t,row3,1);
+
+ puts.clear();
+ p = new Put(row1);
+ p.add(test,dummy,dummy);
+ puts.add(p);
+ p = new Put(row2);
+ p.add(dummy,dummy,dummy);
+ puts.add(p);
+ p = new Put(row3);
+ p.add(test,dummy,dummy);
+ puts.add(p);
+ // before HBASE-4331, this would throw an exception
+ t.put(puts);
+ checkRowAndDelete(t,row1,0);
+ checkRowAndDelete(t,row2,1);
+ checkRowAndDelete(t,row3,0);
+ }
+
+ private void checkRowAndDelete(HTable t, byte[] row, int count) throws
IOException {
+ Get g = new Get(row);
+ Result r = t.get(g);
+ assertEquals(count, r.size());
+ Delete d = new Delete(row);
+ t.delete(d);
+ }
+
+ public static class TestCoprocessor extends BaseRegionObserver {
+ @Override
+ public void prePut(final ObserverContext<RegionCoprocessorEnvironment> e,
+ final Map<byte[], List<KeyValue>> familyMap, final boolean writeToWAL)
+ throws IOException {
+ if (familyMap.containsKey(test)) {
+ e.bypass();
+ }
+ }
+ }
+}
