[04/50] [abbrv] hbase git commit: HBASE-16086 TableCfWALEntryFilter and ScopeWALEntryFilter should not redundantly iterate over cells (Vincent Poon)

[syuanjiang]

HBASE-16086 TableCfWALEntryFilter and ScopeWALEntryFilter should not 
redundantly iterate over cells (Vincent Poon)


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/80d8b210
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/80d8b210
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/80d8b210

Branch: refs/heads/hbase-12439
Commit: 80d8b2100d9f4dc2a01ea6bdbded6ec52d7e4263
Parents: cc2a40a
Author: chenheng 
Authored: Sun Sep 11 09:55:08 2016 +0800
Committer: chenheng 
Committed: Sun Sep 11 09:55:08 2016 +0800

--
 .../hbase/replication/BulkLoadCellFilter.java   |  81 
 .../hbase/replication/ChainWALEntryFilter.java  |  38 +-
 .../hbase/replication/ScopeWALEntryFilter.java  |  94 --
 .../replication/TableCfWALEntryFilter.java  | 124 +++
 .../hadoop/hbase/replication/WALCellFilter.java |  41 ++
 .../TestReplicationWALEntryFilters.java |  12 +-
 6 files changed, 231 insertions(+), 159 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/80d8b210/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/BulkLoadCellFilter.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/BulkLoadCellFilter.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/BulkLoadCellFilter.java
new file mode 100644
index 000..3599d10
--- /dev/null
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/BulkLoadCellFilter.java
@@ -0,0 +1,81 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.replication;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.CellUtil;
+import org.apache.hadoop.hbase.protobuf.generated.WALProtos.BulkLoadDescriptor;
+import org.apache.hadoop.hbase.protobuf.generated.WALProtos.StoreDescriptor;
+import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
+
+import com.google.common.base.Predicate;
+
+public class BulkLoadCellFilter {
+  private static final Log LOG = LogFactory.getLog(BulkLoadCellFilter.class);
+
+  /**
+   * Filters the bulk load cell using the supplied predicate.
+   * @param cell The WAL cell to filter.
+   * @param famPredicate Returns true of given family should be removed.
+   * @return The filtered cell.
+   */
+  public Cell filterCell(Cell cell, Predicate famPredicate) {
+byte[] fam;
+BulkLoadDescriptor bld = null;
+try {
+  bld = WALEdit.getBulkLoadDescriptor(cell);
+} catch (IOException e) {
+  LOG.warn("Failed to get bulk load events information from the WAL 
file.", e);
+  return cell;
+}
+List storesList = bld.getStoresList();
+// Copy the StoreDescriptor list and update it as storesList is a 
unmodifiableList
+List copiedStoresList = new 
ArrayList(storesList);
+Iterator copiedStoresListIterator = 
copiedStoresList.iterator();
+boolean anyStoreRemoved = false;
+while (copiedStoresListIterator.hasNext()) {
+  StoreDescriptor sd = copiedStoresListIterator.next();
+  fam = sd.getFamilyName().toByteArray();
+  if (famPredicate.apply(fam)) {
+copiedStoresListIterator.remove();
+anyStoreRemoved = true;
+  }
+}
+
+if (!anyStoreRemoved) {
+  return cell;
+} else if (copiedStoresList.isEmpty()) {
+  return null;
+}
+BulkLoadDescriptor.Builder newDesc =
+BulkLoadDescriptor.newBuilder().setTableName(bld.getTableName())
+.setEncodedRegionName(bld.getEncodedRegionName())
+.setBulkloadSeqNum(bld.getBulkloadSeqNum());
+newDesc.addAllStores(copiedStoresList);
+BulkLoadDescriptor newBulkLoadDescriptor = newDesc.build();
+return 

hbase git commit: HBASE-16086 TableCfWALEntryFilter and ScopeWALEntryFilter should not redundantly iterate over cells (Vincent Poon)

[chenheng]

Repository: hbase
Updated Branches:
  refs/heads/branch-1.3 a5f0223bd -> d40140784


HBASE-16086 TableCfWALEntryFilter and ScopeWALEntryFilter should not 
redundantly iterate over cells (Vincent Poon)

Conflicts:

hbase-server/src/main/java/org/apache/hadoop/hbase/replication/ScopeWALEntryFilter.java

hbase-server/src/main/java/org/apache/hadoop/hbase/replication/TableCfWALEntryFilter.java

hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationWALEntryFilters.java


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/d4014078
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/d4014078
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/d4014078

Branch: refs/heads/branch-1.3
Commit: d4014078451325c2e1ba18a7f1775a43cde49305
Parents: a5f0223
Author: chenheng 
Authored: Sun Sep 11 09:55:08 2016 +0800
Committer: chenheng 
Committed: Sun Sep 11 10:50:54 2016 +0800

--
 .../hbase/replication/BulkLoadCellFilter.java   |  81 
 .../hbase/replication/ChainWALEntryFilter.java  |  38 +-
 .../hbase/replication/ScopeWALEntryFilter.java  |  98 ---
 .../replication/TableCfWALEntryFilter.java  | 125 +++
 .../hadoop/hbase/replication/WALCellFilter.java |  41 ++
 .../TestReplicationWALEntryFilters.java |  14 +--
 6 files changed, 234 insertions(+), 163 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/d4014078/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/BulkLoadCellFilter.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/BulkLoadCellFilter.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/BulkLoadCellFilter.java
new file mode 100644
index 000..3599d10
--- /dev/null
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/BulkLoadCellFilter.java
@@ -0,0 +1,81 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.replication;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.CellUtil;
+import org.apache.hadoop.hbase.protobuf.generated.WALProtos.BulkLoadDescriptor;
+import org.apache.hadoop.hbase.protobuf.generated.WALProtos.StoreDescriptor;
+import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
+
+import com.google.common.base.Predicate;
+
+public class BulkLoadCellFilter {
+  private static final Log LOG = LogFactory.getLog(BulkLoadCellFilter.class);
+
+  /**
+   * Filters the bulk load cell using the supplied predicate.
+   * @param cell The WAL cell to filter.
+   * @param famPredicate Returns true of given family should be removed.
+   * @return The filtered cell.
+   */
+  public Cell filterCell(Cell cell, Predicate famPredicate) {
+byte[] fam;
+BulkLoadDescriptor bld = null;
+try {
+  bld = WALEdit.getBulkLoadDescriptor(cell);
+} catch (IOException e) {
+  LOG.warn("Failed to get bulk load events information from the WAL 
file.", e);
+  return cell;
+}
+List storesList = bld.getStoresList();
+// Copy the StoreDescriptor list and update it as storesList is a 
unmodifiableList
+List copiedStoresList = new 
ArrayList(storesList);
+Iterator copiedStoresListIterator = 
copiedStoresList.iterator();
+boolean anyStoreRemoved = false;
+while (copiedStoresListIterator.hasNext()) {
+  StoreDescriptor sd = copiedStoresListIterator.next();
+  fam = sd.getFamilyName().toByteArray();
+  if (famPredicate.apply(fam)) {
+copiedStoresListIterator.remove();
+anyStoreRemoved = true;
+  }
+}
+
+if (!anyStoreRemoved) {
+  return cell;
+} else if (copiedStoresList.isEmpty()) {
+  return null;
+}
+ 

hbase git commit: HBASE-16086 TableCfWALEntryFilter and ScopeWALEntryFilter should not redundantly iterate over cells (Vincent Poon)

[chenheng]

Repository: hbase
Updated Branches:
  refs/heads/branch-1 fe57fa4da -> 94026d0d0


HBASE-16086 TableCfWALEntryFilter and ScopeWALEntryFilter should not 
redundantly iterate over cells (Vincent Poon)

Conflicts:

hbase-server/src/main/java/org/apache/hadoop/hbase/replication/ScopeWALEntryFilter.java

hbase-server/src/main/java/org/apache/hadoop/hbase/replication/TableCfWALEntryFilter.java

hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationWALEntryFilters.java


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/94026d0d
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/94026d0d
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/94026d0d

Branch: refs/heads/branch-1
Commit: 94026d0d098b3bdcb5a6251bd1e135a976f796f5
Parents: fe57fa4
Author: chenheng 
Authored: Sun Sep 11 09:55:08 2016 +0800
Committer: chenheng 
Committed: Sun Sep 11 10:48:00 2016 +0800

--
 .../hbase/replication/BulkLoadCellFilter.java   |  81 
 .../hbase/replication/ChainWALEntryFilter.java  |  38 +-
 .../hbase/replication/ScopeWALEntryFilter.java  |  98 ---
 .../replication/TableCfWALEntryFilter.java  | 125 +++
 .../hadoop/hbase/replication/WALCellFilter.java |  41 ++
 .../TestReplicationWALEntryFilters.java |  14 +--
 6 files changed, 234 insertions(+), 163 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/94026d0d/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/BulkLoadCellFilter.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/BulkLoadCellFilter.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/BulkLoadCellFilter.java
new file mode 100644
index 000..3599d10
--- /dev/null
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/BulkLoadCellFilter.java
@@ -0,0 +1,81 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.replication;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.CellUtil;
+import org.apache.hadoop.hbase.protobuf.generated.WALProtos.BulkLoadDescriptor;
+import org.apache.hadoop.hbase.protobuf.generated.WALProtos.StoreDescriptor;
+import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
+
+import com.google.common.base.Predicate;
+
+public class BulkLoadCellFilter {
+  private static final Log LOG = LogFactory.getLog(BulkLoadCellFilter.class);
+
+  /**
+   * Filters the bulk load cell using the supplied predicate.
+   * @param cell The WAL cell to filter.
+   * @param famPredicate Returns true of given family should be removed.
+   * @return The filtered cell.
+   */
+  public Cell filterCell(Cell cell, Predicate famPredicate) {
+byte[] fam;
+BulkLoadDescriptor bld = null;
+try {
+  bld = WALEdit.getBulkLoadDescriptor(cell);
+} catch (IOException e) {
+  LOG.warn("Failed to get bulk load events information from the WAL 
file.", e);
+  return cell;
+}
+List storesList = bld.getStoresList();
+// Copy the StoreDescriptor list and update it as storesList is a 
unmodifiableList
+List copiedStoresList = new 
ArrayList(storesList);
+Iterator copiedStoresListIterator = 
copiedStoresList.iterator();
+boolean anyStoreRemoved = false;
+while (copiedStoresListIterator.hasNext()) {
+  StoreDescriptor sd = copiedStoresListIterator.next();
+  fam = sd.getFamilyName().toByteArray();
+  if (famPredicate.apply(fam)) {
+copiedStoresListIterator.remove();
+anyStoreRemoved = true;
+  }
+}
+
+if (!anyStoreRemoved) {
+  return cell;
+} else if (copiedStoresList.isEmpty()) {
+  return null;
+}
+

hbase git commit: HBASE-16086 TableCfWALEntryFilter and ScopeWALEntryFilter should not redundantly iterate over cells (Vincent Poon)

[chenheng]

Repository: hbase
Updated Branches:
  refs/heads/master cc2a40a78 -> 80d8b2100


HBASE-16086 TableCfWALEntryFilter and ScopeWALEntryFilter should not 
redundantly iterate over cells (Vincent Poon)


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/80d8b210
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/80d8b210
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/80d8b210

Branch: refs/heads/master
Commit: 80d8b2100d9f4dc2a01ea6bdbded6ec52d7e4263
Parents: cc2a40a
Author: chenheng 
Authored: Sun Sep 11 09:55:08 2016 +0800
Committer: chenheng 
Committed: Sun Sep 11 09:55:08 2016 +0800

--
 .../hbase/replication/BulkLoadCellFilter.java   |  81 
 .../hbase/replication/ChainWALEntryFilter.java  |  38 +-
 .../hbase/replication/ScopeWALEntryFilter.java  |  94 --
 .../replication/TableCfWALEntryFilter.java  | 124 +++
 .../hadoop/hbase/replication/WALCellFilter.java |  41 ++
 .../TestReplicationWALEntryFilters.java |  12 +-
 6 files changed, 231 insertions(+), 159 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/80d8b210/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/BulkLoadCellFilter.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/BulkLoadCellFilter.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/BulkLoadCellFilter.java
new file mode 100644
index 000..3599d10
--- /dev/null
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/BulkLoadCellFilter.java
@@ -0,0 +1,81 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.replication;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.CellUtil;
+import org.apache.hadoop.hbase.protobuf.generated.WALProtos.BulkLoadDescriptor;
+import org.apache.hadoop.hbase.protobuf.generated.WALProtos.StoreDescriptor;
+import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
+
+import com.google.common.base.Predicate;
+
+public class BulkLoadCellFilter {
+  private static final Log LOG = LogFactory.getLog(BulkLoadCellFilter.class);
+
+  /**
+   * Filters the bulk load cell using the supplied predicate.
+   * @param cell The WAL cell to filter.
+   * @param famPredicate Returns true of given family should be removed.
+   * @return The filtered cell.
+   */
+  public Cell filterCell(Cell cell, Predicate famPredicate) {
+byte[] fam;
+BulkLoadDescriptor bld = null;
+try {
+  bld = WALEdit.getBulkLoadDescriptor(cell);
+} catch (IOException e) {
+  LOG.warn("Failed to get bulk load events information from the WAL 
file.", e);
+  return cell;
+}
+List storesList = bld.getStoresList();
+// Copy the StoreDescriptor list and update it as storesList is a 
unmodifiableList
+List copiedStoresList = new 
ArrayList(storesList);
+Iterator copiedStoresListIterator = 
copiedStoresList.iterator();
+boolean anyStoreRemoved = false;
+while (copiedStoresListIterator.hasNext()) {
+  StoreDescriptor sd = copiedStoresListIterator.next();
+  fam = sd.getFamilyName().toByteArray();
+  if (famPredicate.apply(fam)) {
+copiedStoresListIterator.remove();
+anyStoreRemoved = true;
+  }
+}
+
+if (!anyStoreRemoved) {
+  return cell;
+} else if (copiedStoresList.isEmpty()) {
+  return null;
+}
+BulkLoadDescriptor.Builder newDesc =
+BulkLoadDescriptor.newBuilder().setTableName(bld.getTableName())
+.setEncodedRegionName(bld.getEncodedRegionName())
+.setBulkloadSeqNum(bld.getBulkloadSeqNum());
+newDesc.addAllStores(copiedStoresList);
+BulkLoadDescriptor