[GitHub] [hbase] openinx commented on a change in pull request #163: HBASE-21995 Add a coprocessor to set HDFS ACL for hbase granted user
openinx commented on a change in pull request #163: HBASE-21995 Add a
coprocessor to set HDFS ACL for hbase granted user
URL: https://github.com/apache/hbase/pull/163#discussion_r295113424
##
File path:
hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/SnapshotScannerHDFSAclController.java
##
@@ -0,0 +1,667 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.security.access;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Optional;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.CellUtil;
+import org.apache.hadoop.hbase.HBaseInterfaceAudience;
+import org.apache.hadoop.hbase.NamespaceDescriptor;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.TableNotFoundException;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
+import org.apache.hadoop.hbase.client.Delete;
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.RegionInfo;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.ResultScanner;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.SnapshotDescription;
+import org.apache.hadoop.hbase.client.Table;
+import org.apache.hadoop.hbase.client.TableDescriptor;
+import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
+import org.apache.hadoop.hbase.coprocessor.CoreCoprocessor;
+import org.apache.hadoop.hbase.coprocessor.HasMasterServices;
+import org.apache.hadoop.hbase.coprocessor.MasterCoprocessor;
+import org.apache.hadoop.hbase.coprocessor.MasterCoprocessorEnvironment;
+import org.apache.hadoop.hbase.coprocessor.MasterObserver;
+import org.apache.hadoop.hbase.coprocessor.ObserverContext;
+import org.apache.hadoop.hbase.master.MasterServices;
+import org.apache.hadoop.hbase.security.User;
+import org.apache.hadoop.hbase.security.UserProvider;
+import org.apache.hadoop.hbase.security.access.Permission.Action;
+import
org.apache.hadoop.hbase.security.access.SnapshotScannerHDFSAclHelper.PathHelper;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.yetus.audience.InterfaceAudience;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Set HDFS ACLs to hFiles to make HBase granted users have permission to scan
snapshot
+ *
+ * To use this feature, please mask sure HDFS config:
+ *
+ * dfs.permissions.enabled = true
+ * fs.permissions.umask-mode = 027 (or smaller umask than 027)
+ *
+ *
+ *
+ * The implementation of this feature is as followings:
+ *
+ * For common directories such as 'data' and 'archive', set other
permission to '--x' to make
+ * everyone have the permission to access the directory.
+ * For namespace or table directories such as 'data/ns/table',
'archive/ns/table' and
+ * '.hbase-snapshot/snapshotName', set user 'r-x' access acl and 'r-x' default
acl when following
+ * operations happen:
+ *
+ * grant user with global, namespace or table permission;
+ * revoke user from global, namespace or table;
+ * snapshot table;
+ * truncate table;
+ *
+ *
+ * Note: Because snapshots are at table level, so this feature just
considers users with global,
+ * namespace or table permissions, ignores users with table CF or cell
permissions.
+ *
+ *
+ */
+@CoreCoprocessor
+@InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.CONFIG)
+public class SnapshotScannerHDFSAclController implements MasterCoprocessor,
MasterObserver {
+ private static final Logger LOG =
LoggerFactory.getLogger(SnapshotScannerHDFSAclController.class);
+
+ private SnapshotScannerHDFSAclHelper hdfsAclHelper = null;
+ private PathHelper pathHelper = null;
+ private FileSystem fs = null;
+ private volatile boolean initialized = false;
+ /** Provider for mapping principal names to Users */
+ private
[jira] [Commented] (HBASE-22048) Incorrect email links on website
[ https://issues.apache.org/jira/browse/HBASE-22048?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16867199#comment-16867199 ] Rishabh Jain commented on HBASE-22048: -- [~psomogyi] do you know if the new version of this plugin has been released? > Incorrect email links on website > > > Key: HBASE-22048 > URL: https://issues.apache.org/jira/browse/HBASE-22048 > Project: HBase > Issue Type: Bug > Components: website >Reporter: Peter Somogyi >Assignee: Rishabh Jain >Priority: Minor > Labels: beginner > > Project members email addresses has incorrect link. > [https://hbase.apache.org/team.html] > Instead of [apach...@apache.org|mailto:apach...@apache.org] it points to > [https://hbase.apache.org/apach...@apache.org] > This change might be related to ASF parent pom upgrade which changed the > maven-project-info-reports-plugin version. -- This message was sent by Atlassian JIRA (v7.6.3#76005)
[GitHub] [hbase] openinx commented on a change in pull request #163: HBASE-21995 Add a coprocessor to set HDFS ACL for hbase granted user
openinx commented on a change in pull request #163: HBASE-21995 Add a
coprocessor to set HDFS ACL for hbase granted user
URL: https://github.com/apache/hbase/pull/163#discussion_r295114178
##
File path:
hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/SnapshotScannerHDFSAclController.java
##
@@ -0,0 +1,652 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.security.access;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Optional;
+import java.util.Set;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.CellUtil;
+import org.apache.hadoop.hbase.HBaseInterfaceAudience;
+import org.apache.hadoop.hbase.NamespaceDescriptor;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.TableNotFoundException;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
+import org.apache.hadoop.hbase.client.Delete;
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.RegionInfo;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.ResultScanner;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.SnapshotDescription;
+import org.apache.hadoop.hbase.client.Table;
+import org.apache.hadoop.hbase.client.TableDescriptor;
+import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
+import org.apache.hadoop.hbase.coprocessor.CoreCoprocessor;
+import org.apache.hadoop.hbase.coprocessor.HasMasterServices;
+import org.apache.hadoop.hbase.coprocessor.MasterCoprocessor;
+import org.apache.hadoop.hbase.coprocessor.MasterCoprocessorEnvironment;
+import org.apache.hadoop.hbase.coprocessor.MasterObserver;
+import org.apache.hadoop.hbase.coprocessor.ObserverContext;
+import org.apache.hadoop.hbase.master.MasterServices;
+import org.apache.hadoop.hbase.security.User;
+import org.apache.hadoop.hbase.security.UserProvider;
+import org.apache.hadoop.hbase.security.access.Permission.Action;
+import
org.apache.hadoop.hbase.security.access.SnapshotScannerHDFSAclHelper.PathHelper;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.yetus.audience.InterfaceAudience;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Set HDFS ACLs to hFiles to make HBase granted users have permission to scan
snapshot
+ *
+ * To use this feature, please mask sure HDFS config:
+ *
+ * dfs.permissions.enabled = true
+ * fs.permissions.umask-mode = 027 (or smaller umask than 027)
+ *
+ *
+ *
+ * The implementation of this feature is as followings:
+ *
+ * For common directories such as 'data' and 'archive', set other
permission to '--x' to make
+ * everyone have the permission to access the directory.
+ * For namespace or table directories such as 'data/ns/table',
'archive/ns/table' and
+ * '.hbase-snapshot/snapshotName', set user 'r-x' access acl and 'r-x' default
acl when following
+ * operations happen:
+ *
+ * grant user with global, namespace or table permission;
+ * revoke user from global, namespace or table;
+ * snapshot table;
+ * truncate table;
+ *
+ *
+ * Note: Because snapshots are at table level, so this feature just
considers users with global,
+ * namespace or table permissions, ignores users with table CF or cell
permissions.
+ *
+ *
+ */
+@CoreCoprocessor
+@InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.CONFIG)
+public class SnapshotScannerHDFSAclController implements MasterCoprocessor,
MasterObserver {
+ private static final Logger LOG =
LoggerFactory.getLogger(SnapshotScannerHDFSAclController.class);
+
+ private SnapshotScannerHDFSAclHelper hdfsAclHelper = null;
+ private PathHelper pathHelper = null;
+ private FileSystem fs = null;
+ private volatile boolean initialized = false;
+ /** Provider for mapping principal names to Users */
+ private UserProvider userProvider;
+
+
[GitHub] [hbase] openinx commented on a change in pull request #163: HBASE-21995 Add a coprocessor to set HDFS ACL for hbase granted user
openinx commented on a change in pull request #163: HBASE-21995 Add a
coprocessor to set HDFS ACL for hbase granted user
URL: https://github.com/apache/hbase/pull/163#discussion_r295109861
##
File path:
hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/SnapshotScannerHDFSAclController.java
##
@@ -0,0 +1,667 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.security.access;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Optional;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.CellUtil;
+import org.apache.hadoop.hbase.HBaseInterfaceAudience;
+import org.apache.hadoop.hbase.NamespaceDescriptor;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.TableNotFoundException;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
+import org.apache.hadoop.hbase.client.Delete;
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.RegionInfo;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.ResultScanner;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.SnapshotDescription;
+import org.apache.hadoop.hbase.client.Table;
+import org.apache.hadoop.hbase.client.TableDescriptor;
+import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
+import org.apache.hadoop.hbase.coprocessor.CoreCoprocessor;
+import org.apache.hadoop.hbase.coprocessor.HasMasterServices;
+import org.apache.hadoop.hbase.coprocessor.MasterCoprocessor;
+import org.apache.hadoop.hbase.coprocessor.MasterCoprocessorEnvironment;
+import org.apache.hadoop.hbase.coprocessor.MasterObserver;
+import org.apache.hadoop.hbase.coprocessor.ObserverContext;
+import org.apache.hadoop.hbase.master.MasterServices;
+import org.apache.hadoop.hbase.security.User;
+import org.apache.hadoop.hbase.security.UserProvider;
+import org.apache.hadoop.hbase.security.access.Permission.Action;
+import
org.apache.hadoop.hbase.security.access.SnapshotScannerHDFSAclHelper.PathHelper;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.yetus.audience.InterfaceAudience;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Set HDFS ACLs to hFiles to make HBase granted users have permission to scan
snapshot
+ *
+ * To use this feature, please mask sure HDFS config:
+ *
+ * dfs.permissions.enabled = true
+ * fs.permissions.umask-mode = 027 (or smaller umask than 027)
+ *
+ *
+ *
+ * The implementation of this feature is as followings:
+ *
+ * For common directories such as 'data' and 'archive', set other
permission to '--x' to make
+ * everyone have the permission to access the directory.
+ * For namespace or table directories such as 'data/ns/table',
'archive/ns/table' and
+ * '.hbase-snapshot/snapshotName', set user 'r-x' access acl and 'r-x' default
acl when following
+ * operations happen:
+ *
+ * grant user with global, namespace or table permission;
+ * revoke user from global, namespace or table;
+ * snapshot table;
+ * truncate table;
+ *
+ *
+ * Note: Because snapshots are at table level, so this feature just
considers users with global,
+ * namespace or table permissions, ignores users with table CF or cell
permissions.
+ *
+ *
+ */
+@CoreCoprocessor
+@InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.CONFIG)
+public class SnapshotScannerHDFSAclController implements MasterCoprocessor,
MasterObserver {
+ private static final Logger LOG =
LoggerFactory.getLogger(SnapshotScannerHDFSAclController.class);
+
+ private SnapshotScannerHDFSAclHelper hdfsAclHelper = null;
+ private PathHelper pathHelper = null;
+ private FileSystem fs = null;
+ private volatile boolean initialized = false;
+ /** Provider for mapping principal names to Users */
+ private
[GitHub] [hbase] openinx commented on a change in pull request #163: HBASE-21995 Add a coprocessor to set HDFS ACL for hbase granted user
openinx commented on a change in pull request #163: HBASE-21995 Add a
coprocessor to set HDFS ACL for hbase granted user
URL: https://github.com/apache/hbase/pull/163#discussion_r295095826
##
File path:
hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/PermissionStorage.java
##
@@ -839,17 +840,23 @@ private static boolean validateCFAndCQ(byte[]
permFamily, byte[] cf, byte[] perm
}
public static boolean isGlobalEntry(byte[] entryName) {
-return entryName != null &&
TableName.valueOf(entryName).equals(ACL_TABLE_NAME);
+return !isNamespaceEntry(entryName) && entryName != null
+&& TableName.valueOf(entryName).equals(ACL_TABLE_NAME);
}
public static boolean isNamespaceEntry(String entryName) {
-return entryName != null && entryName.charAt(0) == NAMESPACE_PREFIX;
+return entryName != null && entryName.length() > 0 && entryName.charAt(0)
== NAMESPACE_PREFIX;
}
public static boolean isNamespaceEntry(byte[] entryName) {
return entryName != null && entryName.length !=0 && entryName[0] ==
NAMESPACE_PREFIX;
}
+ public static boolean isTableEntry(byte[] entryName) {
+return !isNamespaceEntry(entryName) && entryName != null
+&& !TableName.valueOf(entryName).equals(ACL_TABLE_NAME);
Review comment:
`TableName.valueOf(entryName).equals(ACL_TABLE_NAME)`
can be replaced by the :
`Bytes.equals(entryName, ACL_TABLE_NAME.toBytes())`
Because the Bytes#equals will need no the TableName box or unbox, should be
more faster.
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
[GitHub] [hbase] openinx commented on a change in pull request #163: HBASE-21995 Add a coprocessor to set HDFS ACL for hbase granted user
openinx commented on a change in pull request #163: HBASE-21995 Add a
coprocessor to set HDFS ACL for hbase granted user
URL: https://github.com/apache/hbase/pull/163#discussion_r295110653
##
File path:
hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/SnapshotScannerHDFSAclController.java
##
@@ -0,0 +1,667 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.security.access;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Optional;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.CellUtil;
+import org.apache.hadoop.hbase.HBaseInterfaceAudience;
+import org.apache.hadoop.hbase.NamespaceDescriptor;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.TableNotFoundException;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
+import org.apache.hadoop.hbase.client.Delete;
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.RegionInfo;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.ResultScanner;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.SnapshotDescription;
+import org.apache.hadoop.hbase.client.Table;
+import org.apache.hadoop.hbase.client.TableDescriptor;
+import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
+import org.apache.hadoop.hbase.coprocessor.CoreCoprocessor;
+import org.apache.hadoop.hbase.coprocessor.HasMasterServices;
+import org.apache.hadoop.hbase.coprocessor.MasterCoprocessor;
+import org.apache.hadoop.hbase.coprocessor.MasterCoprocessorEnvironment;
+import org.apache.hadoop.hbase.coprocessor.MasterObserver;
+import org.apache.hadoop.hbase.coprocessor.ObserverContext;
+import org.apache.hadoop.hbase.master.MasterServices;
+import org.apache.hadoop.hbase.security.User;
+import org.apache.hadoop.hbase.security.UserProvider;
+import org.apache.hadoop.hbase.security.access.Permission.Action;
+import
org.apache.hadoop.hbase.security.access.SnapshotScannerHDFSAclHelper.PathHelper;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.yetus.audience.InterfaceAudience;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Set HDFS ACLs to hFiles to make HBase granted users have permission to scan
snapshot
+ *
+ * To use this feature, please mask sure HDFS config:
+ *
+ * dfs.permissions.enabled = true
+ * fs.permissions.umask-mode = 027 (or smaller umask than 027)
+ *
+ *
+ *
+ * The implementation of this feature is as followings:
+ *
+ * For common directories such as 'data' and 'archive', set other
permission to '--x' to make
+ * everyone have the permission to access the directory.
+ * For namespace or table directories such as 'data/ns/table',
'archive/ns/table' and
+ * '.hbase-snapshot/snapshotName', set user 'r-x' access acl and 'r-x' default
acl when following
+ * operations happen:
+ *
+ * grant user with global, namespace or table permission;
+ * revoke user from global, namespace or table;
+ * snapshot table;
+ * truncate table;
+ *
+ *
+ * Note: Because snapshots are at table level, so this feature just
considers users with global,
+ * namespace or table permissions, ignores users with table CF or cell
permissions.
+ *
+ *
+ */
+@CoreCoprocessor
+@InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.CONFIG)
+public class SnapshotScannerHDFSAclController implements MasterCoprocessor,
MasterObserver {
+ private static final Logger LOG =
LoggerFactory.getLogger(SnapshotScannerHDFSAclController.class);
+
+ private SnapshotScannerHDFSAclHelper hdfsAclHelper = null;
+ private PathHelper pathHelper = null;
+ private FileSystem fs = null;
+ private volatile boolean initialized = false;
+ /** Provider for mapping principal names to Users */
+ private
[GitHub] [hbase] openinx commented on a change in pull request #163: HBASE-21995 Add a coprocessor to set HDFS ACL for hbase granted user
openinx commented on a change in pull request #163: HBASE-21995 Add a
coprocessor to set HDFS ACL for hbase granted user
URL: https://github.com/apache/hbase/pull/163#discussion_r295108790
##
File path:
hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/SnapshotScannerHDFSAclController.java
##
@@ -0,0 +1,667 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.security.access;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Optional;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.CellUtil;
+import org.apache.hadoop.hbase.HBaseInterfaceAudience;
+import org.apache.hadoop.hbase.NamespaceDescriptor;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.TableNotFoundException;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
+import org.apache.hadoop.hbase.client.Delete;
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.RegionInfo;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.ResultScanner;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.SnapshotDescription;
+import org.apache.hadoop.hbase.client.Table;
+import org.apache.hadoop.hbase.client.TableDescriptor;
+import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
+import org.apache.hadoop.hbase.coprocessor.CoreCoprocessor;
+import org.apache.hadoop.hbase.coprocessor.HasMasterServices;
+import org.apache.hadoop.hbase.coprocessor.MasterCoprocessor;
+import org.apache.hadoop.hbase.coprocessor.MasterCoprocessorEnvironment;
+import org.apache.hadoop.hbase.coprocessor.MasterObserver;
+import org.apache.hadoop.hbase.coprocessor.ObserverContext;
+import org.apache.hadoop.hbase.master.MasterServices;
+import org.apache.hadoop.hbase.security.User;
+import org.apache.hadoop.hbase.security.UserProvider;
+import org.apache.hadoop.hbase.security.access.Permission.Action;
+import
org.apache.hadoop.hbase.security.access.SnapshotScannerHDFSAclHelper.PathHelper;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.yetus.audience.InterfaceAudience;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Set HDFS ACLs to hFiles to make HBase granted users have permission to scan
snapshot
+ *
+ * To use this feature, please mask sure HDFS config:
+ *
+ * dfs.permissions.enabled = true
+ * fs.permissions.umask-mode = 027 (or smaller umask than 027)
+ *
+ *
+ *
+ * The implementation of this feature is as followings:
+ *
+ * For common directories such as 'data' and 'archive', set other
permission to '--x' to make
+ * everyone have the permission to access the directory.
+ * For namespace or table directories such as 'data/ns/table',
'archive/ns/table' and
+ * '.hbase-snapshot/snapshotName', set user 'r-x' access acl and 'r-x' default
acl when following
+ * operations happen:
+ *
+ * grant user with global, namespace or table permission;
+ * revoke user from global, namespace or table;
+ * snapshot table;
+ * truncate table;
+ *
+ *
+ * Note: Because snapshots are at table level, so this feature just
considers users with global,
+ * namespace or table permissions, ignores users with table CF or cell
permissions.
+ *
+ *
+ */
+@CoreCoprocessor
+@InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.CONFIG)
+public class SnapshotScannerHDFSAclController implements MasterCoprocessor,
MasterObserver {
+ private static final Logger LOG =
LoggerFactory.getLogger(SnapshotScannerHDFSAclController.class);
+
+ private SnapshotScannerHDFSAclHelper hdfsAclHelper = null;
+ private PathHelper pathHelper = null;
+ private FileSystem fs = null;
+ private volatile boolean initialized = false;
+ /** Provider for mapping principal names to Users */
+ private
[GitHub] [hbase] openinx commented on a change in pull request #163: HBASE-21995 Add a coprocessor to set HDFS ACL for hbase granted user
openinx commented on a change in pull request #163: HBASE-21995 Add a
coprocessor to set HDFS ACL for hbase granted user
URL: https://github.com/apache/hbase/pull/163#discussion_r295101742
##
File path:
hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/SnapshotScannerHDFSAclController.java
##
@@ -0,0 +1,667 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.security.access;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Optional;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.CellUtil;
+import org.apache.hadoop.hbase.HBaseInterfaceAudience;
+import org.apache.hadoop.hbase.NamespaceDescriptor;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.TableNotFoundException;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
+import org.apache.hadoop.hbase.client.Delete;
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.RegionInfo;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.ResultScanner;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.SnapshotDescription;
+import org.apache.hadoop.hbase.client.Table;
+import org.apache.hadoop.hbase.client.TableDescriptor;
+import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
+import org.apache.hadoop.hbase.coprocessor.CoreCoprocessor;
+import org.apache.hadoop.hbase.coprocessor.HasMasterServices;
+import org.apache.hadoop.hbase.coprocessor.MasterCoprocessor;
+import org.apache.hadoop.hbase.coprocessor.MasterCoprocessorEnvironment;
+import org.apache.hadoop.hbase.coprocessor.MasterObserver;
+import org.apache.hadoop.hbase.coprocessor.ObserverContext;
+import org.apache.hadoop.hbase.master.MasterServices;
+import org.apache.hadoop.hbase.security.User;
+import org.apache.hadoop.hbase.security.UserProvider;
+import org.apache.hadoop.hbase.security.access.Permission.Action;
+import
org.apache.hadoop.hbase.security.access.SnapshotScannerHDFSAclHelper.PathHelper;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.yetus.audience.InterfaceAudience;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Set HDFS ACLs to hFiles to make HBase granted users have permission to scan
snapshot
+ *
+ * To use this feature, please mask sure HDFS config:
+ *
+ * dfs.permissions.enabled = true
+ * fs.permissions.umask-mode = 027 (or smaller umask than 027)
+ *
+ *
+ *
+ * The implementation of this feature is as followings:
+ *
+ * For common directories such as 'data' and 'archive', set other
permission to '--x' to make
+ * everyone have the permission to access the directory.
+ * For namespace or table directories such as 'data/ns/table',
'archive/ns/table' and
+ * '.hbase-snapshot/snapshotName', set user 'r-x' access acl and 'r-x' default
acl when following
+ * operations happen:
+ *
+ * grant user with global, namespace or table permission;
+ * revoke user from global, namespace or table;
+ * snapshot table;
+ * truncate table;
+ *
+ *
+ * Note: Because snapshots are at table level, so this feature just
considers users with global,
+ * namespace or table permissions, ignores users with table CF or cell
permissions.
+ *
+ *
+ */
+@CoreCoprocessor
+@InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.CONFIG)
+public class SnapshotScannerHDFSAclController implements MasterCoprocessor,
MasterObserver {
+ private static final Logger LOG =
LoggerFactory.getLogger(SnapshotScannerHDFSAclController.class);
+
+ private SnapshotScannerHDFSAclHelper hdfsAclHelper = null;
+ private PathHelper pathHelper = null;
+ private FileSystem fs = null;
+ private volatile boolean initialized = false;
+ /** Provider for mapping principal names to Users */
+ private
[GitHub] [hbase] openinx commented on a change in pull request #163: HBASE-21995 Add a coprocessor to set HDFS ACL for hbase granted user
openinx commented on a change in pull request #163: HBASE-21995 Add a
coprocessor to set HDFS ACL for hbase granted user
URL: https://github.com/apache/hbase/pull/163#discussion_r295102461
##
File path:
hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/SnapshotScannerHDFSAclController.java
##
@@ -0,0 +1,667 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.security.access;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Optional;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.CellUtil;
+import org.apache.hadoop.hbase.HBaseInterfaceAudience;
+import org.apache.hadoop.hbase.NamespaceDescriptor;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.TableNotFoundException;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
+import org.apache.hadoop.hbase.client.Delete;
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.RegionInfo;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.ResultScanner;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.SnapshotDescription;
+import org.apache.hadoop.hbase.client.Table;
+import org.apache.hadoop.hbase.client.TableDescriptor;
+import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
+import org.apache.hadoop.hbase.coprocessor.CoreCoprocessor;
+import org.apache.hadoop.hbase.coprocessor.HasMasterServices;
+import org.apache.hadoop.hbase.coprocessor.MasterCoprocessor;
+import org.apache.hadoop.hbase.coprocessor.MasterCoprocessorEnvironment;
+import org.apache.hadoop.hbase.coprocessor.MasterObserver;
+import org.apache.hadoop.hbase.coprocessor.ObserverContext;
+import org.apache.hadoop.hbase.master.MasterServices;
+import org.apache.hadoop.hbase.security.User;
+import org.apache.hadoop.hbase.security.UserProvider;
+import org.apache.hadoop.hbase.security.access.Permission.Action;
+import
org.apache.hadoop.hbase.security.access.SnapshotScannerHDFSAclHelper.PathHelper;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.yetus.audience.InterfaceAudience;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Set HDFS ACLs to hFiles to make HBase granted users have permission to scan
snapshot
+ *
+ * To use this feature, please mask sure HDFS config:
+ *
+ * dfs.permissions.enabled = true
+ * fs.permissions.umask-mode = 027 (or smaller umask than 027)
+ *
+ *
+ *
+ * The implementation of this feature is as followings:
+ *
+ * For common directories such as 'data' and 'archive', set other
permission to '--x' to make
+ * everyone have the permission to access the directory.
+ * For namespace or table directories such as 'data/ns/table',
'archive/ns/table' and
+ * '.hbase-snapshot/snapshotName', set user 'r-x' access acl and 'r-x' default
acl when following
+ * operations happen:
+ *
+ * grant user with global, namespace or table permission;
+ * revoke user from global, namespace or table;
+ * snapshot table;
+ * truncate table;
+ *
+ *
+ * Note: Because snapshots are at table level, so this feature just
considers users with global,
+ * namespace or table permissions, ignores users with table CF or cell
permissions.
+ *
+ *
+ */
+@CoreCoprocessor
+@InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.CONFIG)
+public class SnapshotScannerHDFSAclController implements MasterCoprocessor,
MasterObserver {
+ private static final Logger LOG =
LoggerFactory.getLogger(SnapshotScannerHDFSAclController.class);
+
+ private SnapshotScannerHDFSAclHelper hdfsAclHelper = null;
+ private PathHelper pathHelper = null;
+ private FileSystem fs = null;
+ private volatile boolean initialized = false;
+ /** Provider for mapping principal names to Users */
+ private
[GitHub] [hbase] openinx commented on a change in pull request #163: HBASE-21995 Add a coprocessor to set HDFS ACL for hbase granted user
openinx commented on a change in pull request #163: HBASE-21995 Add a
coprocessor to set HDFS ACL for hbase granted user
URL: https://github.com/apache/hbase/pull/163#discussion_r295096034
##
File path:
hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/PermissionStorage.java
##
@@ -839,17 +840,23 @@ private static boolean validateCFAndCQ(byte[]
permFamily, byte[] cf, byte[] perm
}
public static boolean isGlobalEntry(byte[] entryName) {
-return entryName != null &&
TableName.valueOf(entryName).equals(ACL_TABLE_NAME);
+return !isNamespaceEntry(entryName) && entryName != null
+&& TableName.valueOf(entryName).equals(ACL_TABLE_NAME);
}
public static boolean isNamespaceEntry(String entryName) {
-return entryName != null && entryName.charAt(0) == NAMESPACE_PREFIX;
+return entryName != null && entryName.length() > 0 && entryName.charAt(0)
== NAMESPACE_PREFIX;
}
public static boolean isNamespaceEntry(byte[] entryName) {
return entryName != null && entryName.length !=0 && entryName[0] ==
NAMESPACE_PREFIX;
}
+ public static boolean isTableEntry(byte[] entryName) {
+return !isNamespaceEntry(entryName) && entryName != null
+&& !TableName.valueOf(entryName).equals(ACL_TABLE_NAME);
Review comment:
Or can just call the !isGlobalEntry() ? because the isGlobalEntry is the
`Bytes.equals(entryName, ACL_TABLE_NAME.toBytes())` now...
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
[GitHub] [hbase] openinx commented on a change in pull request #163: HBASE-21995 Add a coprocessor to set HDFS ACL for hbase granted user
openinx commented on a change in pull request #163: HBASE-21995 Add a
coprocessor to set HDFS ACL for hbase granted user
URL: https://github.com/apache/hbase/pull/163#discussion_r295095512
##
File path:
hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/PermissionStorage.java
##
@@ -839,17 +840,23 @@ private static boolean validateCFAndCQ(byte[]
permFamily, byte[] cf, byte[] perm
}
public static boolean isGlobalEntry(byte[] entryName) {
-return entryName != null &&
TableName.valueOf(entryName).equals(ACL_TABLE_NAME);
+return !isNamespaceEntry(entryName) && entryName != null
+&& TableName.valueOf(entryName).equals(ACL_TABLE_NAME);
}
public static boolean isNamespaceEntry(String entryName) {
-return entryName != null && entryName.charAt(0) == NAMESPACE_PREFIX;
+return entryName != null && entryName.length() > 0 && entryName.charAt(0)
== NAMESPACE_PREFIX;
Review comment:
Just call `isNamespaceEntry(Bytes.toBytes(entryName))` ?
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
[GitHub] [hbase] Apache-HBase commented on issue #323: HBASE-22414 Interruption of moving regions in RSGroup will cause regi…
Apache-HBase commented on issue #323: HBASE-22414 Interruption of moving regions in RSGroup will cause regi… URL: https://github.com/apache/hbase/pull/323#issuecomment-503388063 :confetti_ball: **+1 overall** | Vote | Subsystem | Runtime | Comment | |::|--:|:|:| | 0 | reexec | 180 | Docker mode activated. | ||| _ Prechecks _ | | +1 | hbaseanti | 0 | Patch does not have any anti-patterns. | | +1 | @author | 0 | The patch does not contain any @author tags. | | +1 | test4tests | 0 | The patch appears to include 1 new or modified test files. | ||| _ master Compile Tests _ | | +1 | mvninstall | 342 | master passed | | +1 | compile | 31 | master passed | | +1 | checkstyle | 16 | master passed | | +1 | shadedjars | 335 | branch has no errors when building our shaded downstream artifacts. | | +1 | findbugs | 50 | master passed | | +1 | javadoc | 22 | master passed | ||| _ Patch Compile Tests _ | | +1 | mvninstall | 308 | the patch passed | | +1 | compile | 30 | the patch passed | | +1 | javac | 30 | the patch passed | | +1 | checkstyle | 15 | the patch passed | | +1 | whitespace | 0 | The patch has no whitespace issues. | | +1 | shadedjars | 337 | patch has no errors when building our shaded downstream artifacts. | | +1 | hadoopcheck | 967 | Patch does not cause any errors with Hadoop 2.8.5 2.9.2 or 3.1.2. | | +1 | findbugs | 57 | the patch passed | | +1 | javadoc | 24 | the patch passed | ||| _ Other Tests _ | | +1 | unit | 471 | hbase-rsgroup in the patch passed. | | +1 | asflicense | 12 | The patch does not generate ASF License warnings. | | | | 3589 | | | Subsystem | Report/Notes | |--:|:-| | Docker | Client=17.05.0-ce Server=17.05.0-ce base: https://builds.apache.org/job/HBase-PreCommit-GitHub-PR/job/PR-323/1/artifact/out/Dockerfile | | GITHUB PR | https://github.com/apache/hbase/pull/323 | | Optional Tests | dupname asflicense javac javadoc unit findbugs shadedjars hadoopcheck hbaseanti checkstyle compile | | uname | Linux fa1dda8fc2b5 4.4.0-138-generic #164-Ubuntu SMP Tue Oct 2 17:16:02 UTC 2018 x86_64 GNU/Linux | | Build tool | maven | | Personality | /testptch/patchprocess/precommit/personality/provided.sh | | git revision | master / ac3d09e7fd | | maven | version: Apache Maven 3.5.4 (1edded0938998edf8bf061f1ceb3cfdeccf443fe; 2018-06-17T18:33:14Z) | | Default Java | 1.8.0_181 | | findbugs | v3.1.11 | | Test Results | https://builds.apache.org/job/HBase-PreCommit-GitHub-PR/job/PR-323/1/testReport/ | | Max. process+thread count | 4517 (vs. ulimit of 1) | | modules | C: hbase-rsgroup U: hbase-rsgroup | | Console output | https://builds.apache.org/job/HBase-PreCommit-GitHub-PR/job/PR-323/1/console | | Powered by | Apache Yetus 0.9.0 http://yetus.apache.org | This message was automatically generated. This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
[jira] [Commented] (HBASE-22596) [Chore] Separate the execution period between CompactionChecker and PeriodicMemStoreFlusher
[
https://issues.apache.org/jira/browse/HBASE-22596?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16867182#comment-16867182
]
Reid Chan commented on HBASE-22596:
---
bq. all of them still share the same core service with only one thread.
Yeah, at the beginning. It will spawn a new thread if
{{ChoreService#onChoreMissedStartTime}}.
> [Chore] Separate the execution period between CompactionChecker and
> PeriodicMemStoreFlusher
> ---
>
> Key: HBASE-22596
> URL: https://issues.apache.org/jira/browse/HBASE-22596
> Project: HBase
> Issue Type: Improvement
> Components: Compaction
>Reporter: Reid Chan
>Assignee: Reid Chan
>Priority: Minor
> Attachments: HBASE-22596.master.001.patch
>
>
> The story started at tuning RS performance where I found:
> {code}
> public static final String THREAD_WAKE_FREQUENCY =
> "hbase.server.thread.wakefrequency";
> ...
> this.threadWakeFrequency = conf.getInt(HConstants.THREAD_WAKE_FREQUENCY, 10 *
> 1000);
> ...
> this.compactionChecker = new CompactionChecker(this,
> this.threadWakeFrequency, this);
> this.periodicFlusher = new PeriodicMemStoreFlusher(this.threadWakeFrequency,
> this);
> {code}
> CompactionChecker and PeriodicMemStoreFlusher execution period are bound
> together. (as well as LeaseChecker)
> This issue is going to introduce two new parameters such that user/admin can
> tune them according to business workload.
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
[jira] [Commented] (HBASE-22414) Interruption of moving regions in RSGroup will cause regions on wrong rs
[ https://issues.apache.org/jira/browse/HBASE-22414?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16867166#comment-16867166 ] Xiaolin Ha commented on HBASE-22414: Added pull request. > Interruption of moving regions in RSGroup will cause regions on wrong rs > > > Key: HBASE-22414 > URL: https://issues.apache.org/jira/browse/HBASE-22414 > Project: HBase > Issue Type: Bug > Components: rsgroup >Affects Versions: 2.2.0 >Reporter: Xiaolin Ha >Assignee: Xiaolin Ha >Priority: Major > Attachments: HBASE-22414.master.001.patch > > > We bulk moving regions to target RSGroup, and each movement of region will > submit a TRSP, but one TRSP encounters exception will make the whole movement > action terminate. Later regions will not be moved to correct servers unless > reassign. > I think we can skip failed moved regions, and retry to move after all has > been traversed. -- This message was sent by Atlassian JIRA (v7.6.3#76005)
[GitHub] [hbase] sunhelly opened a new pull request #323: HBASE-22414 Interruption of moving regions in RSGroup will cause regi…
sunhelly opened a new pull request #323: HBASE-22414 Interruption of moving regions in RSGroup will cause regi… URL: https://github.com/apache/hbase/pull/323 …ons on wrong rs This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
[GitHub] [hbase] openinx commented on a change in pull request #163: HBASE-21995 Add a coprocessor to set HDFS ACL for hbase granted user
openinx commented on a change in pull request #163: HBASE-21995 Add a
coprocessor to set HDFS ACL for hbase granted user
URL: https://github.com/apache/hbase/pull/163#discussion_r295095325
##
File path:
hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/PermissionStorage.java
##
@@ -839,17 +840,23 @@ private static boolean validateCFAndCQ(byte[]
permFamily, byte[] cf, byte[] perm
}
public static boolean isGlobalEntry(byte[] entryName) {
-return entryName != null &&
TableName.valueOf(entryName).equals(ACL_TABLE_NAME);
+return !isNamespaceEntry(entryName) && entryName != null
Review comment:
Just call the `Bytes.equals(entryName, ACL_TABLE_NAME.toBytes())` ?
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
[jira] [Commented] (HBASE-22414) Interruption of moving regions in RSGroup will cause regions on wrong rs
[ https://issues.apache.org/jira/browse/HBASE-22414?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16867162#comment-16867162 ] Xiaolin Ha commented on HBASE-22414: Yes, of course. Thanks. > Interruption of moving regions in RSGroup will cause regions on wrong rs > > > Key: HBASE-22414 > URL: https://issues.apache.org/jira/browse/HBASE-22414 > Project: HBase > Issue Type: Bug > Components: rsgroup >Affects Versions: 2.2.0 >Reporter: Xiaolin Ha >Assignee: Xiaolin Ha >Priority: Major > Attachments: HBASE-22414.master.001.patch > > > We bulk moving regions to target RSGroup, and each movement of region will > submit a TRSP, but one TRSP encounters exception will make the whole movement > action terminate. Later regions will not be moved to correct servers unless > reassign. > I think we can skip failed moved regions, and retry to move after all has > been traversed. -- This message was sent by Atlassian JIRA (v7.6.3#76005)
[jira] [Commented] (HBASE-22344) Document deprecated public APIs
[
https://issues.apache.org/jira/browse/HBASE-22344?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16867155#comment-16867155
]
Hudson commented on HBASE-22344:
Results for branch branch-2
[build #2007 on
builds.a.o|https://builds.apache.org/job/HBase%20Nightly/job/branch-2/2007/]:
(x) *{color:red}-1 overall{color}*
details (if available):
(/) {color:green}+1 general checks{color}
-- For more information [see general
report|https://builds.apache.org/job/HBase%20Nightly/job/branch-2/2007//General_Nightly_Build_Report/]
(x) {color:red}-1 jdk8 hadoop2 checks{color}
-- For more information [see jdk8 (hadoop2)
report|https://builds.apache.org/job/HBase%20Nightly/job/branch-2/2007//JDK8_Nightly_Build_Report_(Hadoop2)/]
(x) {color:red}-1 jdk8 hadoop3 checks{color}
-- For more information [see jdk8 (hadoop3)
report|https://builds.apache.org/job/HBase%20Nightly/job/branch-2/2007//JDK8_Nightly_Build_Report_(Hadoop3)/]
(/) {color:green}+1 source release artifact{color}
-- See build output for details.
(/) {color:green}+1 client integration test{color}
> Document deprecated public APIs
> ---
>
> Key: HBASE-22344
> URL: https://issues.apache.org/jira/browse/HBASE-22344
> Project: HBase
> Issue Type: Improvement
> Components: API, community, documentation
>Reporter: Jan Hentschel
>Assignee: Jan Hentschel
>Priority: Major
> Fix For: 3.0.0, 1.5.0, 2.3.0, 2.0.6, 2.2.1, 2.1.6, 1.3.6, 1.4.11
>
>
> Currently some public APIs don't document when their deprecation was
> introduced and when they are expected to be removed. The documentation should
> be extended for APIs marked as public and limited private.
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
[jira] [Commented] (HBASE-22601) Misconfigured addition of peers leads to cluster shutdown.
[
https://issues.apache.org/jira/browse/HBASE-22601?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16867140#comment-16867140
]
Rushabh S Shah commented on HBASE-22601:
Fyi I am working on the fix. Since this is my first hbase jira, I am not able
to assign this ticket to me. Please add me to the contributors list.
> Misconfigured addition of peers leads to cluster shutdown.
> --
>
> Key: HBASE-22601
> URL: https://issues.apache.org/jira/browse/HBASE-22601
> Project: HBase
> Issue Type: Bug
>Affects Versions: 1.3.2
>Reporter: Rushabh S Shah
>Priority: Major
>
> Recently we added a peer to a production cluster which were in different
> kerberos realm.
> *Steps to reproduce:*
> 1. Add a misconfigured peer which is in different kerberos realm.
> 2. Remove that peer.
> 3. All region servers will start to crash.
> *RCA*
> Enabled trace logging on one Region server for a short amount of time.
> After adding peer, saw the following log lines.
> {noformat}
> 2019-06-18 22:19:20,949 INFO [main-EventThread]
> replication.ReplicationTrackerZKImpl - /hbase/replication/peers znode
> expired, triggering peerListChanged event
> 2019-06-18 22:19:20,992 INFO [main-EventThread]
> replication.ReplicationPeersZKImpl - Added new peer
> cluster=:/hbase
> 2019-06-18 22:19:21,113 INFO [main-EventThread]
> zookeeper.RecoverableZooKeeper - Process identifier=hconnection-0x794a56d6
> connecting to ZooKeeper ensemble=
> 2019-06-18 22:20:01,280 WARN [main-EventThread] zookeeper.ZKUtil -
> hconnection-0x794a56d6-0x16b56265fbebb1b, quorum=,
> baseZNode=/hbase Unable to set watcher on znode (/hbase/hbaseid)
> org.apache.zookeeper.KeeperException$AuthFailedException: KeeperErrorCode =
> AuthFailed for /hbase/hbaseid
> at
> org.apache.zookeeper.KeeperException.create(KeeperException.java:123)
> at
> org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
> at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1102)
> at
> org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:220)
> at
> org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:421)
> at
> org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:65)
> at
> org.apache.hadoop.hbase.client.ZooKeeperRegistry.getClusterId(ZooKeeperRegistry.java:105)
> at
> org.apache.hadoop.hbase.client.ConnectionManager$HConnectionImplementation.retrieveClusterId(ConnectionManager.java:922)
> at
> org.apache.hadoop.hbase.client.ConnectionManager$HConnectionImplementation.(ConnectionManager.java:706)
> at
> org.apache.hadoop.hbase.client.ConnectionManager$HConnectionImplementation.(ConnectionManager.java:638)
> at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native
> Method)
> at
> sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)
> at
> sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
> at java.lang.reflect.Constructor.newInstance(Constructor.java:423)
> at
> org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:238)
> at
> org.apache.hadoop.hbase.client.ConnectionManager.createConnection(ConnectionManager.java:432)
> at
> org.apache.hadoop.hbase.client.ConnectionManager.createConnectionInternal(ConnectionManager.java:341)
> at
> org.apache.hadoop.hbase.client.HConnectionManager.createConnection(HConnectionManager.java:144)
> at
> org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:135)
> at
> com.salesforce.hbase.replication.TenantReplicationEndpoint.init(TenantReplicationEndpoint.java:30)
> at
> org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceManager.getReplicationSource(ReplicationSourceManager.java:517)
> at
> org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceManager.addSource(ReplicationSourceManager.java:273)
> at
> org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceManager.peerListChanged(ReplicationSourceManager.java:635)
> at
> org.apache.hadoop.hbase.replication.ReplicationTrackerZKImpl$PeersWatcher.nodeChildrenChanged(ReplicationTrackerZKImpl.java:192)
> at
> org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher.process(ZooKeeperWatcher.java:643)
> at
> org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:544)
> at
> org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:519)
> 2019-06-18 22:20:42,999 WARN [Source,] zookeeper.ZKUtil -
> connection
[jira] [Updated] (HBASE-22601) Misconfigured addition of peers leads to cluster shutdown.
[
https://issues.apache.org/jira/browse/HBASE-22601?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Rushabh S Shah updated HBASE-22601:
---
Description:
Recently we added a peer to a production cluster which were in different
kerberos realm.
*Steps to reproduce:*
1. Add a misconfigured peer which is in different kerberos realm.
2. Remove that peer.
3. All region servers will start to crash.
*RCA*
Enabled trace logging on one Region server for a short amount of time.
After adding peer, saw the following log lines.
{noformat}
2019-06-18 22:19:20,949 INFO [main-EventThread]
replication.ReplicationTrackerZKImpl - /hbase/replication/peers znode expired,
triggering peerListChanged event
2019-06-18 22:19:20,992 INFO [main-EventThread]
replication.ReplicationPeersZKImpl - Added new peer
cluster=:/hbase
2019-06-18 22:19:21,113 INFO [main-EventThread] zookeeper.RecoverableZooKeeper
- Process identifier=hconnection-0x794a56d6 connecting to ZooKeeper
ensemble=
2019-06-18 22:20:01,280 WARN [main-EventThread] zookeeper.ZKUtil -
hconnection-0x794a56d6-0x16b56265fbebb1b, quorum=,
baseZNode=/hbase Unable to set watcher on znode (/hbase/hbaseid)
org.apache.zookeeper.KeeperException$AuthFailedException: KeeperErrorCode =
AuthFailed for /hbase/hbaseid
at org.apache.zookeeper.KeeperException.create(KeeperException.java:123)
at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1102)
at
org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:220)
at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:421)
at
org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:65)
at
org.apache.hadoop.hbase.client.ZooKeeperRegistry.getClusterId(ZooKeeperRegistry.java:105)
at
org.apache.hadoop.hbase.client.ConnectionManager$HConnectionImplementation.retrieveClusterId(ConnectionManager.java:922)
at
org.apache.hadoop.hbase.client.ConnectionManager$HConnectionImplementation.(ConnectionManager.java:706)
at
org.apache.hadoop.hbase.client.ConnectionManager$HConnectionImplementation.(ConnectionManager.java:638)
at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
at
sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)
at
sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
at java.lang.reflect.Constructor.newInstance(Constructor.java:423)
at
org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:238)
at
org.apache.hadoop.hbase.client.ConnectionManager.createConnection(ConnectionManager.java:432)
at
org.apache.hadoop.hbase.client.ConnectionManager.createConnectionInternal(ConnectionManager.java:341)
at
org.apache.hadoop.hbase.client.HConnectionManager.createConnection(HConnectionManager.java:144)
at
org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:135)
at
com.salesforce.hbase.replication.TenantReplicationEndpoint.init(TenantReplicationEndpoint.java:30)
at
org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceManager.getReplicationSource(ReplicationSourceManager.java:517)
at
org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceManager.addSource(ReplicationSourceManager.java:273)
at
org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceManager.peerListChanged(ReplicationSourceManager.java:635)
at
org.apache.hadoop.hbase.replication.ReplicationTrackerZKImpl$PeersWatcher.nodeChildrenChanged(ReplicationTrackerZKImpl.java:192)
at
org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher.process(ZooKeeperWatcher.java:643)
at
org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:544)
at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:519)
2019-06-18 22:20:42,999 WARN [Source,] zookeeper.ZKUtil -
connection to cluster: -0x26b56265fe7b5cd,
quorum=, baseZNode=/hbase Unable to set watcher on znode
(/hbase/hbaseid)
org.apache.zookeeper.KeeperException$AuthFailedException: KeeperErrorCode =
AuthFailed for /hbase/hbaseid
at org.apache.zookeeper.KeeperException.create(KeeperException.java:123)
at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1102)
at
org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:220)
at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:421)
at
[jira] [Updated] (HBASE-22601) Misconfigured addition of peers leads to cluster shutdown.
[
https://issues.apache.org/jira/browse/HBASE-22601?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Rushabh S Shah updated HBASE-22601:
---
Description:
Recently we added a peer to a production cluster which were in different
kerberos realm.
*Steps to reproduce:*
1. Add a misconfigured peer which is in different kerberos realm.
2. Remove that peer.
3. All region servers will start to crash.
*RCA*
Enabled trace logging on one Region server for a short amount of time.
After adding peer, saw the following log lines.
{noformat}
2019-06-18 22:19:20,949 INFO [main-EventThread]
replication.ReplicationTrackerZKImpl - /hbase/replication/peers znode expired,
triggering peerListChanged event
2019-06-18 22:19:20,992 INFO [main-EventThread]
replication.ReplicationPeersZKImpl - Added new peer
cluster=:/hbase
2019-06-18 22:19:21,113 INFO [main-EventThread] zookeeper.RecoverableZooKeeper
- Process identifier=hconnection-0x794a56d6 connecting to ZooKeeper
ensemble=
2019-06-18 22:20:01,280 WARN [main-EventThread] zookeeper.ZKUtil -
hconnection-0x794a56d6-0x16b56265fbebb1b, quorum=,
baseZNode=/hbase Unable to set watcher on znode (/hbase/hbaseid)
org.apache.zookeeper.KeeperException$AuthFailedException: KeeperErrorCode =
AuthFailed for /hbase/hbaseid
at org.apache.zookeeper.KeeperException.create(KeeperException.java:123)
at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1102)
at
org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:220)
at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:421)
at
org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:65)
at
org.apache.hadoop.hbase.client.ZooKeeperRegistry.getClusterId(ZooKeeperRegistry.java:105)
at
org.apache.hadoop.hbase.client.ConnectionManager$HConnectionImplementation.retrieveClusterId(ConnectionManager.java:922)
at
org.apache.hadoop.hbase.client.ConnectionManager$HConnectionImplementation.(ConnectionManager.java:706)
at
org.apache.hadoop.hbase.client.ConnectionManager$HConnectionImplementation.(ConnectionManager.java:638)
at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
at
sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)
at
sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
at java.lang.reflect.Constructor.newInstance(Constructor.java:423)
at
org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:238)
at
org.apache.hadoop.hbase.client.ConnectionManager.createConnection(ConnectionManager.java:432)
at
org.apache.hadoop.hbase.client.ConnectionManager.createConnectionInternal(ConnectionManager.java:341)
at
org.apache.hadoop.hbase.client.HConnectionManager.createConnection(HConnectionManager.java:144)
at
org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:135)
at
com.salesforce.hbase.replication.TenantReplicationEndpoint.init(TenantReplicationEndpoint.java:30)
at
org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceManager.getReplicationSource(ReplicationSourceManager.java:517)
at
org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceManager.addSource(ReplicationSourceManager.java:273)
at
org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceManager.peerListChanged(ReplicationSourceManager.java:635)
at
org.apache.hadoop.hbase.replication.ReplicationTrackerZKImpl$PeersWatcher.nodeChildrenChanged(ReplicationTrackerZKImpl.java:192)
at
org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher.process(ZooKeeperWatcher.java:643)
at
org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:544)
at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:519)
2019-06-18 22:20:43,002 TRACE [Source,]
regionserver.ReplicationSource - Cannot contact the peer's zk ensemble,
sleeping 1000 times 1
2019-06-18 22:20:44,008 TRACE [Source,]
regionserver.ReplicationSource - Cannot contact the peer's zk ensemble,
sleeping 1000 times 2
{noformat}
This goes on and on until we removed the peer.
After removing the peer,
{noformat}
2019-06-18 22:21:20,731 INFO [main-EventThread]
replication.ReplicationTrackerZKImpl - /hbase/replication/peers/
znode expired, triggering peerRemoved event
2019-06-18 22:21:20,731 INFO [main-EventThread]
regionserver.ReplicationSourceManager - Closing the following queue
, currently have 2 and another 0 that were recovered
2019-06-18 22:21:20,733 INFO [main-EventThread]
[jira] [Comment Edited] (HBASE-18373) fstat unimplemented error on Rasbian Jessie
[ https://issues.apache.org/jira/browse/HBASE-18373?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16867138#comment-16867138 ] Toshihiro Suzuki edited comment on HBASE-18373 at 6/18/19 11:57 PM: It looks like mounting /tmp with noexec option can cause this issue. In this case, possible solutions are remounting /tmp with exec option or changing "java.io.tmpdir" to non /tmp directory. was (Author: brfrn169): It looks like mounting /tmp with noexec option can cause this issue. In this case, possible solutions are remounting /tmp with exec option and changing "java.io.tmpdir" to non /tmp directory. > fstat unimplemented error on Rasbian Jessie > --- > > Key: HBASE-18373 > URL: https://issues.apache.org/jira/browse/HBASE-18373 > Project: HBase > Issue Type: Bug > Components: shell >Affects Versions: 2.0.0-alpha-1 > Environment: Raspberry Pi 3 > Rasbian Jessie (latest version) >Reporter: Nathan Green >Priority: Blocker > > I have a Hbase distributed cluster setup on my Raspberry PI cluster (1 x > Namenode & 4 x Datanode) > The stable version of Hbase shell works fine... unfortunately I had hadoop > issues (mismatch of jar files gr) so I have installed latest Hbase & > installed matching version of Hadoop (2.7.1). > HBase itself is up and running and all good. Hbase shell however gives this > error: > pi@pidoop1:/opt/hbase/lib/ruby/irb $ hbase shell > 2017-07-13 12:25:11,722 WARN [main] util.NativeCodeLoader: Unable to load > native-hadoop library for your platform... using builtin-java classes where > applicable > SLF4J: Class path contains multiple SLF4J bindings. > SLF4J: Found binding in > [jar:file:/opt/hbase/lib/slf4j-log4j12-1.7.10.jar!/org/slf4j/impl/StaticLoggerBinder.class] > SLF4J: Found binding in > [jar:file:/opt/hadoop/share/hadoop/common/lib/slf4j-log4j12-1.7.10.jar!/org/slf4j/impl/StaticLoggerBinder.class] > SLF4J: See http://www.slf4j.org/codes.html#multiple_bindings for an > explanation. > SLF4J: Actual binding is of type [org.slf4j.impl.Log4jLoggerFactory] > HBase Shell > Use "help" to get list of supported commands. > Use "exit" to quit this interactive shell. > Version 2.0.0-alpha-1, rc830a0f47f58d4892dd3300032c8244d6278aecc, Wed Jun 7 > 22:05:17 PDT 2017 > Took 0.0300 seconds > NotImplementedError: fstat unimplemented unsupported or native support failed > to load; see http://wiki.jruby.org/Native-Libraries > initialize at org/jruby/RubyIO.java:1013 > open at org/jruby/RubyIO.java:1154 > initialize at > uri:classloader:/META-INF/jruby.home/lib/ruby/stdlib/irb/input-method.rb:141 > initialize at > uri:classloader:/META-INF/jruby.home/lib/ruby/stdlib/irb/context.rb:70 > initialize at > uri:classloader:/META-INF/jruby.home/lib/ruby/stdlib/irb.rb:426 > initialize at /opt/hbase/lib/ruby/irb/hirb.rb:43 >start at /opt/hbase/bin/hirb.rb:194 >at /opt/hbase/bin/hirb.rb:206 -- This message was sent by Atlassian JIRA (v7.6.3#76005)
[jira] [Commented] (HBASE-18373) fstat unimplemented error on Rasbian Jessie
[ https://issues.apache.org/jira/browse/HBASE-18373?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16867138#comment-16867138 ] Toshihiro Suzuki commented on HBASE-18373: -- It looks like mounting /tmp with noexec option can cause this issue. In this case, possible solutions are remounting /tmp with exec option and changing "java.io.tmpdir" to non /tmp directory. > fstat unimplemented error on Rasbian Jessie > --- > > Key: HBASE-18373 > URL: https://issues.apache.org/jira/browse/HBASE-18373 > Project: HBase > Issue Type: Bug > Components: shell >Affects Versions: 2.0.0-alpha-1 > Environment: Raspberry Pi 3 > Rasbian Jessie (latest version) >Reporter: Nathan Green >Priority: Blocker > > I have a Hbase distributed cluster setup on my Raspberry PI cluster (1 x > Namenode & 4 x Datanode) > The stable version of Hbase shell works fine... unfortunately I had hadoop > issues (mismatch of jar files gr) so I have installed latest Hbase & > installed matching version of Hadoop (2.7.1). > HBase itself is up and running and all good. Hbase shell however gives this > error: > pi@pidoop1:/opt/hbase/lib/ruby/irb $ hbase shell > 2017-07-13 12:25:11,722 WARN [main] util.NativeCodeLoader: Unable to load > native-hadoop library for your platform... using builtin-java classes where > applicable > SLF4J: Class path contains multiple SLF4J bindings. > SLF4J: Found binding in > [jar:file:/opt/hbase/lib/slf4j-log4j12-1.7.10.jar!/org/slf4j/impl/StaticLoggerBinder.class] > SLF4J: Found binding in > [jar:file:/opt/hadoop/share/hadoop/common/lib/slf4j-log4j12-1.7.10.jar!/org/slf4j/impl/StaticLoggerBinder.class] > SLF4J: See http://www.slf4j.org/codes.html#multiple_bindings for an > explanation. > SLF4J: Actual binding is of type [org.slf4j.impl.Log4jLoggerFactory] > HBase Shell > Use "help" to get list of supported commands. > Use "exit" to quit this interactive shell. > Version 2.0.0-alpha-1, rc830a0f47f58d4892dd3300032c8244d6278aecc, Wed Jun 7 > 22:05:17 PDT 2017 > Took 0.0300 seconds > NotImplementedError: fstat unimplemented unsupported or native support failed > to load; see http://wiki.jruby.org/Native-Libraries > initialize at org/jruby/RubyIO.java:1013 > open at org/jruby/RubyIO.java:1154 > initialize at > uri:classloader:/META-INF/jruby.home/lib/ruby/stdlib/irb/input-method.rb:141 > initialize at > uri:classloader:/META-INF/jruby.home/lib/ruby/stdlib/irb/context.rb:70 > initialize at > uri:classloader:/META-INF/jruby.home/lib/ruby/stdlib/irb.rb:426 > initialize at /opt/hbase/lib/ruby/irb/hirb.rb:43 >start at /opt/hbase/bin/hirb.rb:194 >at /opt/hbase/bin/hirb.rb:206 -- This message was sent by Atlassian JIRA (v7.6.3#76005)
[jira] [Created] (HBASE-22601) Misconfigured addition of peers leads to cluster shutdown.
Rushabh S Shah created HBASE-22601:
--
Summary: Misconfigured addition of peers leads to cluster shutdown.
Key: HBASE-22601
URL: https://issues.apache.org/jira/browse/HBASE-22601
Project: HBase
Issue Type: Bug
Affects Versions: 1.3.2
Reporter: Rushabh S Shah
Recently we added a peer to a production cluster which were in different
kerberos realm.
*Steps to reproduce:*
1. Add a misconfigured peer which is in different kerberos realm.
2. Remove that peer.
3. All region servers will start to crash.
*RCA*
Enabled trace logging on one Region server for a short amount of time.
After adding peer, saw the following log lines.
{noformat}
2019-06-18 22:19:20,949 INFO [main-EventThread]
replication.ReplicationTrackerZKImpl - /hbase/replication/peers znode expired,
triggering peerListChanged event
2019-06-18 22:19:20,992 INFO [main-EventThread]
replication.ReplicationPeersZKImpl - Added new peer
cluster=:/hbase
2019-06-18 22:19:21,113 INFO [main-EventThread] zookeeper.RecoverableZooKeeper
- Process identifier=hconnection-0x794a56d6 connecting to ZooKeeper
ensemble=
2019-06-18 22:20:01,280 WARN [main-EventThread] zookeeper.ZKUtil -
hconnection-0x794a56d6-0x16b56265fbebb1b, quorum=,
baseZNode=/hbase Unable to set watcher on znode (/hbase/hbaseid)
org.apache.zookeeper.KeeperException$AuthFailedException: KeeperErrorCode =
AuthFailed for /hbase/hbaseid
at org.apache.zookeeper.KeeperException.create(KeeperException.java:123)
at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1102)
at
org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:220)
at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:421)
at
org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:65)
at
org.apache.hadoop.hbase.client.ZooKeeperRegistry.getClusterId(ZooKeeperRegistry.java:105)
at
org.apache.hadoop.hbase.client.ConnectionManager$HConnectionImplementation.retrieveClusterId(ConnectionManager.java:922)
at
org.apache.hadoop.hbase.client.ConnectionManager$HConnectionImplementation.(ConnectionManager.java:706)
at
org.apache.hadoop.hbase.client.ConnectionManager$HConnectionImplementation.(ConnectionManager.java:638)
at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
at
sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)
at
sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
at java.lang.reflect.Constructor.newInstance(Constructor.java:423)
at
org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:238)
at
org.apache.hadoop.hbase.client.ConnectionManager.createConnection(ConnectionManager.java:432)
at
org.apache.hadoop.hbase.client.ConnectionManager.createConnectionInternal(ConnectionManager.java:341)
at
org.apache.hadoop.hbase.client.HConnectionManager.createConnection(HConnectionManager.java:144)
at
org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:135)
at
com.salesforce.hbase.replication.TenantReplicationEndpoint.init(TenantReplicationEndpoint.java:30)
at
org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceManager.getReplicationSource(ReplicationSourceManager.java:517)
at
org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceManager.addSource(ReplicationSourceManager.java:273)
at
org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceManager.peerListChanged(ReplicationSourceManager.java:635)
at
org.apache.hadoop.hbase.replication.ReplicationTrackerZKImpl$PeersWatcher.nodeChildrenChanged(ReplicationTrackerZKImpl.java:192)
at
org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher.process(ZooKeeperWatcher.java:643)
at
org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:544)
at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:519)
2019-06-18 22:20:43,002 TRACE [Source,]
regionserver.ReplicationSource - Cannot contact the peer's zk ensemble,
sleeping 1000 times 1
2019-06-18 22:20:44,008 TRACE [Source,]
regionserver.ReplicationSource - Cannot contact the peer's zk ensemble,
sleeping 1000 times 2
{noformat}
This goes on and on until we removed the peer.
After removing the peer,
{noformat}
2019-06-18 22:21:20,731 INFO [main-EventThread]
replication.ReplicationTrackerZKImpl - /hbase/replication/peers/
znode expired, triggering peerRemoved event
2019-06-18 22:21:20,731 INFO [main-EventThread]
regionserver.ReplicationSourceManager -
[jira] [Comment Edited] (HBASE-18373) fstat unimplemented error on Rasbian Jessie
[ https://issues.apache.org/jira/browse/HBASE-18373?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16836830#comment-16836830 ] Toshihiro Suzuki edited comment on HBASE-18373 at 6/18/19 11:52 PM: Looks this issue is not an HBase side problem, but an issue with JRuby and Oracle's JVM. See the following for details: https://github.com/jruby/jruby/issues/2913 https://github.com/elastic/logstash/issues/3127#issuecomment-101068714 https://github.com/elastic/logstash/issues/2962 Closing as not a problem. was (Author: brfrn169): Looks this issue is not HBase side problem, but an issue with JRuby and Oracle's JVM. See the following for details: https://github.com/jruby/jruby/issues/2913 https://github.com/elastic/logstash/issues/3127#issuecomment-101068714 https://github.com/elastic/logstash/issues/2962 Closing as not a problem. > fstat unimplemented error on Rasbian Jessie > --- > > Key: HBASE-18373 > URL: https://issues.apache.org/jira/browse/HBASE-18373 > Project: HBase > Issue Type: Bug > Components: shell >Affects Versions: 2.0.0-alpha-1 > Environment: Raspberry Pi 3 > Rasbian Jessie (latest version) >Reporter: Nathan Green >Priority: Blocker > > I have a Hbase distributed cluster setup on my Raspberry PI cluster (1 x > Namenode & 4 x Datanode) > The stable version of Hbase shell works fine... unfortunately I had hadoop > issues (mismatch of jar files gr) so I have installed latest Hbase & > installed matching version of Hadoop (2.7.1). > HBase itself is up and running and all good. Hbase shell however gives this > error: > pi@pidoop1:/opt/hbase/lib/ruby/irb $ hbase shell > 2017-07-13 12:25:11,722 WARN [main] util.NativeCodeLoader: Unable to load > native-hadoop library for your platform... using builtin-java classes where > applicable > SLF4J: Class path contains multiple SLF4J bindings. > SLF4J: Found binding in > [jar:file:/opt/hbase/lib/slf4j-log4j12-1.7.10.jar!/org/slf4j/impl/StaticLoggerBinder.class] > SLF4J: Found binding in > [jar:file:/opt/hadoop/share/hadoop/common/lib/slf4j-log4j12-1.7.10.jar!/org/slf4j/impl/StaticLoggerBinder.class] > SLF4J: See http://www.slf4j.org/codes.html#multiple_bindings for an > explanation. > SLF4J: Actual binding is of type [org.slf4j.impl.Log4jLoggerFactory] > HBase Shell > Use "help" to get list of supported commands. > Use "exit" to quit this interactive shell. > Version 2.0.0-alpha-1, rc830a0f47f58d4892dd3300032c8244d6278aecc, Wed Jun 7 > 22:05:17 PDT 2017 > Took 0.0300 seconds > NotImplementedError: fstat unimplemented unsupported or native support failed > to load; see http://wiki.jruby.org/Native-Libraries > initialize at org/jruby/RubyIO.java:1013 > open at org/jruby/RubyIO.java:1154 > initialize at > uri:classloader:/META-INF/jruby.home/lib/ruby/stdlib/irb/input-method.rb:141 > initialize at > uri:classloader:/META-INF/jruby.home/lib/ruby/stdlib/irb/context.rb:70 > initialize at > uri:classloader:/META-INF/jruby.home/lib/ruby/stdlib/irb.rb:426 > initialize at /opt/hbase/lib/ruby/irb/hirb.rb:43 >start at /opt/hbase/bin/hirb.rb:194 >at /opt/hbase/bin/hirb.rb:206 -- This message was sent by Atlassian JIRA (v7.6.3#76005)
[jira] [Commented] (HBASE-22464) Improvements to hbase-vote script
[
https://issues.apache.org/jira/browse/HBASE-22464?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16867055#comment-16867055
]
HBase QA commented on HBASE-22464:
--
| (/) *{color:green}+1 overall{color}* |
\\
\\
|| Vote || Subsystem || Runtime || Comment ||
| {color:blue}0{color} | {color:blue} reexec {color} | {color:blue} 0m
21s{color} | {color:blue} Docker mode activated. {color} |
|| || || || {color:brown} Prechecks {color} ||
| {color:blue}0{color} | {color:blue} shelldocs {color} | {color:blue} 0m
1s{color} | {color:blue} Shelldocs was not available. {color} |
| {color:green}+1{color} | {color:green} @author {color} | {color:green} 0m
0s{color} | {color:green} The patch does not contain any @author tags. {color} |
|| || || || {color:brown} master Compile Tests {color} ||
|| || || || {color:brown} Patch Compile Tests {color} ||
| {color:green}+1{color} | {color:green} shellcheck {color} | {color:green} 0m
0s{color} | {color:green} There were no new shellcheck issues. {color} |
| {color:green}+1{color} | {color:green} whitespace {color} | {color:green} 0m
0s{color} | {color:green} The patch has no whitespace issues. {color} |
|| || || || {color:brown} Other Tests {color} ||
| {color:green}+1{color} | {color:green} asflicense {color} | {color:green} 0m
9s{color} | {color:green} The patch does not generate ASF License warnings.
{color} |
| {color:black}{color} | {color:black} {color} | {color:black} 0m 48s{color} |
{color:black} {color} |
\\
\\
|| Subsystem || Report/Notes ||
| Docker | Client=17.05.0-ce Server=17.05.0-ce base:
https://builds.apache.org/job/PreCommit-HBASE-Build/554/artifact/patchprocess/Dockerfile
|
| JIRA Issue | HBASE-22464 |
| JIRA Patch URL |
https://issues.apache.org/jira/secure/attachment/12972141/HBASE-22464.v04.patch
|
| Optional Tests | dupname asflicense shellcheck shelldocs |
| uname | Linux c2f394bf4104 4.4.0-143-generic #169~14.04.2-Ubuntu SMP Wed Feb
13 15:00:41 UTC 2019 x86_64 GNU/Linux |
| Build tool | maven |
| Personality | dev-support/hbase-personality.sh |
| git revision | master / ac3d09e7fd |
| maven | version: Apache Maven 3.5.4
(1edded0938998edf8bf061f1ceb3cfdeccf443fe; 2018-06-17T18:33:14Z) |
| shellcheck | v0.4.4 |
| Max. process+thread count | 40 (vs. ulimit of 1) |
| modules | C: . U: . |
| Console output |
https://builds.apache.org/job/PreCommit-HBASE-Build/554/console |
| Powered by | Apache Yetus 0.9.0 http://yetus.apache.org |
This message was automatically generated.
> Improvements to hbase-vote script
> -
>
> Key: HBASE-22464
> URL: https://issues.apache.org/jira/browse/HBASE-22464
> Project: HBase
> Issue Type: Improvement
>Affects Versions: 3.0.0, 1.5.0, 2.2.0, 1.4.9, 1.3.4, 2.3.0, 2.1.4
>Reporter: Artem Ervits
>Assignee: Artem Ervits
>Priority: Trivial
> Attachments: HBASE-22464.branch-1.patch, HBASE-22464.v01.patch,
> HBASE-22464.v02.patch, HBASE-22464.v03.patch, HBASE-22464.v04.patch
>
>
> This Jira proposes to add the following two improvements:
> # wget -N option for checking timestamp of downloaded RC files, if the local
> copy of downloaded file is identical on the server, that file will be ignored.
> # mvn -fn to never fail running the tests, previously with option -fae I
> could not get the whole suite to run w/out failing.
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
[jira] [Updated] (HBASE-22464) Improvements to hbase-vote script
[ https://issues.apache.org/jira/browse/HBASE-22464?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Artem Ervits updated HBASE-22464: - Attachment: HBASE-22464.v04.patch > Improvements to hbase-vote script > - > > Key: HBASE-22464 > URL: https://issues.apache.org/jira/browse/HBASE-22464 > Project: HBase > Issue Type: Improvement >Affects Versions: 3.0.0, 1.5.0, 2.2.0, 1.4.9, 1.3.4, 2.3.0, 2.1.4 >Reporter: Artem Ervits >Assignee: Artem Ervits >Priority: Trivial > Attachments: HBASE-22464.branch-1.patch, HBASE-22464.v01.patch, > HBASE-22464.v02.patch, HBASE-22464.v03.patch, HBASE-22464.v04.patch > > > This Jira proposes to add the following two improvements: > # wget -N option for checking timestamp of downloaded RC files, if the local > copy of downloaded file is identical on the server, that file will be ignored. > # mvn -fn to never fail running the tests, previously with option -fae I > could not get the whole suite to run w/out failing. -- This message was sent by Atlassian JIRA (v7.6.3#76005)
[jira] [Updated] (HBASE-22464) Improvements to hbase-vote script
[ https://issues.apache.org/jira/browse/HBASE-22464?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Artem Ervits updated HBASE-22464: - Status: Patch Available (was: Open) > Improvements to hbase-vote script > - > > Key: HBASE-22464 > URL: https://issues.apache.org/jira/browse/HBASE-22464 > Project: HBase > Issue Type: Improvement >Affects Versions: 2.1.4, 1.3.4, 1.4.9, 2.2.0, 3.0.0, 1.5.0, 2.3.0 >Reporter: Artem Ervits >Assignee: Artem Ervits >Priority: Trivial > Attachments: HBASE-22464.branch-1.patch, HBASE-22464.v01.patch, > HBASE-22464.v02.patch, HBASE-22464.v03.patch, HBASE-22464.v04.patch > > > This Jira proposes to add the following two improvements: > # wget -N option for checking timestamp of downloaded RC files, if the local > copy of downloaded file is identical on the server, that file will be ignored. > # mvn -fn to never fail running the tests, previously with option -fae I > could not get the whole suite to run w/out failing. -- This message was sent by Atlassian JIRA (v7.6.3#76005)
[jira] [Updated] (HBASE-22464) Improvements to hbase-vote script
[ https://issues.apache.org/jira/browse/HBASE-22464?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Artem Ervits updated HBASE-22464: - Attachment: (was: HBASE-22464.v04.patch) > Improvements to hbase-vote script > - > > Key: HBASE-22464 > URL: https://issues.apache.org/jira/browse/HBASE-22464 > Project: HBase > Issue Type: Improvement >Affects Versions: 3.0.0, 1.5.0, 2.2.0, 1.4.9, 1.3.4, 2.3.0, 2.1.4 >Reporter: Artem Ervits >Assignee: Artem Ervits >Priority: Trivial > Attachments: HBASE-22464.branch-1.patch, HBASE-22464.v01.patch, > HBASE-22464.v02.patch, HBASE-22464.v03.patch > > > This Jira proposes to add the following two improvements: > # wget -N option for checking timestamp of downloaded RC files, if the local > copy of downloaded file is identical on the server, that file will be ignored. > # mvn -fn to never fail running the tests, previously with option -fae I > could not get the whole suite to run w/out failing. -- This message was sent by Atlassian JIRA (v7.6.3#76005)
[jira] [Commented] (HBASE-22464) Improvements to hbase-vote script
[ https://issues.apache.org/jira/browse/HBASE-22464?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16867048#comment-16867048 ] Artem Ervits commented on HBASE-22464: -- [~busbey] please see .v04, it addresses shellcheck comments from yetus [https://builds.apache.org/job/PreCommit-HBASE-Build/553/artifact/patchprocess/diff-patch-shellcheck.txt] > Improvements to hbase-vote script > - > > Key: HBASE-22464 > URL: https://issues.apache.org/jira/browse/HBASE-22464 > Project: HBase > Issue Type: Improvement >Affects Versions: 3.0.0, 1.5.0, 2.2.0, 1.4.9, 1.3.4, 2.3.0, 2.1.4 >Reporter: Artem Ervits >Assignee: Artem Ervits >Priority: Trivial > Attachments: HBASE-22464.branch-1.patch, HBASE-22464.v01.patch, > HBASE-22464.v02.patch, HBASE-22464.v03.patch, HBASE-22464.v04.patch > > > This Jira proposes to add the following two improvements: > # wget -N option for checking timestamp of downloaded RC files, if the local > copy of downloaded file is identical on the server, that file will be ignored. > # mvn -fn to never fail running the tests, previously with option -fae I > could not get the whole suite to run w/out failing. -- This message was sent by Atlassian JIRA (v7.6.3#76005)
[jira] [Updated] (HBASE-22464) Improvements to hbase-vote script
[ https://issues.apache.org/jira/browse/HBASE-22464?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Artem Ervits updated HBASE-22464: - Attachment: HBASE-22464.v04.patch > Improvements to hbase-vote script > - > > Key: HBASE-22464 > URL: https://issues.apache.org/jira/browse/HBASE-22464 > Project: HBase > Issue Type: Improvement >Affects Versions: 3.0.0, 1.5.0, 2.2.0, 1.4.9, 1.3.4, 2.3.0, 2.1.4 >Reporter: Artem Ervits >Assignee: Artem Ervits >Priority: Trivial > Attachments: HBASE-22464.branch-1.patch, HBASE-22464.v01.patch, > HBASE-22464.v02.patch, HBASE-22464.v03.patch, HBASE-22464.v04.patch > > > This Jira proposes to add the following two improvements: > # wget -N option for checking timestamp of downloaded RC files, if the local > copy of downloaded file is identical on the server, that file will be ignored. > # mvn -fn to never fail running the tests, previously with option -fae I > could not get the whole suite to run w/out failing. -- This message was sent by Atlassian JIRA (v7.6.3#76005)
[jira] [Updated] (HBASE-22464) Improvements to hbase-vote script
[ https://issues.apache.org/jira/browse/HBASE-22464?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Artem Ervits updated HBASE-22464: - Status: Open (was: Patch Available) > Improvements to hbase-vote script > - > > Key: HBASE-22464 > URL: https://issues.apache.org/jira/browse/HBASE-22464 > Project: HBase > Issue Type: Improvement >Affects Versions: 2.1.4, 1.3.4, 1.4.9, 2.2.0, 3.0.0, 1.5.0, 2.3.0 >Reporter: Artem Ervits >Assignee: Artem Ervits >Priority: Trivial > Attachments: HBASE-22464.branch-1.patch, HBASE-22464.v01.patch, > HBASE-22464.v02.patch, HBASE-22464.v03.patch, HBASE-22464.v04.patch > > > This Jira proposes to add the following two improvements: > # wget -N option for checking timestamp of downloaded RC files, if the local > copy of downloaded file is identical on the server, that file will be ignored. > # mvn -fn to never fail running the tests, previously with option -fae I > could not get the whole suite to run w/out failing. -- This message was sent by Atlassian JIRA (v7.6.3#76005)
[jira] [Commented] (HBASE-22596) [Chore] Separate the execution period between CompactionChecker and PeriodicMemStoreFlusher
[
https://issues.apache.org/jira/browse/HBASE-22596?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16867040#comment-16867040
]
Zach York commented on HBASE-22596:
---
Otherwise, the change looks good. +1 from me.
> [Chore] Separate the execution period between CompactionChecker and
> PeriodicMemStoreFlusher
> ---
>
> Key: HBASE-22596
> URL: https://issues.apache.org/jira/browse/HBASE-22596
> Project: HBase
> Issue Type: Improvement
> Components: Compaction
>Reporter: Reid Chan
>Assignee: Reid Chan
>Priority: Minor
> Attachments: HBASE-22596.master.001.patch
>
>
> The story started at tuning RS performance where I found:
> {code}
> public static final String THREAD_WAKE_FREQUENCY =
> "hbase.server.thread.wakefrequency";
> ...
> this.threadWakeFrequency = conf.getInt(HConstants.THREAD_WAKE_FREQUENCY, 10 *
> 1000);
> ...
> this.compactionChecker = new CompactionChecker(this,
> this.threadWakeFrequency, this);
> this.periodicFlusher = new PeriodicMemStoreFlusher(this.threadWakeFrequency,
> this);
> {code}
> CompactionChecker and PeriodicMemStoreFlusher execution period are bound
> together. (as well as LeaseChecker)
> This issue is going to introduce two new parameters such that user/admin can
> tune them according to business workload.
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
[jira] [Commented] (HBASE-22596) [Chore] Separate the execution period between CompactionChecker and PeriodicMemStoreFlusher
[
https://issues.apache.org/jira/browse/HBASE-22596?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16867036#comment-16867036
]
Zach York commented on HBASE-22596:
---
The other consideration is that looking at the code it appears all of them
still share the same core service with only one thread. So even if you change
the wake threshold it appears that it would be possible for one of these chores
to monopolize the time.
> [Chore] Separate the execution period between CompactionChecker and
> PeriodicMemStoreFlusher
> ---
>
> Key: HBASE-22596
> URL: https://issues.apache.org/jira/browse/HBASE-22596
> Project: HBase
> Issue Type: Improvement
> Components: Compaction
>Reporter: Reid Chan
>Assignee: Reid Chan
>Priority: Minor
> Attachments: HBASE-22596.master.001.patch
>
>
> The story started at tuning RS performance where I found:
> {code}
> public static final String THREAD_WAKE_FREQUENCY =
> "hbase.server.thread.wakefrequency";
> ...
> this.threadWakeFrequency = conf.getInt(HConstants.THREAD_WAKE_FREQUENCY, 10 *
> 1000);
> ...
> this.compactionChecker = new CompactionChecker(this,
> this.threadWakeFrequency, this);
> this.periodicFlusher = new PeriodicMemStoreFlusher(this.threadWakeFrequency,
> this);
> {code}
> CompactionChecker and PeriodicMemStoreFlusher execution period are bound
> together. (as well as LeaseChecker)
> This issue is going to introduce two new parameters such that user/admin can
> tune them according to business workload.
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
[jira] [Commented] (HBASE-22464) Improvements to hbase-vote script
[
https://issues.apache.org/jira/browse/HBASE-22464?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16867032#comment-16867032
]
HBase QA commented on HBASE-22464:
--
| (x) *{color:red}-1 overall{color}* |
\\
\\
|| Vote || Subsystem || Runtime || Comment ||
| {color:blue}0{color} | {color:blue} reexec {color} | {color:blue} 1m
10s{color} | {color:blue} Docker mode activated. {color} |
|| || || || {color:brown} Prechecks {color} ||
| {color:blue}0{color} | {color:blue} shelldocs {color} | {color:blue} 0m
0s{color} | {color:blue} Shelldocs was not available. {color} |
| {color:green}+1{color} | {color:green} @author {color} | {color:green} 0m
0s{color} | {color:green} The patch does not contain any @author tags. {color} |
|| || || || {color:brown} master Compile Tests {color} ||
|| || || || {color:brown} Patch Compile Tests {color} ||
| {color:red}-1{color} | {color:red} shellcheck {color} | {color:red} 0m
1s{color} | {color:red} The patch generated 1 new + 0 unchanged - 0 fixed = 1
total (was 0) {color} |
| {color:green}+1{color} | {color:green} whitespace {color} | {color:green} 0m
0s{color} | {color:green} The patch has no whitespace issues. {color} |
|| || || || {color:brown} Other Tests {color} ||
| {color:green}+1{color} | {color:green} asflicense {color} | {color:green} 0m
14s{color} | {color:green} The patch does not generate ASF License warnings.
{color} |
| {color:black}{color} | {color:black} {color} | {color:black} 1m 40s{color} |
{color:black} {color} |
\\
\\
|| Subsystem || Report/Notes ||
| Docker | Client=17.05.0-ce Server=17.05.0-ce base:
https://builds.apache.org/job/PreCommit-HBASE-Build/553/artifact/patchprocess/Dockerfile
|
| JIRA Issue | HBASE-22464 |
| JIRA Patch URL |
https://issues.apache.org/jira/secure/attachment/12972133/HBASE-22464.v03.patch
|
| Optional Tests | dupname asflicense shellcheck shelldocs |
| uname | Linux 8fe9cf3e2b08 4.4.0-143-generic #169~14.04.2-Ubuntu SMP Wed Feb
13 15:00:41 UTC 2019 x86_64 GNU/Linux |
| Build tool | maven |
| Personality | dev-support/hbase-personality.sh |
| git revision | master / ac3d09e7fd |
| maven | version: Apache Maven 3.5.4
(1edded0938998edf8bf061f1ceb3cfdeccf443fe; 2018-06-17T18:33:14Z) |
| shellcheck | v0.4.4 |
| shellcheck |
https://builds.apache.org/job/PreCommit-HBASE-Build/553/artifact/patchprocess/diff-patch-shellcheck.txt
|
| Max. process+thread count | 41 (vs. ulimit of 1) |
| modules | C: . U: . |
| Console output |
https://builds.apache.org/job/PreCommit-HBASE-Build/553/console |
| Powered by | Apache Yetus 0.9.0 http://yetus.apache.org |
This message was automatically generated.
> Improvements to hbase-vote script
> -
>
> Key: HBASE-22464
> URL: https://issues.apache.org/jira/browse/HBASE-22464
> Project: HBase
> Issue Type: Improvement
>Affects Versions: 3.0.0, 1.5.0, 2.2.0, 1.4.9, 1.3.4, 2.3.0, 2.1.4
>Reporter: Artem Ervits
>Assignee: Artem Ervits
>Priority: Trivial
> Attachments: HBASE-22464.branch-1.patch, HBASE-22464.v01.patch,
> HBASE-22464.v02.patch, HBASE-22464.v03.patch
>
>
> This Jira proposes to add the following two improvements:
> # wget -N option for checking timestamp of downloaded RC files, if the local
> copy of downloaded file is identical on the server, that file will be ignored.
> # mvn -fn to never fail running the tests, previously with option -fae I
> could not get the whole suite to run w/out failing.
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
[jira] [Commented] (HBASE-22464) Improvements to hbase-vote script
[ https://issues.apache.org/jira/browse/HBASE-22464?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16867027#comment-16867027 ] Artem Ervits commented on HBASE-22464: -- [~busbey] addressing your comments in .v03 and provided a new option _-P _ to opt-in into running larger scoped tests. Please review. > Improvements to hbase-vote script > - > > Key: HBASE-22464 > URL: https://issues.apache.org/jira/browse/HBASE-22464 > Project: HBase > Issue Type: Improvement >Affects Versions: 3.0.0, 1.5.0, 2.2.0, 1.4.9, 1.3.4, 2.3.0, 2.1.4 >Reporter: Artem Ervits >Assignee: Artem Ervits >Priority: Trivial > Attachments: HBASE-22464.branch-1.patch, HBASE-22464.v01.patch, > HBASE-22464.v02.patch, HBASE-22464.v03.patch > > > This Jira proposes to add the following two improvements: > # wget -N option for checking timestamp of downloaded RC files, if the local > copy of downloaded file is identical on the server, that file will be ignored. > # mvn -fn to never fail running the tests, previously with option -fae I > could not get the whole suite to run w/out failing. -- This message was sent by Atlassian JIRA (v7.6.3#76005)
[jira] [Updated] (HBASE-22464) Improvements to hbase-vote script
[ https://issues.apache.org/jira/browse/HBASE-22464?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Artem Ervits updated HBASE-22464: - Attachment: HBASE-22464.v03.patch > Improvements to hbase-vote script > - > > Key: HBASE-22464 > URL: https://issues.apache.org/jira/browse/HBASE-22464 > Project: HBase > Issue Type: Improvement >Affects Versions: 3.0.0, 1.5.0, 2.2.0, 1.4.9, 1.3.4, 2.3.0, 2.1.4 >Reporter: Artem Ervits >Assignee: Artem Ervits >Priority: Trivial > Attachments: HBASE-22464.branch-1.patch, HBASE-22464.v01.patch, > HBASE-22464.v02.patch, HBASE-22464.v03.patch > > > This Jira proposes to add the following two improvements: > # wget -N option for checking timestamp of downloaded RC files, if the local > copy of downloaded file is identical on the server, that file will be ignored. > # mvn -fn to never fail running the tests, previously with option -fae I > could not get the whole suite to run w/out failing. -- This message was sent by Atlassian JIRA (v7.6.3#76005)
[GitHub] [hbase] Apache-HBase commented on issue #301: HBASE-22547 Document for offheap read in HBase Book
Apache-HBase commented on issue #301: HBASE-22547 Document for offheap read in HBase Book URL: https://github.com/apache/hbase/pull/301#issuecomment-503295850 :broken_heart: **-1 overall** | Vote | Subsystem | Runtime | Comment | |::|--:|:|:| | 0 | reexec | 44 | Docker mode activated. | ||| _ Prechecks _ | | +1 | hbaseanti | 0 | Patch does not have any anti-patterns. | | +1 | @author | 0 | The patch does not contain any @author tags. | | +1 | test4tests | 0 | The patch appears to include 2 new or modified test files. | ||| _ HBASE-21879 Compile Tests _ | | 0 | mvndep | 13 | Maven dependency ordering for branch | | +1 | mvninstall | 256 | HBASE-21879 passed | | +1 | compile | 171 | HBASE-21879 passed | | +1 | checkstyle | 144 | HBASE-21879 passed | | +1 | mvnsite | 1152 | HBASE-21879 passed | | 0 | refguide | 483 | branch has no errors when building the reference guide. See footer for rendered docs, which you should manually inspect. | | +1 | shadedjars | 273 | branch has no errors when building our shaded downstream artifacts. | | 0 | findbugs | 0 | Skipped patched modules with no Java source: . | | +1 | findbugs | 245 | HBASE-21879 passed | | +1 | javadoc | 206 | HBASE-21879 passed | ||| _ Patch Compile Tests _ | | 0 | mvndep | 11 | Maven dependency ordering for patch | | +1 | mvninstall | 249 | the patch passed | | +1 | compile | 172 | the patch passed | | +1 | javac | 172 | the patch passed | | -1 | checkstyle | 151 | root: The patch generated 2 new + 77 unchanged - 0 fixed = 79 total (was 77) | | +1 | mvnsite | 1160 | the patch passed | | +1 | whitespace | 0 | The patch has no whitespace issues. | | 0 | refguide | 487 | patch has no errors when building the reference guide. See footer for rendered docs, which you should manually inspect. | | +1 | shadedjars | 278 | patch has no errors when building our shaded downstream artifacts. | | +1 | hadoopcheck | 778 | Patch does not cause any errors with Hadoop 2.8.5 2.9.2 or 3.1.2. | | 0 | findbugs | 0 | Skipped patched modules with no Java source: . | | +1 | findbugs | 263 | the patch passed | | +1 | javadoc | 208 | the patch passed | ||| _ Other Tests _ | | -1 | unit | 17356 | root in the patch failed. | | +1 | asflicense | 101 | The patch does not generate ASF License warnings. | | | | 24519 | | | Reason | Tests | |---:|:--| | Failed junit tests | hadoop.hbase.client.TestTableShell | | | hadoop.hbase.client.TestAdminShell | | Subsystem | Report/Notes | |--:|:-| | Docker | Client=17.05.0-ce Server=17.05.0-ce base: https://builds.apache.org/job/HBase-PreCommit-GitHub-PR/job/PR-301/6/artifact/out/Dockerfile | | GITHUB PR | https://github.com/apache/hbase/pull/301 | | Optional Tests | dupname asflicense javac javadoc unit findbugs shadedjars hadoopcheck hbaseanti checkstyle compile refguide mvnsite | | uname | Linux ab68cbc68874 4.4.0-141-generic #167~14.04.1-Ubuntu SMP Mon Dec 10 13:20:24 UTC 2018 x86_64 GNU/Linux | | Build tool | maven | | Personality | /testptch/patchprocess/precommit/personality/provided.sh | | git revision | HBASE-21879 / afaf7a9e35 | | maven | version: Apache Maven 3.5.4 (1edded0938998edf8bf061f1ceb3cfdeccf443fe; 2018-06-17T18:33:14Z) | | Default Java | 1.8.0_181 | | refguide | https://builds.apache.org/job/HBase-PreCommit-GitHub-PR/job/PR-301/6/artifact/out/branch-site/book.html | | findbugs | v3.1.11 | | checkstyle | https://builds.apache.org/job/HBase-PreCommit-GitHub-PR/job/PR-301/6/artifact/out/diff-checkstyle-root.txt | | refguide | https://builds.apache.org/job/HBase-PreCommit-GitHub-PR/job/PR-301/6/artifact/out/patch-site/book.html | | unit | https://builds.apache.org/job/HBase-PreCommit-GitHub-PR/job/PR-301/6/artifact/out/patch-unit-root.txt | | Test Results | https://builds.apache.org/job/HBase-PreCommit-GitHub-PR/job/PR-301/6/testReport/ | | Max. process+thread count | 5040 (vs. ulimit of 1) | | modules | C: hbase-common hbase-server . U: . | | Console output | https://builds.apache.org/job/HBase-PreCommit-GitHub-PR/job/PR-301/6/console | | Powered by | Apache Yetus 0.9.0 http://yetus.apache.org | This message was automatically generated. This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
[jira] [Commented] (HBASE-22561) modify HFilePrettyPrinter to accept non-hbase.rootdir directories
[
https://issues.apache.org/jira/browse/HBASE-22561?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16867010#comment-16867010
]
HBase QA commented on HBASE-22561:
--
| (x) *{color:red}-1 overall{color}* |
\\
\\
|| Vote || Subsystem || Runtime || Comment ||
| {color:blue}0{color} | {color:blue} reexec {color} | {color:blue} 2m
13s{color} | {color:blue} Docker mode activated. {color} |
|| || || || {color:brown} Prechecks {color} ||
| {color:green}+1{color} | {color:green} hbaseanti {color} | {color:green} 0m
0s{color} | {color:green} Patch does not have any anti-patterns. {color} |
| {color:green}+1{color} | {color:green} @author {color} | {color:green} 0m
0s{color} | {color:green} The patch does not contain any @author tags. {color} |
| {color:green}+1{color} | {color:green} test4tests {color} | {color:green} 0m
0s{color} | {color:green} The patch appears to include 1 new or modified test
files. {color} |
|| || || || {color:brown} master Compile Tests {color} ||
| {color:green}+1{color} | {color:green} mvninstall {color} | {color:green} 4m
33s{color} | {color:green} master passed {color} |
| {color:green}+1{color} | {color:green} compile {color} | {color:green} 0m
50s{color} | {color:green} master passed {color} |
| {color:green}+1{color} | {color:green} checkstyle {color} | {color:green} 1m
17s{color} | {color:green} master passed {color} |
| {color:green}+1{color} | {color:green} shadedjars {color} | {color:green} 4m
35s{color} | {color:green} branch has no errors when building our shaded
downstream artifacts. {color} |
| {color:green}+1{color} | {color:green} findbugs {color} | {color:green} 3m
23s{color} | {color:green} master passed {color} |
| {color:green}+1{color} | {color:green} javadoc {color} | {color:green} 0m
32s{color} | {color:green} master passed {color} |
|| || || || {color:brown} Patch Compile Tests {color} ||
| {color:green}+1{color} | {color:green} mvninstall {color} | {color:green} 4m
15s{color} | {color:green} the patch passed {color} |
| {color:green}+1{color} | {color:green} compile {color} | {color:green} 0m
51s{color} | {color:green} the patch passed {color} |
| {color:green}+1{color} | {color:green} javac {color} | {color:green} 0m
51s{color} | {color:green} the patch passed {color} |
| {color:red}-1{color} | {color:red} checkstyle {color} | {color:red} 1m
14s{color} | {color:red} hbase-server: The patch generated 1 new + 13 unchanged
- 1 fixed = 14 total (was 14) {color} |
| {color:green}+1{color} | {color:green} whitespace {color} | {color:green} 0m
0s{color} | {color:green} The patch has no whitespace issues. {color} |
| {color:green}+1{color} | {color:green} shadedjars {color} | {color:green} 4m
37s{color} | {color:green} patch has no errors when building our shaded
downstream artifacts. {color} |
| {color:green}+1{color} | {color:green} hadoopcheck {color} | {color:green}
13m 2s{color} | {color:green} Patch does not cause any errors with Hadoop
2.8.5 2.9.2 or 3.1.2. {color} |
| {color:green}+1{color} | {color:green} findbugs {color} | {color:green} 3m
38s{color} | {color:green} the patch passed {color} |
| {color:green}+1{color} | {color:green} javadoc {color} | {color:green} 0m
30s{color} | {color:green} the patch passed {color} |
|| || || || {color:brown} Other Tests {color} ||
| {color:green}+1{color} | {color:green} unit {color} | {color:green}215m
45s{color} | {color:green} hbase-server in the patch passed. {color} |
| {color:green}+1{color} | {color:green} asflicense {color} | {color:green} 0m
24s{color} | {color:green} The patch does not generate ASF License warnings.
{color} |
| {color:black}{color} | {color:black} {color} | {color:black}266m 23s{color} |
{color:black} {color} |
\\
\\
|| Subsystem || Report/Notes ||
| Docker | Client=17.05.0-ce Server=17.05.0-ce base:
https://builds.apache.org/job/PreCommit-HBASE-Build/552/artifact/patchprocess/Dockerfile
|
| JIRA Issue | HBASE-22561 |
| JIRA Patch URL |
https://issues.apache.org/jira/secure/attachment/12972111/HBASE-22561.v12.patch
|
| Optional Tests | dupname asflicense javac javadoc unit findbugs
shadedjars hadoopcheck hbaseanti checkstyle compile |
| uname | Linux 77abaab6a97c 4.4.0-143-generic #169~14.04.2-Ubuntu SMP Wed Feb
13 15:00:41 UTC 2019 x86_64 GNU/Linux |
| Build tool | maven |
| Personality | dev-support/hbase-personality.sh |
| git revision | master / ac3d09e7fd |
| maven | version: Apache Maven 3.5.4
(1edded0938998edf8bf061f1ceb3cfdeccf443fe; 2018-06-17T18:33:14Z) |
| Default Java | 1.8.0_181 |
| findbugs | v3.1.11 |
| checkstyle |
https://builds.apache.org/job/PreCommit-HBASE-Build/552/artifact/patchprocess/diff-checkstyle-hbase-server.txt
|
| Test Results |
https://builds.apache.org/job/PreCommit-HBASE-Build/552/testReport/ |
| Max. process+thread count | 4917 (vs. ulimit of 1) |
| modules | C: hbase-server U: hbase-server |
| Console output |
[jira] [Commented] (HBASE-22553) NPE in RegionReplicaReplicationEndpoint
[
https://issues.apache.org/jira/browse/HBASE-22553?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16866963#comment-16866963
]
Hudson commented on HBASE-22553:
Results for branch master
[build #1154 on
builds.a.o|https://builds.apache.org/job/HBase%20Nightly/job/master/1154/]: (x)
*{color:red}-1 overall{color}*
details (if available):
(/) {color:green}+1 general checks{color}
-- For more information [see general
report|https://builds.apache.org/job/HBase%20Nightly/job/master/1154//General_Nightly_Build_Report/]
(x) {color:red}-1 jdk8 hadoop2 checks{color}
-- For more information [see jdk8 (hadoop2)
report|https://builds.apache.org/job/HBase%20Nightly/job/master/1154//JDK8_Nightly_Build_Report_(Hadoop2)/]
(/) {color:green}+1 jdk8 hadoop3 checks{color}
-- For more information [see jdk8 (hadoop3)
report|https://builds.apache.org/job/HBase%20Nightly/job/master/1154//JDK8_Nightly_Build_Report_(Hadoop3)/]
(/) {color:green}+1 source release artifact{color}
-- See build output for details.
(/) {color:green}+1 client integration test{color}
> NPE in RegionReplicaReplicationEndpoint
> ---
>
> Key: HBASE-22553
> URL: https://issues.apache.org/jira/browse/HBASE-22553
> Project: HBase
> Issue Type: Sub-task
> Components: read replicas, Replication
>Reporter: Duo Zhang
>Assignee: Duo Zhang
>Priority: Major
> Fix For: 3.0.0
>
>
> https://builds.apache.org/job/HBase-Flaky-Tests/job/HBASE-21512/1197/artifact/hbase-server/target/surefire-reports/org.apache.hadoop.hbase.replication.regionserver.TestRegionReplicaReplicationEndpoint-output.txt/*view*/
> {noformat}
> 2019-06-07 20:54:10,610 ERROR [RS-EventLoopGroup-4-17] util.FutureUtils(70):
> Unexpected error caught when processing CompletableFuture
> java.lang.NullPointerException
> at
> org.apache.hadoop.hbase.replication.regionserver.RegionReplicaReplicationEndpoint.replicate(RegionReplicaReplicationEndpoint.java:207)
> at
> org.apache.hadoop.hbase.replication.regionserver.RegionReplicaReplicationEndpoint.lambda$replicate$2(RegionReplicaReplicationEndpoint.java:249)
> at
> org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:68)
> at
> java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:760)
> at
> java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:736)
> at
> java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:474)
> at
> java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:1962)
> at
> org.apache.hadoop.hbase.replication.regionserver.RegionReplicaReplicationEndpoint.lambda$getRegionLocations$0(RegionReplicaReplicationEndpoint.java:161)
> at
> org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:68)
> at
> java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:760)
> at
> java.util.concurrent.CompletableFuture.uniWhenCompleteStage(CompletableFuture.java:778)
> at
> java.util.concurrent.CompletableFuture.whenComplete(CompletableFuture.java:2140)
> at
> org.apache.hadoop.hbase.util.FutureUtils.addListener(FutureUtils.java:61)
> at
> org.apache.hadoop.hbase.replication.regionserver.RegionReplicaReplicationEndpoint.getRegionLocations(RegionReplicaReplicationEndpoint.java:153)
> at
> org.apache.hadoop.hbase.replication.regionserver.RegionReplicaReplicationEndpoint.lambda$getRegionLocations$0(RegionReplicaReplicationEndpoint.java:173)
> at
> org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:68)
> at
> java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:760)
> at
> java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:736)
> at
> java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:474)
> at
> java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:1962)
> at
> org.apache.hadoop.hbase.client.AsyncNonMetaRegionLocator$TableCache.tryComplete(AsyncNonMetaRegionLocator.java:186)
> at
> org.apache.hadoop.hbase.client.AsyncNonMetaRegionLocator$TableCache.clearCompletedRequests(AsyncNonMetaRegionLocator.java:152)
> at
> org.apache.hadoop.hbase.client.AsyncNonMetaRegionLocator.complete(AsyncNonMetaRegionLocator.java:289)
> at
> org.apache.hadoop.hbase.client.AsyncNonMetaRegionLocator.onScanNext(AsyncNonMetaRegionLocator.java:348)
> at
> org.apache.hadoop.hbase.client.AsyncNonMetaRegionLocator.access$600(AsyncNonMetaRegionLocator.java:71)
> at
> org.apache.hadoop.hbase.client.AsyncNonMetaRegionLocator$1.onNext(AsyncNonMetaRegionLocator.java:469)
> at
>
[jira] [Commented] (HBASE-22223) Implement RegionLocator based on AsyncTableRegionLocator
[
https://issues.apache.org/jira/browse/HBASE-3?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16866950#comment-16866950
]
Hudson commented on HBASE-3:
Results for branch master
[build #1154 on
builds.a.o|https://builds.apache.org/job/HBase%20Nightly/job/master/1154/]: (x)
*{color:red}-1 overall{color}*
details (if available):
(/) {color:green}+1 general checks{color}
-- For more information [see general
report|https://builds.apache.org/job/HBase%20Nightly/job/master/1154//General_Nightly_Build_Report/]
(x) {color:red}-1 jdk8 hadoop2 checks{color}
-- For more information [see jdk8 (hadoop2)
report|https://builds.apache.org/job/HBase%20Nightly/job/master/1154//JDK8_Nightly_Build_Report_(Hadoop2)/]
(/) {color:green}+1 jdk8 hadoop3 checks{color}
-- For more information [see jdk8 (hadoop3)
report|https://builds.apache.org/job/HBase%20Nightly/job/master/1154//JDK8_Nightly_Build_Report_(Hadoop3)/]
(/) {color:green}+1 source release artifact{color}
-- See build output for details.
(/) {color:green}+1 client integration test{color}
> Implement RegionLocator based on AsyncTableRegionLocator
>
>
> Key: HBASE-3
> URL: https://issues.apache.org/jira/browse/HBASE-3
> Project: HBase
> Issue Type: Sub-task
>Reporter: Duo Zhang
>Assignee: Duo Zhang
>Priority: Major
> Fix For: 3.0.0
>
>
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
[jira] [Commented] (HBASE-22351) Fix creating table timeout for TestProcedurePriority
[
https://issues.apache.org/jira/browse/HBASE-22351?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16866960#comment-16866960
]
Hudson commented on HBASE-22351:
Results for branch master
[build #1154 on
builds.a.o|https://builds.apache.org/job/HBase%20Nightly/job/master/1154/]: (x)
*{color:red}-1 overall{color}*
details (if available):
(/) {color:green}+1 general checks{color}
-- For more information [see general
report|https://builds.apache.org/job/HBase%20Nightly/job/master/1154//General_Nightly_Build_Report/]
(x) {color:red}-1 jdk8 hadoop2 checks{color}
-- For more information [see jdk8 (hadoop2)
report|https://builds.apache.org/job/HBase%20Nightly/job/master/1154//JDK8_Nightly_Build_Report_(Hadoop2)/]
(/) {color:green}+1 jdk8 hadoop3 checks{color}
-- For more information [see jdk8 (hadoop3)
report|https://builds.apache.org/job/HBase%20Nightly/job/master/1154//JDK8_Nightly_Build_Report_(Hadoop3)/]
(/) {color:green}+1 source release artifact{color}
-- See build output for details.
(/) {color:green}+1 client integration test{color}
> Fix creating table timeout for TestProcedurePriority
>
>
> Key: HBASE-22351
> URL: https://issues.apache.org/jira/browse/HBASE-22351
> Project: HBase
> Issue Type: Sub-task
>Reporter: Duo Zhang
>Assignee: Duo Zhang
>Priority: Major
> Fix For: 3.0.0
>
> Attachments: HBASE-22351-HBASE-21512-addendum.patch,
> HBASE-22351-HBASE-21512.patch
>
>
> It is easy to timeout in setUp.
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
[jira] [Commented] (HBASE-21515) Also initialize an AsyncClusterConnection in HRegionServer
[
https://issues.apache.org/jira/browse/HBASE-21515?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16866934#comment-16866934
]
Hudson commented on HBASE-21515:
Results for branch master
[build #1154 on
builds.a.o|https://builds.apache.org/job/HBase%20Nightly/job/master/1154/]: (x)
*{color:red}-1 overall{color}*
details (if available):
(/) {color:green}+1 general checks{color}
-- For more information [see general
report|https://builds.apache.org/job/HBase%20Nightly/job/master/1154//General_Nightly_Build_Report/]
(x) {color:red}-1 jdk8 hadoop2 checks{color}
-- For more information [see jdk8 (hadoop2)
report|https://builds.apache.org/job/HBase%20Nightly/job/master/1154//JDK8_Nightly_Build_Report_(Hadoop2)/]
(/) {color:green}+1 jdk8 hadoop3 checks{color}
-- For more information [see jdk8 (hadoop3)
report|https://builds.apache.org/job/HBase%20Nightly/job/master/1154//JDK8_Nightly_Build_Report_(Hadoop3)/]
(/) {color:green}+1 source release artifact{color}
-- See build output for details.
(/) {color:green}+1 client integration test{color}
> Also initialize an AsyncClusterConnection in HRegionServer
> --
>
> Key: HBASE-21515
> URL: https://issues.apache.org/jira/browse/HBASE-21515
> Project: HBase
> Issue Type: Sub-task
> Components: regionserver
>Reporter: Duo Zhang
>Assignee: Duo Zhang
>Priority: Major
> Fix For: 3.0.0
>
> Attachments: HBASE-21515-v1.patch, HBASE-21515-v2.patch,
> HBASE-21515-v3.patch, HBASE-21515-v4.patch, HBASE-21515-v4.patch,
> HBASE-21515.patch
>
>
> Plan to do this incrementally, so first we will only introduce the new
> AsyncClusterConnection, without deleting the old ClusterConnection. And we
> can move the code which uses the ClusterConnection to use
> AsyncClusterConnection, through different sub tasks here. And finally we can
> remove the ClusterConnection completely.
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
[jira] [Commented] (HBASE-21717) Implement Connection based on AsyncConnection
[
https://issues.apache.org/jira/browse/HBASE-21717?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16866945#comment-16866945
]
Hudson commented on HBASE-21717:
Results for branch master
[build #1154 on
builds.a.o|https://builds.apache.org/job/HBase%20Nightly/job/master/1154/]: (x)
*{color:red}-1 overall{color}*
details (if available):
(/) {color:green}+1 general checks{color}
-- For more information [see general
report|https://builds.apache.org/job/HBase%20Nightly/job/master/1154//General_Nightly_Build_Report/]
(x) {color:red}-1 jdk8 hadoop2 checks{color}
-- For more information [see jdk8 (hadoop2)
report|https://builds.apache.org/job/HBase%20Nightly/job/master/1154//JDK8_Nightly_Build_Report_(Hadoop2)/]
(/) {color:green}+1 jdk8 hadoop3 checks{color}
-- For more information [see jdk8 (hadoop3)
report|https://builds.apache.org/job/HBase%20Nightly/job/master/1154//JDK8_Nightly_Build_Report_(Hadoop3)/]
(/) {color:green}+1 source release artifact{color}
-- See build output for details.
(/) {color:green}+1 client integration test{color}
> Implement Connection based on AsyncConnection
> -
>
> Key: HBASE-21717
> URL: https://issues.apache.org/jira/browse/HBASE-21717
> Project: HBase
> Issue Type: Sub-task
> Components: asyncclient, Client
>Reporter: Duo Zhang
>Assignee: Duo Zhang
>Priority: Major
> Fix For: 3.0.0
>
> Attachments: HBASE-21717-HBASE-21512-v1.patch,
> HBASE-21717-HBASE-21512-v10.patch, HBASE-21717-HBASE-21512-v2.patch,
> HBASE-21717-HBASE-21512-v3.patch, HBASE-21717-HBASE-21512-v4.patch,
> HBASE-21717-HBASE-21512-v5.patch, HBASE-21717-HBASE-21512-v6.patch,
> HBASE-21717-HBASE-21512-v7.patch, HBASE-21717-HBASE-21512-v8.patch,
> HBASE-21717-HBASE-21512-v9.patch, HBASE-21717-HBASE-21512-v9.patch,
> HBASE-21717-HBASE-21512-v9.patch, HBASE-21717-HBASE-21512-v9.patch,
> HBASE-21717-HBASE-21512.patch
>
>
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
[jira] [Commented] (HBASE-21778) Remove the usage of the locateRegion related methods in ClusterConnection
[
https://issues.apache.org/jira/browse/HBASE-21778?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16866942#comment-16866942
]
Hudson commented on HBASE-21778:
Results for branch master
[build #1154 on
builds.a.o|https://builds.apache.org/job/HBase%20Nightly/job/master/1154/]: (x)
*{color:red}-1 overall{color}*
details (if available):
(/) {color:green}+1 general checks{color}
-- For more information [see general
report|https://builds.apache.org/job/HBase%20Nightly/job/master/1154//General_Nightly_Build_Report/]
(x) {color:red}-1 jdk8 hadoop2 checks{color}
-- For more information [see jdk8 (hadoop2)
report|https://builds.apache.org/job/HBase%20Nightly/job/master/1154//JDK8_Nightly_Build_Report_(Hadoop2)/]
(/) {color:green}+1 jdk8 hadoop3 checks{color}
-- For more information [see jdk8 (hadoop3)
report|https://builds.apache.org/job/HBase%20Nightly/job/master/1154//JDK8_Nightly_Build_Report_(Hadoop3)/]
(/) {color:green}+1 source release artifact{color}
-- See build output for details.
(/) {color:green}+1 client integration test{color}
> Remove the usage of the locateRegion related methods in ClusterConnection
> -
>
> Key: HBASE-21778
> URL: https://issues.apache.org/jira/browse/HBASE-21778
> Project: HBase
> Issue Type: Sub-task
> Components: Client
>Reporter: Duo Zhang
>Assignee: Duo Zhang
>Priority: Major
> Fix For: 3.0.0
>
> Attachments: HBASE-21778-HBASE-21512-v1.patch,
> HBASE-21778-HBASE-21512-v2.patch, HBASE-21778-HBASE-21512-v3.patch,
> HBASE-21778-HBASE-21512.patch
>
>
> As now RegionLocator can give you everything. So later we could remove the
> ClusterConnection completely.
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
[jira] [Commented] (HBASE-22302) Fix TestHbck
[
https://issues.apache.org/jira/browse/HBASE-22302?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16866954#comment-16866954
]
Hudson commented on HBASE-22302:
Results for branch master
[build #1154 on
builds.a.o|https://builds.apache.org/job/HBase%20Nightly/job/master/1154/]: (x)
*{color:red}-1 overall{color}*
details (if available):
(/) {color:green}+1 general checks{color}
-- For more information [see general
report|https://builds.apache.org/job/HBase%20Nightly/job/master/1154//General_Nightly_Build_Report/]
(x) {color:red}-1 jdk8 hadoop2 checks{color}
-- For more information [see jdk8 (hadoop2)
report|https://builds.apache.org/job/HBase%20Nightly/job/master/1154//JDK8_Nightly_Build_Report_(Hadoop2)/]
(/) {color:green}+1 jdk8 hadoop3 checks{color}
-- For more information [see jdk8 (hadoop3)
report|https://builds.apache.org/job/HBase%20Nightly/job/master/1154//JDK8_Nightly_Build_Report_(Hadoop3)/]
(/) {color:green}+1 source release artifact{color}
-- See build output for details.
(/) {color:green}+1 client integration test{color}
> Fix TestHbck
>
>
> Key: HBASE-22302
> URL: https://issues.apache.org/jira/browse/HBASE-22302
> Project: HBase
> Issue Type: Sub-task
> Components: test
>Reporter: Duo Zhang
>Assignee: Duo Zhang
>Priority: Major
> Fix For: 3.0.0
>
> Attachments: HBASE-22302-HBASE-21512.patch
>
>
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
[jira] [Commented] (HBASE-22295) Fix TestClientOperationTimeout
[
https://issues.apache.org/jira/browse/HBASE-22295?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16866952#comment-16866952
]
Hudson commented on HBASE-22295:
Results for branch master
[build #1154 on
builds.a.o|https://builds.apache.org/job/HBase%20Nightly/job/master/1154/]: (x)
*{color:red}-1 overall{color}*
details (if available):
(/) {color:green}+1 general checks{color}
-- For more information [see general
report|https://builds.apache.org/job/HBase%20Nightly/job/master/1154//General_Nightly_Build_Report/]
(x) {color:red}-1 jdk8 hadoop2 checks{color}
-- For more information [see jdk8 (hadoop2)
report|https://builds.apache.org/job/HBase%20Nightly/job/master/1154//JDK8_Nightly_Build_Report_(Hadoop2)/]
(/) {color:green}+1 jdk8 hadoop3 checks{color}
-- For more information [see jdk8 (hadoop3)
report|https://builds.apache.org/job/HBase%20Nightly/job/master/1154//JDK8_Nightly_Build_Report_(Hadoop3)/]
(/) {color:green}+1 source release artifact{color}
-- See build output for details.
(/) {color:green}+1 client integration test{color}
> Fix TestClientOperationTimeout
> --
>
> Key: HBASE-22295
> URL: https://issues.apache.org/jira/browse/HBASE-22295
> Project: HBase
> Issue Type: Sub-task
> Components: test
>Reporter: Duo Zhang
>Assignee: Duo Zhang
>Priority: Major
> Fix For: 3.0.0
>
> Attachments: HBASE-22295-HBASE-21512-addendum.patch,
> HBASE-22295-HBASE-21512.patch
>
>
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
[jira] [Commented] (HBASE-22037) Re-enable TestAvoidCellReferencesIntoShippedBlocks
[
https://issues.apache.org/jira/browse/HBASE-22037?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16866958#comment-16866958
]
Hudson commented on HBASE-22037:
Results for branch master
[build #1154 on
builds.a.o|https://builds.apache.org/job/HBase%20Nightly/job/master/1154/]: (x)
*{color:red}-1 overall{color}*
details (if available):
(/) {color:green}+1 general checks{color}
-- For more information [see general
report|https://builds.apache.org/job/HBase%20Nightly/job/master/1154//General_Nightly_Build_Report/]
(x) {color:red}-1 jdk8 hadoop2 checks{color}
-- For more information [see jdk8 (hadoop2)
report|https://builds.apache.org/job/HBase%20Nightly/job/master/1154//JDK8_Nightly_Build_Report_(Hadoop2)/]
(/) {color:green}+1 jdk8 hadoop3 checks{color}
-- For more information [see jdk8 (hadoop3)
report|https://builds.apache.org/job/HBase%20Nightly/job/master/1154//JDK8_Nightly_Build_Report_(Hadoop3)/]
(/) {color:green}+1 source release artifact{color}
-- See build output for details.
(/) {color:green}+1 client integration test{color}
> Re-enable TestAvoidCellReferencesIntoShippedBlocks
> --
>
> Key: HBASE-22037
> URL: https://issues.apache.org/jira/browse/HBASE-22037
> Project: HBase
> Issue Type: Sub-task
>Reporter: Duo Zhang
>Assignee: Duo Zhang
>Priority: Major
> Fix For: 3.0.0
>
>
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
[jira] [Commented] (HBASE-22550) Throw exception when creating thread pool if the connection has already been closed
[
https://issues.apache.org/jira/browse/HBASE-22550?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16866962#comment-16866962
]
Hudson commented on HBASE-22550:
Results for branch master
[build #1154 on
builds.a.o|https://builds.apache.org/job/HBase%20Nightly/job/master/1154/]: (x)
*{color:red}-1 overall{color}*
details (if available):
(/) {color:green}+1 general checks{color}
-- For more information [see general
report|https://builds.apache.org/job/HBase%20Nightly/job/master/1154//General_Nightly_Build_Report/]
(x) {color:red}-1 jdk8 hadoop2 checks{color}
-- For more information [see jdk8 (hadoop2)
report|https://builds.apache.org/job/HBase%20Nightly/job/master/1154//JDK8_Nightly_Build_Report_(Hadoop2)/]
(/) {color:green}+1 jdk8 hadoop3 checks{color}
-- For more information [see jdk8 (hadoop3)
report|https://builds.apache.org/job/HBase%20Nightly/job/master/1154//JDK8_Nightly_Build_Report_(Hadoop3)/]
(/) {color:green}+1 source release artifact{color}
-- See build output for details.
(/) {color:green}+1 client integration test{color}
> Throw exception when creating thread pool if the connection has already been
> closed
> ---
>
> Key: HBASE-22550
> URL: https://issues.apache.org/jira/browse/HBASE-22550
> Project: HBase
> Issue Type: Sub-task
> Components: Client
>Reporter: Duo Zhang
>Assignee: Duo Zhang
>Priority: Major
> Fix For: 3.0.0
>
> Attachments: HBASE-22550-HBASE-21512.patch
>
>
> It will inherit the ThreadGroup of the current thread, which is not necessary
> here but could introduce problems if the thread which creates the thread pool
> is dead. See this:
> https://builds.apache.org/job/HBase-Flaky-Tests/job/HBASE-21512/1191/artifact/hbase-server/target/surefire-reports/org.apache.hadoop.hbase.master.procedure.TestTruncateTableProcedure-output.txt/*view*/
> {noformat}
> 2019-06-07 00:58:47,175 ERROR [PEWorker-1]
> procedure2.ProcedureExecutor(1667): CODE-BUG: Uncaught runtime exception:
> pid=307, state=RUNNABLE:SPLIT_TABLE_REGION_UPDATE_META, hasLock=true;
> SplitTableRegionProcedure table=testTruncateWithPreserveAfterSplit,
> parent=bc630be809a91baf71d0a05051bd502b,
> daughterA=ce5cf8608cf97afcd09a33545d4169ab,
> daughterB=ec4d88471626f18b811bb6910f33a067
> java.lang.IllegalThreadStateException
> at java.lang.ThreadGroup.addUnstarted(ThreadGroup.java:867)
> at java.lang.Thread.init(Thread.java:405)
> at java.lang.Thread.init(Thread.java:349)
> at java.lang.Thread.(Thread.java:599)
> at org.apache.hadoop.hbase.util.Threads$2.newThread(Threads.java:220)
> at org.apache.hadoop.hbase.util.Threads$3.newThread(Threads.java:246)
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.(ThreadPoolExecutor.java:619)
> at
> java.util.concurrent.ThreadPoolExecutor.addWorker(ThreadPoolExecutor.java:932)
> at
> java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1367)
> at
> java.util.concurrent.AbstractExecutorService.submit(AbstractExecutorService.java:134)
> at
> org.apache.hadoop.hbase.client.TableOverAsyncTable.coprocssorService(TableOverAsyncTable.java:431)
> at
> org.apache.hadoop.hbase.client.TableOverAsyncTable.coprocessorService(TableOverAsyncTable.java:469)
> at
> org.apache.hadoop.hbase.client.Table.coprocessorService(Table.java:527)
> at
> org.apache.hadoop.hbase.MetaTableAccessor.multiMutate(MetaTableAccessor.java:1768)
> at
> org.apache.hadoop.hbase.MetaTableAccessor.multiMutate(MetaTableAccessor.java:1729)
> at
> org.apache.hadoop.hbase.MetaTableAccessor.splitRegion(MetaTableAccessor.java:1688)
> at
> org.apache.hadoop.hbase.master.assignment.RegionStateStore.splitRegion(RegionStateStore.java:237)
> at
> org.apache.hadoop.hbase.master.assignment.AssignmentManager.markRegionAsSplit(AssignmentManager.java:1643)
> at
> org.apache.hadoop.hbase.master.assignment.SplitTableRegionProcedure.updateMeta(SplitTableRegionProcedure.java:824)
> at
> org.apache.hadoop.hbase.master.assignment.SplitTableRegionProcedure.executeFromState(SplitTableRegionProcedure.java:304)
> at
> org.apache.hadoop.hbase.master.assignment.SplitTableRegionProcedure.executeFromState(SplitTableRegionProcedure.java:91)
> at
> org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:194)
> at
> org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:959)
> at
> org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1648)
> at
> org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395)
> at
>
[jira] [Commented] (HBASE-22328) NPE in RegionReplicaReplicationEndpoint
[
https://issues.apache.org/jira/browse/HBASE-22328?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16866957#comment-16866957
]
Hudson commented on HBASE-22328:
Results for branch master
[build #1154 on
builds.a.o|https://builds.apache.org/job/HBase%20Nightly/job/master/1154/]: (x)
*{color:red}-1 overall{color}*
details (if available):
(/) {color:green}+1 general checks{color}
-- For more information [see general
report|https://builds.apache.org/job/HBase%20Nightly/job/master/1154//General_Nightly_Build_Report/]
(x) {color:red}-1 jdk8 hadoop2 checks{color}
-- For more information [see jdk8 (hadoop2)
report|https://builds.apache.org/job/HBase%20Nightly/job/master/1154//JDK8_Nightly_Build_Report_(Hadoop2)/]
(/) {color:green}+1 jdk8 hadoop3 checks{color}
-- For more information [see jdk8 (hadoop3)
report|https://builds.apache.org/job/HBase%20Nightly/job/master/1154//JDK8_Nightly_Build_Report_(Hadoop3)/]
(/) {color:green}+1 source release artifact{color}
-- See build output for details.
(/) {color:green}+1 client integration test{color}
> NPE in RegionReplicaReplicationEndpoint
> ---
>
> Key: HBASE-22328
> URL: https://issues.apache.org/jira/browse/HBASE-22328
> Project: HBase
> Issue Type: Sub-task
> Components: Replication
>Reporter: Duo Zhang
>Assignee: Duo Zhang
>Priority: Major
> Labels: replication
> Fix For: 3.0.0
>
> Attachments: HBASE-22328-HBASE-21512.patch
>
>
> {noformat}
> 2019-04-29 22:29:01,414 ERROR [Time-limited test] util.FutureUtils(72):
> Unexpected error caught when processing CompletableFuture
> java.lang.NullPointerException
> at
> org.apache.hadoop.hbase.replication.regionserver.RegionReplicaReplicationEndpoint.replicate(RegionReplicaReplicationEndpoint.java:205)
> at
> org.apache.hadoop.hbase.replication.regionserver.RegionReplicaReplicationEndpoint.lambda$2(RegionReplicaReplicationEndpoint.java:247)
> at
> org.apache.hadoop.hbase.util.FutureUtils.lambda$0(FutureUtils.java:70)
> at
> java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:760)
> at
> java.util.concurrent.CompletableFuture.uniWhenCompleteStage(CompletableFuture.java:778)
> at
> java.util.concurrent.CompletableFuture.whenComplete(CompletableFuture.java:2140)
> at
> org.apache.hadoop.hbase.util.FutureUtils.addListener(FutureUtils.java:63)
> at
> org.apache.hadoop.hbase.replication.regionserver.RegionReplicaReplicationEndpoint.replicate(RegionReplicaReplicationEndpoint.java:243)
> at
> org.apache.hadoop.hbase.replication.regionserver.RegionReplicaReplicationEndpoint.replicate(RegionReplicaReplicationEndpoint.java:301)
> at
> org.apache.hadoop.hbase.replication.regionserver.TestRegionReplicaReplicationEndpoint.testRegionReplicaReplicationIgnores(TestRegionReplicaReplicationEndpoint.java:449)
> at
> org.apache.hadoop.hbase.replication.regionserver.TestRegionReplicaReplicationEndpoint.testRegionReplicaReplicationIgnoresDisabledTables(TestRegionReplicaReplicationEndpoint.java:373)
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> at
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
> at
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> at java.lang.reflect.Method.invoke(Method.java:498)
> at
> org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:50)
> at
> org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
> at
> org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:47)
> at
> org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
> at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:55)
> at org.junit.rules.RunRules.evaluate(RunRules.java:20)
> at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:325)
> at
> org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:78)
> at
> org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:57)
> at org.junit.runners.ParentRunner$3.run(ParentRunner.java:290)
> at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:71)
> at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:288)
> at org.junit.runners.ParentRunner.access$000(ParentRunner.java:58)
> at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:268)
> at
> org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
> at
> org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
> at
>
[jira] [Commented] (HBASE-22297) Fix TestRegionMergeTransitionOnCluster and TestSplitTransactionOnCluster
[
https://issues.apache.org/jira/browse/HBASE-22297?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16866953#comment-16866953
]
Hudson commented on HBASE-22297:
Results for branch master
[build #1154 on
builds.a.o|https://builds.apache.org/job/HBase%20Nightly/job/master/1154/]: (x)
*{color:red}-1 overall{color}*
details (if available):
(/) {color:green}+1 general checks{color}
-- For more information [see general
report|https://builds.apache.org/job/HBase%20Nightly/job/master/1154//General_Nightly_Build_Report/]
(x) {color:red}-1 jdk8 hadoop2 checks{color}
-- For more information [see jdk8 (hadoop2)
report|https://builds.apache.org/job/HBase%20Nightly/job/master/1154//JDK8_Nightly_Build_Report_(Hadoop2)/]
(/) {color:green}+1 jdk8 hadoop3 checks{color}
-- For more information [see jdk8 (hadoop3)
report|https://builds.apache.org/job/HBase%20Nightly/job/master/1154//JDK8_Nightly_Build_Report_(Hadoop3)/]
(/) {color:green}+1 source release artifact{color}
-- See build output for details.
(/) {color:green}+1 client integration test{color}
> Fix TestRegionMergeTransitionOnCluster and TestSplitTransactionOnCluster
>
>
> Key: HBASE-22297
> URL: https://issues.apache.org/jira/browse/HBASE-22297
> Project: HBase
> Issue Type: Sub-task
> Components: test
>Reporter: Duo Zhang
>Assignee: Duo Zhang
>Priority: Major
> Fix For: 3.0.0
>
> Attachments: HBASE-22297-HBASE-21512-addendum.patch,
> HBASE-22297-HBASE-21512.patch
>
>
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
[jira] [Commented] (HBASE-21725) Implement BufferedMutator Based on AsyncBufferedMutator
[
https://issues.apache.org/jira/browse/HBASE-21725?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16866948#comment-16866948
]
Hudson commented on HBASE-21725:
Results for branch master
[build #1154 on
builds.a.o|https://builds.apache.org/job/HBase%20Nightly/job/master/1154/]: (x)
*{color:red}-1 overall{color}*
details (if available):
(/) {color:green}+1 general checks{color}
-- For more information [see general
report|https://builds.apache.org/job/HBase%20Nightly/job/master/1154//General_Nightly_Build_Report/]
(x) {color:red}-1 jdk8 hadoop2 checks{color}
-- For more information [see jdk8 (hadoop2)
report|https://builds.apache.org/job/HBase%20Nightly/job/master/1154//JDK8_Nightly_Build_Report_(Hadoop2)/]
(/) {color:green}+1 jdk8 hadoop3 checks{color}
-- For more information [see jdk8 (hadoop3)
report|https://builds.apache.org/job/HBase%20Nightly/job/master/1154//JDK8_Nightly_Build_Report_(Hadoop3)/]
(/) {color:green}+1 source release artifact{color}
-- See build output for details.
(/) {color:green}+1 client integration test{color}
> Implement BufferedMutator Based on AsyncBufferedMutator
> ---
>
> Key: HBASE-21725
> URL: https://issues.apache.org/jira/browse/HBASE-21725
> Project: HBase
> Issue Type: Sub-task
> Components: asyncclient
>Reporter: Duo Zhang
>Assignee: Duo Zhang
>Priority: Major
> Fix For: 3.0.0
>
>
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
[jira] [Commented] (HBASE-22577) BufferedMutatorOverAsyncBufferedMutator.tryCompleteFuture consume too much CPU time
[
https://issues.apache.org/jira/browse/HBASE-22577?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16866964#comment-16866964
]
Hudson commented on HBASE-22577:
Results for branch master
[build #1154 on
builds.a.o|https://builds.apache.org/job/HBase%20Nightly/job/master/1154/]: (x)
*{color:red}-1 overall{color}*
details (if available):
(/) {color:green}+1 general checks{color}
-- For more information [see general
report|https://builds.apache.org/job/HBase%20Nightly/job/master/1154//General_Nightly_Build_Report/]
(x) {color:red}-1 jdk8 hadoop2 checks{color}
-- For more information [see jdk8 (hadoop2)
report|https://builds.apache.org/job/HBase%20Nightly/job/master/1154//JDK8_Nightly_Build_Report_(Hadoop2)/]
(/) {color:green}+1 jdk8 hadoop3 checks{color}
-- For more information [see jdk8 (hadoop3)
report|https://builds.apache.org/job/HBase%20Nightly/job/master/1154//JDK8_Nightly_Build_Report_(Hadoop3)/]
(/) {color:green}+1 source release artifact{color}
-- See build output for details.
(/) {color:green}+1 client integration test{color}
> BufferedMutatorOverAsyncBufferedMutator.tryCompleteFuture consume too much
> CPU time
> ---
>
> Key: HBASE-22577
> URL: https://issues.apache.org/jira/browse/HBASE-22577
> Project: HBase
> Issue Type: Sub-task
> Components: Client
>Reporter: Duo Zhang
>Assignee: Duo Zhang
>Priority: Major
> Fix For: 3.0.0
>
>
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
[jira] [Commented] (HBASE-21585) Remove ClusterConnection
[
https://issues.apache.org/jira/browse/HBASE-21585?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16866944#comment-16866944
]
Hudson commented on HBASE-21585:
Results for branch master
[build #1154 on
builds.a.o|https://builds.apache.org/job/HBase%20Nightly/job/master/1154/]: (x)
*{color:red}-1 overall{color}*
details (if available):
(/) {color:green}+1 general checks{color}
-- For more information [see general
report|https://builds.apache.org/job/HBase%20Nightly/job/master/1154//General_Nightly_Build_Report/]
(x) {color:red}-1 jdk8 hadoop2 checks{color}
-- For more information [see jdk8 (hadoop2)
report|https://builds.apache.org/job/HBase%20Nightly/job/master/1154//JDK8_Nightly_Build_Report_(Hadoop2)/]
(/) {color:green}+1 jdk8 hadoop3 checks{color}
-- For more information [see jdk8 (hadoop3)
report|https://builds.apache.org/job/HBase%20Nightly/job/master/1154//JDK8_Nightly_Build_Report_(Hadoop3)/]
(/) {color:green}+1 source release artifact{color}
-- See build output for details.
(/) {color:green}+1 client integration test{color}
> Remove ClusterConnection
>
>
> Key: HBASE-21585
> URL: https://issues.apache.org/jira/browse/HBASE-21585
> Project: HBase
> Issue Type: Sub-task
> Components: Client
>Reporter: Duo Zhang
>Assignee: Duo Zhang
>Priority: Major
> Fix For: 3.0.0
>
> Attachments: HBASE-21585-HBASE-21512-v1.patch,
> HBASE-21585-HBASE-21512-v2.patch, HBASE-21585-HBASE-21512-v3.patch,
> HBASE-21585-HBASE-21512-v4.patch, HBASE-21585-HBASE-21512-v5.patch,
> HBASE-21585-HBASE-21512-v5.patch, HBASE-21585-HBASE-21512.patch
>
>
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
[jira] [Commented] (HBASE-21538) Rewrite RegionReplicaFlushHandler to use AsyncClusterConnection
[
https://issues.apache.org/jira/browse/HBASE-21538?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16866938#comment-16866938
]
Hudson commented on HBASE-21538:
Results for branch master
[build #1154 on
builds.a.o|https://builds.apache.org/job/HBase%20Nightly/job/master/1154/]: (x)
*{color:red}-1 overall{color}*
details (if available):
(/) {color:green}+1 general checks{color}
-- For more information [see general
report|https://builds.apache.org/job/HBase%20Nightly/job/master/1154//General_Nightly_Build_Report/]
(x) {color:red}-1 jdk8 hadoop2 checks{color}
-- For more information [see jdk8 (hadoop2)
report|https://builds.apache.org/job/HBase%20Nightly/job/master/1154//JDK8_Nightly_Build_Report_(Hadoop2)/]
(/) {color:green}+1 jdk8 hadoop3 checks{color}
-- For more information [see jdk8 (hadoop3)
report|https://builds.apache.org/job/HBase%20Nightly/job/master/1154//JDK8_Nightly_Build_Report_(Hadoop3)/]
(/) {color:green}+1 source release artifact{color}
-- See build output for details.
(/) {color:green}+1 client integration test{color}
> Rewrite RegionReplicaFlushHandler to use AsyncClusterConnection
> ---
>
> Key: HBASE-21538
> URL: https://issues.apache.org/jira/browse/HBASE-21538
> Project: HBase
> Issue Type: Sub-task
> Components: read replicas
>Reporter: Duo Zhang
>Assignee: Duo Zhang
>Priority: Major
> Fix For: 3.0.0
>
> Attachments: HBASE-21538-HBASE-21512-v1.patch,
> HBASE-21538-HBASE-21512-v2.patch, HBASE-21538-HBASE-21512-v2.patch,
> HBASE-21538-HBASE-21512-v3.patch, HBASE-21538-HBASE-21512.patch
>
>
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
[jira] [Commented] (HBASE-21718) Implement Admin based on AsyncAdmin
[
https://issues.apache.org/jira/browse/HBASE-21718?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16866946#comment-16866946
]
Hudson commented on HBASE-21718:
Results for branch master
[build #1154 on
builds.a.o|https://builds.apache.org/job/HBase%20Nightly/job/master/1154/]: (x)
*{color:red}-1 overall{color}*
details (if available):
(/) {color:green}+1 general checks{color}
-- For more information [see general
report|https://builds.apache.org/job/HBase%20Nightly/job/master/1154//General_Nightly_Build_Report/]
(x) {color:red}-1 jdk8 hadoop2 checks{color}
-- For more information [see jdk8 (hadoop2)
report|https://builds.apache.org/job/HBase%20Nightly/job/master/1154//JDK8_Nightly_Build_Report_(Hadoop2)/]
(/) {color:green}+1 jdk8 hadoop3 checks{color}
-- For more information [see jdk8 (hadoop3)
report|https://builds.apache.org/job/HBase%20Nightly/job/master/1154//JDK8_Nightly_Build_Report_(Hadoop3)/]
(/) {color:green}+1 source release artifact{color}
-- See build output for details.
(/) {color:green}+1 client integration test{color}
> Implement Admin based on AsyncAdmin
> ---
>
> Key: HBASE-21718
> URL: https://issues.apache.org/jira/browse/HBASE-21718
> Project: HBase
> Issue Type: Sub-task
> Components: Admin
>Reporter: Duo Zhang
>Assignee: Duo Zhang
>Priority: Major
> Fix For: 3.0.0
>
> Attachments: HBASE-21718-HBASE-21512-v1.patch,
> HBASE-21718-HBASE-21512-v2.patch, HBASE-21718-HBASE-21512-v3.patch,
> HBASE-21718-HBASE-21512-v4.patch, HBASE-21718-HBASE-21512-v5.patch,
> HBASE-21718-HBASE-21512-v6.patch, HBASE-21718-HBASE-21512.patch
>
>
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
[jira] [Commented] (HBASE-22281) Fix failed shell UTs
[
https://issues.apache.org/jira/browse/HBASE-22281?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16866951#comment-16866951
]
Hudson commented on HBASE-22281:
Results for branch master
[build #1154 on
builds.a.o|https://builds.apache.org/job/HBase%20Nightly/job/master/1154/]: (x)
*{color:red}-1 overall{color}*
details (if available):
(/) {color:green}+1 general checks{color}
-- For more information [see general
report|https://builds.apache.org/job/HBase%20Nightly/job/master/1154//General_Nightly_Build_Report/]
(x) {color:red}-1 jdk8 hadoop2 checks{color}
-- For more information [see jdk8 (hadoop2)
report|https://builds.apache.org/job/HBase%20Nightly/job/master/1154//JDK8_Nightly_Build_Report_(Hadoop2)/]
(/) {color:green}+1 jdk8 hadoop3 checks{color}
-- For more information [see jdk8 (hadoop3)
report|https://builds.apache.org/job/HBase%20Nightly/job/master/1154//JDK8_Nightly_Build_Report_(Hadoop3)/]
(/) {color:green}+1 source release artifact{color}
-- See build output for details.
(/) {color:green}+1 client integration test{color}
> Fix failed shell UTs
>
>
> Key: HBASE-22281
> URL: https://issues.apache.org/jira/browse/HBASE-22281
> Project: HBase
> Issue Type: Sub-task
> Components: shell
>Reporter: Duo Zhang
>Assignee: Duo Zhang
>Priority: Major
> Fix For: 3.0.0
>
> Attachments: HBASE-22281.patch
>
>
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
[jira] [Commented] (HBASE-22238) Fix TestRpcControllerFactory
[
https://issues.apache.org/jira/browse/HBASE-22238?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16866949#comment-16866949
]
Hudson commented on HBASE-22238:
Results for branch master
[build #1154 on
builds.a.o|https://builds.apache.org/job/HBase%20Nightly/job/master/1154/]: (x)
*{color:red}-1 overall{color}*
details (if available):
(/) {color:green}+1 general checks{color}
-- For more information [see general
report|https://builds.apache.org/job/HBase%20Nightly/job/master/1154//General_Nightly_Build_Report/]
(x) {color:red}-1 jdk8 hadoop2 checks{color}
-- For more information [see jdk8 (hadoop2)
report|https://builds.apache.org/job/HBase%20Nightly/job/master/1154//JDK8_Nightly_Build_Report_(Hadoop2)/]
(/) {color:green}+1 jdk8 hadoop3 checks{color}
-- For more information [see jdk8 (hadoop3)
report|https://builds.apache.org/job/HBase%20Nightly/job/master/1154//JDK8_Nightly_Build_Report_(Hadoop3)/]
(/) {color:green}+1 source release artifact{color}
-- See build output for details.
(/) {color:green}+1 client integration test{color}
> Fix TestRpcControllerFactory
>
>
> Key: HBASE-22238
> URL: https://issues.apache.org/jira/browse/HBASE-22238
> Project: HBase
> Issue Type: Sub-task
> Components: test
>Reporter: Duo Zhang
>Assignee: Duo Zhang
>Priority: Major
> Fix For: 3.0.0
>
>
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
[jira] [Commented] (HBASE-21779) Reimplement BulkLoadHFilesTool to use AsyncClusterConnection
[
https://issues.apache.org/jira/browse/HBASE-21779?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16866943#comment-16866943
]
Hudson commented on HBASE-21779:
Results for branch master
[build #1154 on
builds.a.o|https://builds.apache.org/job/HBase%20Nightly/job/master/1154/]: (x)
*{color:red}-1 overall{color}*
details (if available):
(/) {color:green}+1 general checks{color}
-- For more information [see general
report|https://builds.apache.org/job/HBase%20Nightly/job/master/1154//General_Nightly_Build_Report/]
(x) {color:red}-1 jdk8 hadoop2 checks{color}
-- For more information [see jdk8 (hadoop2)
report|https://builds.apache.org/job/HBase%20Nightly/job/master/1154//JDK8_Nightly_Build_Report_(Hadoop2)/]
(/) {color:green}+1 jdk8 hadoop3 checks{color}
-- For more information [see jdk8 (hadoop3)
report|https://builds.apache.org/job/HBase%20Nightly/job/master/1154//JDK8_Nightly_Build_Report_(Hadoop3)/]
(/) {color:green}+1 source release artifact{color}
-- See build output for details.
(/) {color:green}+1 client integration test{color}
> Reimplement BulkLoadHFilesTool to use AsyncClusterConnection
>
>
> Key: HBASE-21779
> URL: https://issues.apache.org/jira/browse/HBASE-21779
> Project: HBase
> Issue Type: Sub-task
> Components: mapreduce
>Reporter: Duo Zhang
>Assignee: Duo Zhang
>Priority: Major
> Fix For: 3.0.0
>
> Attachments: HBASE-21779-HBASE-21512-v1.patch,
> HBASE-21779-HBASE-21512-v2.patch, HBASE-21779-HBASE-21512-v3.patch,
> HBASE-21779-HBASE-21512.patch
>
>
> So we will not rely on the RpcRetryingCaller and ServiceCallable any more.
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
[jira] [Commented] (HBASE-21723) Remove ConnectionImplementation and related classes
[
https://issues.apache.org/jira/browse/HBASE-21723?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16866959#comment-16866959
]
Hudson commented on HBASE-21723:
Results for branch master
[build #1154 on
builds.a.o|https://builds.apache.org/job/HBase%20Nightly/job/master/1154/]: (x)
*{color:red}-1 overall{color}*
details (if available):
(/) {color:green}+1 general checks{color}
-- For more information [see general
report|https://builds.apache.org/job/HBase%20Nightly/job/master/1154//General_Nightly_Build_Report/]
(x) {color:red}-1 jdk8 hadoop2 checks{color}
-- For more information [see jdk8 (hadoop2)
report|https://builds.apache.org/job/HBase%20Nightly/job/master/1154//JDK8_Nightly_Build_Report_(Hadoop2)/]
(/) {color:green}+1 jdk8 hadoop3 checks{color}
-- For more information [see jdk8 (hadoop3)
report|https://builds.apache.org/job/HBase%20Nightly/job/master/1154//JDK8_Nightly_Build_Report_(Hadoop3)/]
(/) {color:green}+1 source release artifact{color}
-- See build output for details.
(/) {color:green}+1 client integration test{color}
> Remove ConnectionImplementation and related classes
> ---
>
> Key: HBASE-21723
> URL: https://issues.apache.org/jira/browse/HBASE-21723
> Project: HBase
> Issue Type: Sub-task
> Components: asyncclient, Client
>Reporter: Duo Zhang
>Assignee: Duo Zhang
>Priority: Major
> Fix For: 3.0.0
>
>
> This is the final goal here. Especially to get rid of the complicated
> AsyncProcess class.
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
[jira] [Commented] (HBASE-21526) Use AsyncClusterConnection in ServerManager for getRsAdmin
[
https://issues.apache.org/jira/browse/HBASE-21526?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16866936#comment-16866936
]
Hudson commented on HBASE-21526:
Results for branch master
[build #1154 on
builds.a.o|https://builds.apache.org/job/HBase%20Nightly/job/master/1154/]: (x)
*{color:red}-1 overall{color}*
details (if available):
(/) {color:green}+1 general checks{color}
-- For more information [see general
report|https://builds.apache.org/job/HBase%20Nightly/job/master/1154//General_Nightly_Build_Report/]
(x) {color:red}-1 jdk8 hadoop2 checks{color}
-- For more information [see jdk8 (hadoop2)
report|https://builds.apache.org/job/HBase%20Nightly/job/master/1154//JDK8_Nightly_Build_Report_(Hadoop2)/]
(/) {color:green}+1 jdk8 hadoop3 checks{color}
-- For more information [see jdk8 (hadoop3)
report|https://builds.apache.org/job/HBase%20Nightly/job/master/1154//JDK8_Nightly_Build_Report_(Hadoop3)/]
(/) {color:green}+1 source release artifact{color}
-- See build output for details.
(/) {color:green}+1 client integration test{color}
> Use AsyncClusterConnection in ServerManager for getRsAdmin
> --
>
> Key: HBASE-21526
> URL: https://issues.apache.org/jira/browse/HBASE-21526
> Project: HBase
> Issue Type: Sub-task
>Reporter: Duo Zhang
>Assignee: Duo Zhang
>Priority: Major
> Fix For: 3.0.0
>
> Attachments: HBASE-21526-HBASE-21512-v1.patch,
> HBASE-21526-HBASE-21512-v2.patch, HBASE-21526-HBASE-21512.patch
>
>
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
[jira] [Commented] (HBASE-22241) Fix TestRegionServerCoprocessorEndpoint
[
https://issues.apache.org/jira/browse/HBASE-22241?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16866947#comment-16866947
]
Hudson commented on HBASE-22241:
Results for branch master
[build #1154 on
builds.a.o|https://builds.apache.org/job/HBase%20Nightly/job/master/1154/]: (x)
*{color:red}-1 overall{color}*
details (if available):
(/) {color:green}+1 general checks{color}
-- For more information [see general
report|https://builds.apache.org/job/HBase%20Nightly/job/master/1154//General_Nightly_Build_Report/]
(x) {color:red}-1 jdk8 hadoop2 checks{color}
-- For more information [see jdk8 (hadoop2)
report|https://builds.apache.org/job/HBase%20Nightly/job/master/1154//JDK8_Nightly_Build_Report_(Hadoop2)/]
(/) {color:green}+1 jdk8 hadoop3 checks{color}
-- For more information [see jdk8 (hadoop3)
report|https://builds.apache.org/job/HBase%20Nightly/job/master/1154//JDK8_Nightly_Build_Report_(Hadoop3)/]
(/) {color:green}+1 source release artifact{color}
-- See build output for details.
(/) {color:green}+1 client integration test{color}
> Fix TestRegionServerCoprocessorEndpoint
> ---
>
> Key: HBASE-22241
> URL: https://issues.apache.org/jira/browse/HBASE-22241
> Project: HBase
> Issue Type: Sub-task
> Components: test
>Reporter: Duo Zhang
>Assignee: Duo Zhang
>Priority: Major
> Fix For: 3.0.0
>
>
> In AsyncAdmin we will unwrap the RemoteException so the assertion should be
> changed.
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
[jira] [Commented] (HBASE-21719) Rewrite RegionPlacementMaintainer to use AsyncClusterConnection
[
https://issues.apache.org/jira/browse/HBASE-21719?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16866941#comment-16866941
]
Hudson commented on HBASE-21719:
Results for branch master
[build #1154 on
builds.a.o|https://builds.apache.org/job/HBase%20Nightly/job/master/1154/]: (x)
*{color:red}-1 overall{color}*
details (if available):
(/) {color:green}+1 general checks{color}
-- For more information [see general
report|https://builds.apache.org/job/HBase%20Nightly/job/master/1154//General_Nightly_Build_Report/]
(x) {color:red}-1 jdk8 hadoop2 checks{color}
-- For more information [see jdk8 (hadoop2)
report|https://builds.apache.org/job/HBase%20Nightly/job/master/1154//JDK8_Nightly_Build_Report_(Hadoop2)/]
(/) {color:green}+1 jdk8 hadoop3 checks{color}
-- For more information [see jdk8 (hadoop3)
report|https://builds.apache.org/job/HBase%20Nightly/job/master/1154//JDK8_Nightly_Build_Report_(Hadoop3)/]
(/) {color:green}+1 source release artifact{color}
-- See build output for details.
(/) {color:green}+1 client integration test{color}
> Rewrite RegionPlacementMaintainer to use AsyncClusterConnection
> ---
>
> Key: HBASE-21719
> URL: https://issues.apache.org/jira/browse/HBASE-21719
> Project: HBase
> Issue Type: Sub-task
> Components: Region Assignment
>Reporter: Duo Zhang
>Assignee: Duo Zhang
>Priority: Major
> Fix For: 3.0.0
>
> Attachments: HBASE-21719-HBASE-21512-v1.patch,
> HBASE-21719-HBASE-21512.patch
>
>
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
[jira] [Commented] (HBASE-21516) Use AsyncConnection instead of Connection in SecureBulkLoadManager
[
https://issues.apache.org/jira/browse/HBASE-21516?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16866935#comment-16866935
]
Hudson commented on HBASE-21516:
Results for branch master
[build #1154 on
builds.a.o|https://builds.apache.org/job/HBase%20Nightly/job/master/1154/]: (x)
*{color:red}-1 overall{color}*
details (if available):
(/) {color:green}+1 general checks{color}
-- For more information [see general
report|https://builds.apache.org/job/HBase%20Nightly/job/master/1154//General_Nightly_Build_Report/]
(x) {color:red}-1 jdk8 hadoop2 checks{color}
-- For more information [see jdk8 (hadoop2)
report|https://builds.apache.org/job/HBase%20Nightly/job/master/1154//JDK8_Nightly_Build_Report_(Hadoop2)/]
(/) {color:green}+1 jdk8 hadoop3 checks{color}
-- For more information [see jdk8 (hadoop3)
report|https://builds.apache.org/job/HBase%20Nightly/job/master/1154//JDK8_Nightly_Build_Report_(Hadoop3)/]
(/) {color:green}+1 source release artifact{color}
-- See build output for details.
(/) {color:green}+1 client integration test{color}
> Use AsyncConnection instead of Connection in SecureBulkLoadManager
> --
>
> Key: HBASE-21516
> URL: https://issues.apache.org/jira/browse/HBASE-21516
> Project: HBase
> Issue Type: Sub-task
> Components: regionserver
>Reporter: Duo Zhang
>Assignee: Duo Zhang
>Priority: Major
> Fix For: 3.0.0
>
> Attachments: HBASE-21516-HBASE-21512-v1.patch,
> HBASE-21516-HBASE-21512.patch
>
>
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
[jira] [Commented] (HBASE-21579) Use AsyncClusterConnection for HBaseInterClusterReplicationEndpoint
[
https://issues.apache.org/jira/browse/HBASE-21579?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16866937#comment-16866937
]
Hudson commented on HBASE-21579:
Results for branch master
[build #1154 on
builds.a.o|https://builds.apache.org/job/HBase%20Nightly/job/master/1154/]: (x)
*{color:red}-1 overall{color}*
details (if available):
(/) {color:green}+1 general checks{color}
-- For more information [see general
report|https://builds.apache.org/job/HBase%20Nightly/job/master/1154//General_Nightly_Build_Report/]
(x) {color:red}-1 jdk8 hadoop2 checks{color}
-- For more information [see jdk8 (hadoop2)
report|https://builds.apache.org/job/HBase%20Nightly/job/master/1154//JDK8_Nightly_Build_Report_(Hadoop2)/]
(/) {color:green}+1 jdk8 hadoop3 checks{color}
-- For more information [see jdk8 (hadoop3)
report|https://builds.apache.org/job/HBase%20Nightly/job/master/1154//JDK8_Nightly_Build_Report_(Hadoop3)/]
(/) {color:green}+1 source release artifact{color}
-- See build output for details.
(/) {color:green}+1 client integration test{color}
> Use AsyncClusterConnection for HBaseInterClusterReplicationEndpoint
> ---
>
> Key: HBASE-21579
> URL: https://issues.apache.org/jira/browse/HBASE-21579
> Project: HBase
> Issue Type: Sub-task
> Components: Replication
>Reporter: Duo Zhang
>Assignee: Duo Zhang
>Priority: Major
> Fix For: 3.0.0
>
> Attachments: HBASE-21579-HBASE-21512-v1.patch,
> HBASE-21579-HBASE-21512.patch
>
>
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
[jira] [Commented] (HBASE-22526) RejectedExecutionException could be thrown from TableOverAsyncTable.coprocessor service if the connection has been shutown
[
https://issues.apache.org/jira/browse/HBASE-22526?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16866961#comment-16866961
]
Hudson commented on HBASE-22526:
Results for branch master
[build #1154 on
builds.a.o|https://builds.apache.org/job/HBase%20Nightly/job/master/1154/]: (x)
*{color:red}-1 overall{color}*
details (if available):
(/) {color:green}+1 general checks{color}
-- For more information [see general
report|https://builds.apache.org/job/HBase%20Nightly/job/master/1154//General_Nightly_Build_Report/]
(x) {color:red}-1 jdk8 hadoop2 checks{color}
-- For more information [see jdk8 (hadoop2)
report|https://builds.apache.org/job/HBase%20Nightly/job/master/1154//JDK8_Nightly_Build_Report_(Hadoop2)/]
(/) {color:green}+1 jdk8 hadoop3 checks{color}
-- For more information [see jdk8 (hadoop3)
report|https://builds.apache.org/job/HBase%20Nightly/job/master/1154//JDK8_Nightly_Build_Report_(Hadoop3)/]
(/) {color:green}+1 source release artifact{color}
-- See build output for details.
(/) {color:green}+1 client integration test{color}
> RejectedExecutionException could be thrown from
> TableOverAsyncTable.coprocessor service if the connection has been shutown
> --
>
> Key: HBASE-22526
> URL: https://issues.apache.org/jira/browse/HBASE-22526
> Project: HBase
> Issue Type: Sub-task
> Components: Client
>Reporter: Duo Zhang
>Assignee: Duo Zhang
>Priority: Major
> Fix For: 3.0.0
>
>
> This breaks TestMasterAbortWhileMergingTable I believe.
> https://builds.apache.org/job/HBase-Flaky-Tests/job/HBASE-21512/1151/artifact/hbase-server/target/surefire-reports/org.apache.hadoop.hbase.master.assignment.TestMasterAbortWhileMergingTable-output.txt/*view*/
> {noformat}
> 2019-06-01 00:40:09,104 ERROR [PEWorker-13]
> procedure2.ProcedureExecutor(1667): CODE-BUG: Uncaught runtime exception:
> pid=9, state=RUNNABLE:MERGE_TABLE_REGIONS_UPDATE_META, hasLock=true;
> MergeTableRegionsProcedure table=test,
> regions=[048d06685b8e1df26516884c41b59c9b, 9f7c60e69f7144b32deb828e3a60c8a8],
> forcibly=false
> java.util.concurrent.RejectedExecutionException: Task
> java.util.concurrent.FutureTask@4ba1959e rejected from
> java.util.concurrent.ThreadPoolExecutor@67913a08[Terminated, pool size = 0,
> active threads = 0, queued tasks = 0, completed tasks = 0]
> at
> java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(ThreadPoolExecutor.java:2063)
> at
> java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:830)
> at
> java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1379)
> at
> java.util.concurrent.AbstractExecutorService.submit(AbstractExecutorService.java:134)
> at
> org.apache.hadoop.hbase.client.TableOverAsyncTable.coprocssorService(TableOverAsyncTable.java:427)
> at
> org.apache.hadoop.hbase.client.TableOverAsyncTable.coprocessorService(TableOverAsyncTable.java:457)
> at
> org.apache.hadoop.hbase.client.Table.coprocessorService(Table.java:527)
> at
> org.apache.hadoop.hbase.MetaTableAccessor.multiMutate(MetaTableAccessor.java:1768)
> at
> org.apache.hadoop.hbase.MetaTableAccessor.mergeRegions(MetaTableAccessor.java:1634)
> at
> org.apache.hadoop.hbase.master.assignment.RegionStateStore.mergeRegions(RegionStateStore.java:253)
> at
> org.apache.hadoop.hbase.master.assignment.AssignmentManager.markRegionAsMerged(AssignmentManager.java:1665)
> at
> org.apache.hadoop.hbase.master.assignment.MergeTableRegionsProcedure.updateMetaForMergedRegions(MergeTableRegionsProcedure.java:731)
> at
> org.apache.hadoop.hbase.master.assignment.MergeTableRegionsProcedure.executeFromState(MergeTableRegionsProcedure.java:276)
> at
> org.apache.hadoop.hbase.master.assignment.MergeTableRegionsProcedure.executeFromState(MergeTableRegionsProcedure.java:84)
> at
> org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:194)
> at
> org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:959)
> at
> org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1648)
> at
> org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395)
> at
> org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1100(ProcedureExecutor.java:78)
> at
> org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1964)
> {noformat}
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
[jira] [Commented] (HBASE-21537) Rewrite ServerManager.closeRegionSilentlyAndWait to use AsyncClusterConnection
[
https://issues.apache.org/jira/browse/HBASE-21537?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16866940#comment-16866940
]
Hudson commented on HBASE-21537:
Results for branch master
[build #1154 on
builds.a.o|https://builds.apache.org/job/HBase%20Nightly/job/master/1154/]: (x)
*{color:red}-1 overall{color}*
details (if available):
(/) {color:green}+1 general checks{color}
-- For more information [see general
report|https://builds.apache.org/job/HBase%20Nightly/job/master/1154//General_Nightly_Build_Report/]
(x) {color:red}-1 jdk8 hadoop2 checks{color}
-- For more information [see jdk8 (hadoop2)
report|https://builds.apache.org/job/HBase%20Nightly/job/master/1154//JDK8_Nightly_Build_Report_(Hadoop2)/]
(/) {color:green}+1 jdk8 hadoop3 checks{color}
-- For more information [see jdk8 (hadoop3)
report|https://builds.apache.org/job/HBase%20Nightly/job/master/1154//JDK8_Nightly_Build_Report_(Hadoop3)/]
(/) {color:green}+1 source release artifact{color}
-- See build output for details.
(/) {color:green}+1 client integration test{color}
> Rewrite ServerManager.closeRegionSilentlyAndWait to use AsyncClusterConnection
> --
>
> Key: HBASE-21537
> URL: https://issues.apache.org/jira/browse/HBASE-21537
> Project: HBase
> Issue Type: Sub-task
>Reporter: Duo Zhang
>Assignee: Duo Zhang
>Priority: Major
> Fix For: 3.0.0
>
> Attachments: HBASE-21537-HBASE-21512.patch
>
>
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
[jira] [Commented] (HBASE-22036) Rewrite TestScannerHeartbeatMessages
[
https://issues.apache.org/jira/browse/HBASE-22036?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16866956#comment-16866956
]
Hudson commented on HBASE-22036:
Results for branch master
[build #1154 on
builds.a.o|https://builds.apache.org/job/HBase%20Nightly/job/master/1154/]: (x)
*{color:red}-1 overall{color}*
details (if available):
(/) {color:green}+1 general checks{color}
-- For more information [see general
report|https://builds.apache.org/job/HBase%20Nightly/job/master/1154//General_Nightly_Build_Report/]
(x) {color:red}-1 jdk8 hadoop2 checks{color}
-- For more information [see jdk8 (hadoop2)
report|https://builds.apache.org/job/HBase%20Nightly/job/master/1154//JDK8_Nightly_Build_Report_(Hadoop2)/]
(/) {color:green}+1 jdk8 hadoop3 checks{color}
-- For more information [see jdk8 (hadoop3)
report|https://builds.apache.org/job/HBase%20Nightly/job/master/1154//JDK8_Nightly_Build_Report_(Hadoop3)/]
(/) {color:green}+1 source release artifact{color}
-- See build output for details.
(/) {color:green}+1 client integration test{color}
> Rewrite TestScannerHeartbeatMessages
>
>
> Key: HBASE-22036
> URL: https://issues.apache.org/jira/browse/HBASE-22036
> Project: HBase
> Issue Type: Sub-task
> Components: test
>Reporter: Duo Zhang
>Assignee: Duo Zhang
>Priority: Major
> Fix For: 3.0.0
>
>
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
[jira] [Commented] (HBASE-21671) Rewrite RegionReplicaReplicationEndpoint to use AsyncClusterConnection
[
https://issues.apache.org/jira/browse/HBASE-21671?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16866939#comment-16866939
]
Hudson commented on HBASE-21671:
Results for branch master
[build #1154 on
builds.a.o|https://builds.apache.org/job/HBase%20Nightly/job/master/1154/]: (x)
*{color:red}-1 overall{color}*
details (if available):
(/) {color:green}+1 general checks{color}
-- For more information [see general
report|https://builds.apache.org/job/HBase%20Nightly/job/master/1154//General_Nightly_Build_Report/]
(x) {color:red}-1 jdk8 hadoop2 checks{color}
-- For more information [see jdk8 (hadoop2)
report|https://builds.apache.org/job/HBase%20Nightly/job/master/1154//JDK8_Nightly_Build_Report_(Hadoop2)/]
(/) {color:green}+1 jdk8 hadoop3 checks{color}
-- For more information [see jdk8 (hadoop3)
report|https://builds.apache.org/job/HBase%20Nightly/job/master/1154//JDK8_Nightly_Build_Report_(Hadoop3)/]
(/) {color:green}+1 source release artifact{color}
-- See build output for details.
(/) {color:green}+1 client integration test{color}
> Rewrite RegionReplicaReplicationEndpoint to use AsyncClusterConnection
> --
>
> Key: HBASE-21671
> URL: https://issues.apache.org/jira/browse/HBASE-21671
> Project: HBase
> Issue Type: Sub-task
> Components: read replicas
>Reporter: Duo Zhang
>Assignee: Duo Zhang
>Priority: Major
> Fix For: 3.0.0
>
> Attachments: HBASE-21671-HBASE-21512-v1.patch,
> HBASE-21671-HBASE-21512-v2.patch, HBASE-21671-HBASE-21512-v3.patch,
> HBASE-21671-HBASE-21512.patch
>
>
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
[jira] [Commented] (HBASE-22239) Also catch RemoteException in SyncReplicationTestBase.verifyReplicationRequestRejection
[
https://issues.apache.org/jira/browse/HBASE-22239?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16866955#comment-16866955
]
Hudson commented on HBASE-22239:
Results for branch master
[build #1154 on
builds.a.o|https://builds.apache.org/job/HBase%20Nightly/job/master/1154/]: (x)
*{color:red}-1 overall{color}*
details (if available):
(/) {color:green}+1 general checks{color}
-- For more information [see general
report|https://builds.apache.org/job/HBase%20Nightly/job/master/1154//General_Nightly_Build_Report/]
(x) {color:red}-1 jdk8 hadoop2 checks{color}
-- For more information [see jdk8 (hadoop2)
report|https://builds.apache.org/job/HBase%20Nightly/job/master/1154//JDK8_Nightly_Build_Report_(Hadoop2)/]
(/) {color:green}+1 jdk8 hadoop3 checks{color}
-- For more information [see jdk8 (hadoop3)
report|https://builds.apache.org/job/HBase%20Nightly/job/master/1154//JDK8_Nightly_Build_Report_(Hadoop3)/]
(/) {color:green}+1 source release artifact{color}
-- See build output for details.
(/) {color:green}+1 client integration test{color}
> Also catch RemoteException in
> SyncReplicationTestBase.verifyReplicationRequestRejection
> ---
>
> Key: HBASE-22239
> URL: https://issues.apache.org/jira/browse/HBASE-22239
> Project: HBase
> Issue Type: Sub-task
> Components: asyncclient
>Reporter: Duo Zhang
>Assignee: Duo Zhang
>Priority: Major
> Labels: async-client
> Fix For: 3.0.0
>
> Attachments: HBASE-22239-HBASE-21512.patch
>
>
> See the last few comments in HBASE-22303. We do not have short circuit
> connection for async client so we need to deal with RemoteException.
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
[jira] [Commented] (HBASE-22597) Remove usage of deprecated RandomStringUtils
[
https://issues.apache.org/jira/browse/HBASE-22597?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16866915#comment-16866915
]
Hadoop QA commented on HBASE-22597:
---
| (/) *{color:green}+1 overall{color}* |
\\
\\
|| Vote || Subsystem || Runtime || Comment ||
| {color:blue}0{color} | {color:blue} reexec {color} | {color:blue} 0m
26s{color} | {color:blue} Docker mode activated. {color} |
|| || || || {color:brown} Prechecks {color} ||
| {color:green}+1{color} | {color:green} hbaseanti {color} | {color:green} 0m
0s{color} | {color:green} Patch does not have any anti-patterns. {color} |
| {color:green}+1{color} | {color:green} @author {color} | {color:green} 0m
0s{color} | {color:green} The patch does not contain any @author tags. {color} |
| {color:green}+1{color} | {color:green} test4tests {color} | {color:green} 0m
0s{color} | {color:green} The patch appears to include 6 new or modified test
files. {color} |
|| || || || {color:brown} master Compile Tests {color} ||
| {color:blue}0{color} | {color:blue} mvndep {color} | {color:blue} 0m
24s{color} | {color:blue} Maven dependency ordering for branch {color} |
| {color:green}+1{color} | {color:green} mvninstall {color} | {color:green} 4m
2s{color} | {color:green} master passed {color} |
| {color:green}+1{color} | {color:green} compile {color} | {color:green} 2m
42s{color} | {color:green} master passed {color} |
| {color:green}+1{color} | {color:green} checkstyle {color} | {color:green} 2m
18s{color} | {color:green} master passed {color} |
| {color:green}+1{color} | {color:green} shadedjars {color} | {color:green} 4m
19s{color} | {color:green} branch has no errors when building our shaded
downstream artifacts. {color} |
| {color:blue}0{color} | {color:blue} findbugs {color} | {color:blue} 0m
0s{color} | {color:blue} Skipped patched modules with no Java source:
hbase-shaded . {color} |
| {color:green}+1{color} | {color:green} findbugs {color} | {color:green} 3m
20s{color} | {color:green} master passed {color} |
| {color:green}+1{color} | {color:green} javadoc {color} | {color:green} 3m
21s{color} | {color:green} master passed {color} |
|| || || || {color:brown} Patch Compile Tests {color} ||
| {color:blue}0{color} | {color:blue} mvndep {color} | {color:blue} 0m
13s{color} | {color:blue} Maven dependency ordering for patch {color} |
| {color:green}+1{color} | {color:green} mvninstall {color} | {color:green} 3m
55s{color} | {color:green} the patch passed {color} |
| {color:green}+1{color} | {color:green} compile {color} | {color:green} 2m
39s{color} | {color:green} the patch passed {color} |
| {color:green}+1{color} | {color:green} javac {color} | {color:green} 2m
39s{color} | {color:green} the patch passed {color} |
| {color:green}+1{color} | {color:green} checkstyle {color} | {color:green} 2m
12s{color} | {color:green} the patch passed {color} |
| {color:green}+1{color} | {color:green} whitespace {color} | {color:green} 0m
0s{color} | {color:green} The patch has no whitespace issues. {color} |
| {color:green}+1{color} | {color:green} xml {color} | {color:green} 0m
5s{color} | {color:green} The patch has no ill-formed XML file. {color} |
| {color:green}+1{color} | {color:green} shadedjars {color} | {color:green} 4m
22s{color} | {color:green} patch has no errors when building our shaded
downstream artifacts. {color} |
| {color:green}+1{color} | {color:green} hadoopcheck {color} | {color:green}
11m 59s{color} | {color:green} Patch does not cause any errors with Hadoop
2.8.5 2.9.2 or 3.1.2. {color} |
| {color:blue}0{color} | {color:blue} findbugs {color} | {color:blue} 0m
0s{color} | {color:blue} Skipped patched modules with no Java source:
hbase-shaded . {color} |
| {color:green}+1{color} | {color:green} findbugs {color} | {color:green} 3m
17s{color} | {color:green} the patch passed {color} |
| {color:green}+1{color} | {color:green} javadoc {color} | {color:green} 3m
16s{color} | {color:green} the patch passed {color} |
|| || || || {color:brown} Other Tests {color} ||
| {color:green}+1{color} | {color:green} unit {color} | {color:green}222m
32s{color} | {color:green} root in the patch passed. {color} |
| {color:green}+1{color} | {color:green} asflicense {color} | {color:green} 1m
56s{color} | {color:green} The patch does not generate ASF License warnings.
{color} |
| {color:black}{color} | {color:black} {color} | {color:black}282m 46s{color} |
{color:black} {color} |
\\
\\
|| Subsystem || Report/Notes ||
| Docker | Client=17.05.0-ce Server=17.05.0-ce base:
https://builds.apache.org/job/HBase-PreCommit-GitHub-PR/job/PR-317/2/artifact/out/Dockerfile
|
| GITHUB PR | https://github.com/apache/hbase/pull/317 |
| JIRA Issue | HBASE-22597 |
| Optional Tests | dupname asflicense javac javadoc unit shadedjars
hadoopcheck xml compile findbugs hbaseanti checkstyle |
| uname | Linux 7d16ac63b631 4.4.0-138-generic #164-Ubuntu SMP
[GitHub] [hbase] Apache-HBase commented on issue #317: [HBASE-22597] : Replacing usages of deprecated RandomStringUtils
Apache-HBase commented on issue #317: [HBASE-22597] : Replacing usages of deprecated RandomStringUtils URL: https://github.com/apache/hbase/pull/317#issuecomment-503256454 :confetti_ball: **+1 overall** | Vote | Subsystem | Runtime | Comment | |::|--:|:|:| | 0 | reexec | 26 | Docker mode activated. | ||| _ Prechecks _ | | +1 | hbaseanti | 0 | Patch does not have any anti-patterns. | | +1 | @author | 0 | The patch does not contain any @author tags. | | +1 | test4tests | 0 | The patch appears to include 6 new or modified test files. | ||| _ master Compile Tests _ | | 0 | mvndep | 24 | Maven dependency ordering for branch | | +1 | mvninstall | 242 | master passed | | +1 | compile | 162 | master passed | | +1 | checkstyle | 138 | master passed | | +1 | shadedjars | 259 | branch has no errors when building our shaded downstream artifacts. | | 0 | findbugs | 0 | Skipped patched modules with no Java source: hbase-shaded . | | +1 | findbugs | 200 | master passed | | +1 | javadoc | 201 | master passed | ||| _ Patch Compile Tests _ | | 0 | mvndep | 13 | Maven dependency ordering for patch | | +1 | mvninstall | 235 | the patch passed | | +1 | compile | 159 | the patch passed | | +1 | javac | 159 | the patch passed | | +1 | checkstyle | 132 | the patch passed | | +1 | whitespace | 0 | The patch has no whitespace issues. | | +1 | xml | 5 | The patch has no ill-formed XML file. | | +1 | shadedjars | 262 | patch has no errors when building our shaded downstream artifacts. | | +1 | hadoopcheck | 719 | Patch does not cause any errors with Hadoop 2.8.5 2.9.2 or 3.1.2. | | 0 | findbugs | 0 | Skipped patched modules with no Java source: hbase-shaded . | | +1 | findbugs | 197 | the patch passed | | +1 | javadoc | 196 | the patch passed | ||| _ Other Tests _ | | +1 | unit | 13352 | root in the patch passed. | | +1 | asflicense | 116 | The patch does not generate ASF License warnings. | | | | 16966 | | | Subsystem | Report/Notes | |--:|:-| | Docker | Client=17.05.0-ce Server=17.05.0-ce base: https://builds.apache.org/job/HBase-PreCommit-GitHub-PR/job/PR-317/2/artifact/out/Dockerfile | | GITHUB PR | https://github.com/apache/hbase/pull/317 | | JIRA Issue | HBASE-22597 | | Optional Tests | dupname asflicense javac javadoc unit shadedjars hadoopcheck xml compile findbugs hbaseanti checkstyle | | uname | Linux 7d16ac63b631 4.4.0-138-generic #164-Ubuntu SMP Tue Oct 2 17:16:02 UTC 2018 x86_64 GNU/Linux | | Build tool | maven | | Personality | /testptch/patchprocess/precommit/personality/provided.sh | | git revision | master / 5da5deb59d | | maven | version: Apache Maven 3.5.4 (1edded0938998edf8bf061f1ceb3cfdeccf443fe; 2018-06-17T18:33:14Z) | | Default Java | 1.8.0_181 | | findbugs | v3.1.11 | | Test Results | https://builds.apache.org/job/HBase-PreCommit-GitHub-PR/job/PR-317/2/testReport/ | | Max. process+thread count | 5664 (vs. ulimit of 1) | | modules | C: hbase-server hbase-shaded . U: . | | Console output | https://builds.apache.org/job/HBase-PreCommit-GitHub-PR/job/PR-317/2/console | | Powered by | Apache Yetus 0.9.0 http://yetus.apache.org | This message was automatically generated. This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
[GitHub] [hbase] syedmurtazahassan opened a new pull request #322: HBASE-22586 Javadoc Warnings related to @param tag
syedmurtazahassan opened a new pull request #322: HBASE-22586 Javadoc Warnings related to @param tag URL: https://github.com/apache/hbase/pull/322 Fixed javadoc warnings related to @param tag. This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
[GitHub] [hbase] virajjasani commented on a change in pull request #311: [HBASE-22592] : HMaster Construction failure stacktrace to be availab…
virajjasani commented on a change in pull request #311: [HBASE-22592] : HMaster
Construction failure stacktrace to be availab…
URL: https://github.com/apache/hbase/pull/311#discussion_r294091754
##
File path:
hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMasterCommandLine.java
##
@@ -244,6 +244,7 @@ private int startMaster() {
throw new RuntimeException("HMaster Aborted");
}
} catch (Throwable t) {
+ t.printStackTrace();
Review comment:
I thought about that but as per the screenshot provided in the description,
the Exception provided is not available in log file because Logger is not yet
loaded.
Anyways, adding printstacktrace() would ensure that the StackTrace is
atleast available in .out file. If the Logger is created and loaded by that
time, then it would also be present in .log file based on: LOG.error("Master
exiting", t);
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
[GitHub] [hbase] Apache-HBase commented on issue #301: HBASE-22547 Document for offheap read in HBase Book
Apache-HBase commented on issue #301: HBASE-22547 Document for offheap read in HBase Book URL: https://github.com/apache/hbase/pull/301#issuecomment-503241775 :broken_heart: **-1 overall** | Vote | Subsystem | Runtime | Comment | |::|--:|:|:| | 0 | reexec | 31 | Docker mode activated. | ||| _ Prechecks _ | | +1 | hbaseanti | 0 | Patch does not have any anti-patterns. | | +1 | @author | 0 | The patch does not contain any @author tags. | | -0 | test4tests | 0 | The patch doesn't appear to include any new or modified tests. Please justify why no new tests are needed for this patch. Also please list what manual steps were performed to verify this patch. | ||| _ HBASE-21879 Compile Tests _ | | 0 | mvndep | 21 | Maven dependency ordering for branch | | +1 | mvninstall | 252 | HBASE-21879 passed | | +1 | compile | 173 | HBASE-21879 passed | | +1 | checkstyle | 129 | HBASE-21879 passed | | +1 | mvnsite | 1127 | HBASE-21879 passed | | 0 | refguide | 465 | branch has no errors when building the reference guide. See footer for rendered docs, which you should manually inspect. | | +1 | shadedjars | 270 | branch has no errors when building our shaded downstream artifacts. | | 0 | findbugs | 0 | Skipped patched modules with no Java source: . | | +1 | findbugs | 37 | HBASE-21879 passed | | +1 | javadoc | 175 | HBASE-21879 passed | ||| _ Patch Compile Tests _ | | 0 | mvndep | 14 | Maven dependency ordering for patch | | +1 | mvninstall | 234 | the patch passed | | +1 | compile | 161 | the patch passed | | +1 | javac | 161 | the patch passed | | +1 | checkstyle | 135 | the patch passed | | +1 | mvnsite | 1034 | the patch passed | | +1 | whitespace | 0 | The patch has no whitespace issues. | | 0 | refguide | 448 | patch has no errors when building the reference guide. See footer for rendered docs, which you should manually inspect. | | +1 | shadedjars | 265 | patch has no errors when building our shaded downstream artifacts. | | +1 | hadoopcheck | 732 | Patch does not cause any errors with Hadoop 2.8.5 2.9.2 or 3.1.2. | | 0 | findbugs | 0 | Skipped patched modules with no Java source: . | | +1 | findbugs | 47 | the patch passed | | +1 | javadoc | 181 | the patch passed | ||| _ Other Tests _ | | -1 | unit | 12273 | root in the patch failed. | | +1 | asflicense | 79 | The patch does not generate ASF License warnings. | | | | 18610 | | | Reason | Tests | |---:|:--| | Failed junit tests | hadoop.hbase.client.TestAdminShell | | | hadoop.hbase.client.TestTableShell | | Subsystem | Report/Notes | |--:|:-| | Docker | Client=17.05.0-ce Server=17.05.0-ce base: https://builds.apache.org/job/HBase-PreCommit-GitHub-PR/job/PR-301/5/artifact/out/Dockerfile | | GITHUB PR | https://github.com/apache/hbase/pull/301 | | Optional Tests | dupname asflicense javac javadoc unit findbugs shadedjars hadoopcheck hbaseanti checkstyle compile refguide mvnsite | | uname | Linux e055d9ffdc2e 4.4.0-138-generic #164-Ubuntu SMP Tue Oct 2 17:16:02 UTC 2018 x86_64 GNU/Linux | | Build tool | maven | | Personality | /testptch/patchprocess/precommit/personality/provided.sh | | git revision | HBASE-21879 / afaf7a9e35 | | maven | version: Apache Maven 3.5.4 (1edded0938998edf8bf061f1ceb3cfdeccf443fe; 2018-06-17T18:33:14Z) | | Default Java | 1.8.0_181 | | refguide | https://builds.apache.org/job/HBase-PreCommit-GitHub-PR/job/PR-301/5/artifact/out/branch-site/book.html | | findbugs | v3.1.11 | | refguide | https://builds.apache.org/job/HBase-PreCommit-GitHub-PR/job/PR-301/5/artifact/out/patch-site/book.html | | unit | https://builds.apache.org/job/HBase-PreCommit-GitHub-PR/job/PR-301/5/artifact/out/patch-unit-root.txt | | Test Results | https://builds.apache.org/job/HBase-PreCommit-GitHub-PR/job/PR-301/5/testReport/ | | Max. process+thread count | 5572 (vs. ulimit of 1) | | modules | C: hbase-common . U: . | | Console output | https://builds.apache.org/job/HBase-PreCommit-GitHub-PR/job/PR-301/5/console | | Powered by | Apache Yetus 0.9.0 http://yetus.apache.org | This message was automatically generated. This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
[GitHub] [hbase] Apache-HBase commented on issue #321: HBASE-22600 Document that LoadIncrementalHFiles will be removed in 3.0.0
Apache-HBase commented on issue #321: HBASE-22600 Document that LoadIncrementalHFiles will be removed in 3.0.0 URL: https://github.com/apache/hbase/pull/321#issuecomment-503236395 :confetti_ball: **+1 overall** | Vote | Subsystem | Runtime | Comment | |::|--:|:|:| | 0 | reexec | 21 | Docker mode activated. | ||| _ Prechecks _ | | +1 | hbaseanti | 0 | Patch does not have any anti-patterns. | | +1 | @author | 0 | The patch does not contain any @author tags. | | -0 | test4tests | 0 | The patch doesn't appear to include any new or modified tests. Please justify why no new tests are needed for this patch. Also please list what manual steps were performed to verify this patch. | ||| _ branch-2 Compile Tests _ | | +1 | mvninstall | 289 | branch-2 passed | | +1 | compile | 57 | branch-2 passed | | +1 | checkstyle | 78 | branch-2 passed | | +1 | shadedjars | 270 | branch has no errors when building our shaded downstream artifacts. | | +1 | findbugs | 220 | branch-2 passed | | +1 | javadoc | 39 | branch-2 passed | ||| _ Patch Compile Tests _ | | +1 | mvninstall | 275 | the patch passed | | +1 | compile | 64 | the patch passed | | +1 | javac | 64 | the patch passed | | +1 | checkstyle | 84 | the patch passed | | +1 | whitespace | 0 | The patch has no whitespace issues. | | +1 | shadedjars | 278 | patch has no errors when building our shaded downstream artifacts. | | +1 | hadoopcheck | 760 | Patch does not cause any errors with Hadoop 2.8.5 2.9.2 or 3.1.2. | | +1 | findbugs | 184 | the patch passed | | +1 | javadoc | 31 | the patch passed | ||| _ Other Tests _ | | +1 | unit | 7995 | hbase-server in the patch passed. | | +1 | asflicense | 26 | The patch does not generate ASF License warnings. | | | | 11007 | | | Subsystem | Report/Notes | |--:|:-| | Docker | Client=17.05.0-ce Server=17.05.0-ce base: https://builds.apache.org/job/HBase-PreCommit-GitHub-PR/job/PR-321/1/artifact/out/Dockerfile | | GITHUB PR | https://github.com/apache/hbase/pull/321 | | Optional Tests | dupname asflicense javac javadoc unit findbugs shadedjars hadoopcheck hbaseanti checkstyle compile | | uname | Linux ea4dda5bf1ba 4.4.0-143-generic #169~14.04.2-Ubuntu SMP Wed Feb 13 15:00:41 UTC 2019 x86_64 GNU/Linux | | Build tool | maven | | Personality | /testptch/patchprocess/precommit/personality/provided.sh | | git revision | branch-2 / 6cff25cc23 | | maven | version: Apache Maven 3.5.4 (1edded0938998edf8bf061f1ceb3cfdeccf443fe; 2018-06-17T18:33:14Z) | | Default Java | 1.8.0_181 | | findbugs | v3.1.11 | | Test Results | https://builds.apache.org/job/HBase-PreCommit-GitHub-PR/job/PR-321/1/testReport/ | | Max. process+thread count | 4342 (vs. ulimit of 1) | | modules | C: hbase-server U: hbase-server | | Console output | https://builds.apache.org/job/HBase-PreCommit-GitHub-PR/job/PR-321/1/console | | Powered by | Apache Yetus 0.9.0 http://yetus.apache.org | This message was automatically generated. This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
[GitHub] [hbase] Apache-HBase commented on issue #251: HBASE-22114 Port HBASE-15560 (TinyLFU-based BlockCache) to branch-1
Apache-HBase commented on issue #251: HBASE-22114 Port HBASE-15560 (TinyLFU-based BlockCache) to branch-1 URL: https://github.com/apache/hbase/pull/251#issuecomment-503235039 :broken_heart: **-1 overall** | Vote | Subsystem | Runtime | Comment | |::|--:|:|:| | 0 | reexec | 141 | Docker mode activated. | ||| _ Prechecks _ | | 0 | findbugs | 1 | Findbugs executables are not available. | | -1 | hbaseanti | 0 | The patch appears use Hadoop classification instead of HBase. | | +1 | @author | 0 | The patch does not contain any @author tags. | | +1 | test4tests | 0 | The patch appears to include 2 new or modified test files. | ||| _ branch-1 Compile Tests _ | | 0 | mvndep | 122 | Maven dependency ordering for branch | | +1 | mvninstall | 141 | branch-1 passed | | +1 | compile | 91 | branch-1 passed | | +1 | checkstyle | 398 | branch-1 passed | | 0 | refguide | 175 | branch has no errors when building the reference guide. See footer for rendered docs, which you should manually inspect. | | -1 | shadedjars | 11 | branch has 10 errors when building our shaded downstream artifacts. | | +1 | javadoc | 179 | branch-1 passed | ||| _ Patch Compile Tests _ | | 0 | mvndep | 5 | Maven dependency ordering for patch | | -1 | mvninstall | 5 | root in the patch failed. | | -1 | compile | 6 | root in the patch failed. | | -1 | javac | 6 | root in the patch failed. | | -1 | checkstyle | 4 | The patch fails to run checkstyle in root | | +1 | checkstyle | 20 | The patch passed checkstyle in hbase-common | | -1 | checkstyle | 7 | The patch fails to run checkstyle in hbase-it | | +1 | checkstyle | 6 | The patch passed checkstyle in hbase-resource-bundle | | +1 | checkstyle | 74 | hbase-server: The patch generated 0 new + 83 unchanged - 12 fixed = 83 total (was 95) | | -1 | checkstyle | 3 | The patch fails to run checkstyle in hbase-tinylfu-blockcache | | +1 | whitespace | 0 | The patch has no whitespace issues. | | +1 | xml | 7 | The patch has no ill-formed XML file. | | -1 | refguide | 3 | patch has 11 errors when building the reference guide. | | -1 | shadedjars | 4 | patch has 11 errors when building our shaded downstream artifacts. | | -1 | hadoopcheck | 4 | The patch causes 11 errors with Hadoop v2.8.5. | | -1 | hadoopcheck | 7 | The patch causes 11 errors with Hadoop v2.9.2. | | -1 | javadoc | 6 | root in the patch failed. | | -1 | javadoc | 7 | hbase-it in the patch failed. | | -1 | javadoc | 5 | hbase-tinylfu-blockcache in the patch failed. | ||| _ Other Tests _ | | -1 | unit | 6 | root in the patch failed. | | +1 | asflicense | 40 | The patch does not generate ASF License warnings. | | | | 1621 | | | Subsystem | Report/Notes | |--:|:-| | Docker | Client=18.09.5 Server=18.09.5 base: https://builds.apache.org/job/HBase-PreCommit-GitHub-PR/job/PR-251/8/artifact/out/Dockerfile | | GITHUB PR | https://github.com/apache/hbase/pull/251 | | Optional Tests | dupname asflicense javac javadoc unit findbugs shadedjars hadoopcheck hbaseanti checkstyle compile refguide xml | | uname | Linux 6e510a5df94a 4.15.0-48-generic #51-Ubuntu SMP Wed Apr 3 08:28:49 UTC 2019 x86_64 x86_64 x86_64 GNU/Linux | | Build tool | maven | | Personality | /testptch/patchprocess/precommit/personality/provided.sh | | git revision | branch-1 / 55e2a36 | | maven | version: Apache Maven 3.0.5 | | Default Java | 1.8.0_212 | | refguide | https://builds.apache.org/job/HBase-PreCommit-GitHub-PR/job/PR-251/8/artifact/out/branch-site/book.html | | shadedjars | https://builds.apache.org/job/HBase-PreCommit-GitHub-PR/job/PR-251/8/artifact/out/branch-shadedjars.txt | | mvninstall | https://builds.apache.org/job/HBase-PreCommit-GitHub-PR/job/PR-251/8/artifact/out/patch-mvninstall-root.txt | | compile | https://builds.apache.org/job/HBase-PreCommit-GitHub-PR/job/PR-251/8/artifact/out/patch-compile-root.txt | | javac | https://builds.apache.org/job/HBase-PreCommit-GitHub-PR/job/PR-251/8/artifact/out/patch-compile-root.txt | | checkstyle | https://builds.apache.org/job/HBase-PreCommit-GitHub-PR/job/PR-251/8/artifact/out//testptch/patchprocess/maven-patch-checkstyle-root.txt | | checkstyle | https://builds.apache.org/job/HBase-PreCommit-GitHub-PR/job/PR-251/8/artifact/out//testptch/patchprocess/maven-patch-checkstyle-hbase-it.txt | | checkstyle | https://builds.apache.org/job/HBase-PreCommit-GitHub-PR/job/PR-251/8/artifact/out//testptch/patchprocess/maven-patch-checkstyle-hbase-tinylfu-blockcache.txt | | refguide | https://builds.apache.org/job/HBase-PreCommit-GitHub-PR/job/PR-251/8/artifact/out/patch-refguide.log | | shadedjars | https://builds.apache.org/job/HBase-PreCommit-GitHub-PR/job/PR-251/8/artifact/out/patch-shadedjars.txt | | hadoopcheck |
[jira] [Commented] (HBASE-21879) Read HFile's block to ByteBuffer directly instead of to byte for reducing young gc purpose
[
https://issues.apache.org/jira/browse/HBASE-21879?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16866827#comment-16866827
]
Hudson commented on HBASE-21879:
Results for branch HBASE-21879
[build #150 on
builds.a.o|https://builds.apache.org/job/HBase%20Nightly/job/HBASE-21879/150/]:
(x) *{color:red}-1 overall{color}*
details (if available):
(x) {color:red}-1 general checks{color}
-- For more information [see general
report|https://builds.apache.org/job/HBase%20Nightly/job/HBASE-21879/150//General_Nightly_Build_Report/]
(x) {color:red}-1 jdk8 hadoop2 checks{color}
-- For more information [see jdk8 (hadoop2)
report|https://builds.apache.org/job/HBase%20Nightly/job/HBASE-21879/150//JDK8_Nightly_Build_Report_(Hadoop2)/]
(x) {color:red}-1 jdk8 hadoop3 checks{color}
-- For more information [see jdk8 (hadoop3)
report|https://builds.apache.org/job/HBase%20Nightly/job/HBASE-21879/150//JDK8_Nightly_Build_Report_(Hadoop3)/]
(/) {color:green}+1 source release artifact{color}
-- See build output for details.
(/) {color:green}+1 client integration test{color}
> Read HFile's block to ByteBuffer directly instead of to byte for reducing
> young gc purpose
> --
>
> Key: HBASE-21879
> URL: https://issues.apache.org/jira/browse/HBASE-21879
> Project: HBase
> Issue Type: Improvement
>Reporter: Zheng Hu
>Assignee: Zheng Hu
>Priority: Major
> Fix For: 3.0.0, 2.3.0
>
> Attachments: HBASE-21879.v1.patch, HBASE-21879.v1.patch,
> QPS-latencies-before-HBASE-21879.png, gc-data-before-HBASE-21879.png
>
>
> In HFileBlock#readBlockDataInternal, we have the following:
> {code}
> @VisibleForTesting
> protected HFileBlock readBlockDataInternal(FSDataInputStream is, long offset,
> long onDiskSizeWithHeaderL, boolean pread, boolean verifyChecksum,
> boolean updateMetrics)
> throws IOException {
> // .
> // TODO: Make this ByteBuffer-based. Will make it easier to go to HDFS with
> BBPool (offheap).
> byte [] onDiskBlock = new byte[onDiskSizeWithHeader + hdrSize];
> int nextBlockOnDiskSize = readAtOffset(is, onDiskBlock, preReadHeaderSize,
> onDiskSizeWithHeader - preReadHeaderSize, true, offset +
> preReadHeaderSize, pread);
> if (headerBuf != null) {
> // ...
> }
> // ...
> }
> {code}
> In the read path, we still read the block from hfile to on-heap byte[], then
> copy the on-heap byte[] to offheap bucket cache asynchronously, and in my
> 100% get performance test, I also observed some frequent young gc, The
> largest memory footprint in the young gen should be the on-heap block byte[].
> In fact, we can read HFile's block to ByteBuffer directly instead of to
> byte[] for reducing young gc purpose. we did not implement this before,
> because no ByteBuffer reading interface in the older HDFS client, but 2.7+
> has supported this now, so we can fix this now. I think.
> Will provide an patch and some perf-comparison for this.
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
[jira] [Commented] (HBASE-22561) modify HFilePrettyPrinter to accept non-hbase.rootdir directories
[
https://issues.apache.org/jira/browse/HBASE-22561?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16866778#comment-16866778
]
Artem Ervits commented on HBASE-22561:
--
I'll work on PRs going forward, I attached .v12 for your review, thanks
[~openinx].
> modify HFilePrettyPrinter to accept non-hbase.rootdir directories
> -
>
> Key: HBASE-22561
> URL: https://issues.apache.org/jira/browse/HBASE-22561
> Project: HBase
> Issue Type: Improvement
>Affects Versions: 3.0.0, 2.2.0, 2.3.0, 2.1.5
>Reporter: Artem Ervits
>Assignee: Artem Ervits
>Priority: Minor
> Attachments: HBASE-22561.v06.patch, HBASE-22561.v07.patch,
> HBASE-22561.v11.patch, HBASE-22561.v12.patch, non_root_dir.png, root_dir.png
>
>
> Today {{hbase hfile}} utility accepts only _hbase.rootDir_ directory. It is
> more useful to pass other locations with HFile as well.
> {code:java}
> hbase org.apache.hadoop.hbase.io.hfile.HFilePrettyPrinter hfileoutput
> ...
> ERROR, file (hfileoutput) is not in HBase's root directory
> (hdfs://hadoop.example.com:9000/apps/hbase/)
> {code}
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
[jira] [Updated] (HBASE-22561) modify HFilePrettyPrinter to accept non-hbase.rootdir directories
[
https://issues.apache.org/jira/browse/HBASE-22561?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Artem Ervits updated HBASE-22561:
-
Attachment: HBASE-22561.v12.patch
> modify HFilePrettyPrinter to accept non-hbase.rootdir directories
> -
>
> Key: HBASE-22561
> URL: https://issues.apache.org/jira/browse/HBASE-22561
> Project: HBase
> Issue Type: Improvement
>Affects Versions: 3.0.0, 2.2.0, 2.3.0, 2.1.5
>Reporter: Artem Ervits
>Assignee: Artem Ervits
>Priority: Minor
> Attachments: HBASE-22561.v06.patch, HBASE-22561.v07.patch,
> HBASE-22561.v11.patch, HBASE-22561.v12.patch, non_root_dir.png, root_dir.png
>
>
> Today {{hbase hfile}} utility accepts only _hbase.rootDir_ directory. It is
> more useful to pass other locations with HFile as well.
> {code:java}
> hbase org.apache.hadoop.hbase.io.hfile.HFilePrettyPrinter hfileoutput
> ...
> ERROR, file (hfileoutput) is not in HBase's root directory
> (hdfs://hadoop.example.com:9000/apps/hbase/)
> {code}
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
[jira] [Commented] (HBASE-22414) Interruption of moving regions in RSGroup will cause regions on wrong rs
[ https://issues.apache.org/jira/browse/HBASE-22414?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16866715#comment-16866715 ] Wellington Chevreuil commented on HBASE-22414: -- Thanks for submitting this patch, [~Xiaolin Ha]! I have a few remarks, would you be able to submit this patch as a pull request, for easing code comments/reviews? > Interruption of moving regions in RSGroup will cause regions on wrong rs > > > Key: HBASE-22414 > URL: https://issues.apache.org/jira/browse/HBASE-22414 > Project: HBase > Issue Type: Bug > Components: rsgroup >Affects Versions: 2.2.0 >Reporter: Xiaolin Ha >Assignee: Xiaolin Ha >Priority: Major > Attachments: HBASE-22414.master.001.patch > > > We bulk moving regions to target RSGroup, and each movement of region will > submit a TRSP, but one TRSP encounters exception will make the whole movement > action terminate. Later regions will not be moved to correct servers unless > reassign. > I think we can skip failed moved regions, and retry to move after all has > been traversed. -- This message was sent by Atlassian JIRA (v7.6.3#76005)
[jira] [Updated] (HBASE-22600) Document that LoadIncrementalHFiles will be removed in 3.0.0
[ https://issues.apache.org/jira/browse/HBASE-22600?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Duo Zhang updated HBASE-22600: -- Fix Version/s: 2.2.1 2.3.0 > Document that LoadIncrementalHFiles will be removed in 3.0.0 > > > Key: HBASE-22600 > URL: https://issues.apache.org/jira/browse/HBASE-22600 > Project: HBase > Issue Type: Sub-task >Reporter: Duo Zhang >Assignee: Duo Zhang >Priority: Major > Fix For: 2.3.0, 2.2.1 > > > Here we break the rule, it should be removed in 4.0.0 by default. So we need > to document clearly that it will be removed 3.0.0, and also explain the > reason. -- This message was sent by Atlassian JIRA (v7.6.3#76005)
[GitHub] [hbase] Apache9 opened a new pull request #321: HBASE-22600 Document that LoadIncrementalHFiles will be removed in 3.0.0
Apache9 opened a new pull request #321: HBASE-22600 Document that LoadIncrementalHFiles will be removed in 3.0.0 URL: https://github.com/apache/hbase/pull/321 This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
[jira] [Assigned] (HBASE-22600) Document that LoadIncrementalHFiles will be removed in 3.0.0
[ https://issues.apache.org/jira/browse/HBASE-22600?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Duo Zhang reassigned HBASE-22600: - Assignee: Duo Zhang > Document that LoadIncrementalHFiles will be removed in 3.0.0 > > > Key: HBASE-22600 > URL: https://issues.apache.org/jira/browse/HBASE-22600 > Project: HBase > Issue Type: Sub-task >Reporter: Duo Zhang >Assignee: Duo Zhang >Priority: Major > > Here we break the rule, it should be removed in 4.0.0 by default. So we need > to document clearly that it will be removed 3.0.0, and also explain the > reason. -- This message was sent by Atlassian JIRA (v7.6.3#76005)
[jira] [Commented] (HBASE-22589) Change our RpcServices to implement protobuf Interface instead of BlockingInterface
[ https://issues.apache.org/jira/browse/HBASE-22589?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16866635#comment-16866635 ] Duo Zhang commented on HBASE-22589: --- I'd say this is not easy, should be related with the SEDA changes at server side. Ping [~carp84], do you guys still focus on this? Can share something? Thanks. > Change our RpcServices to implement protobuf Interface instead of > BlockingInterface > --- > > Key: HBASE-22589 > URL: https://issues.apache.org/jira/browse/HBASE-22589 > Project: HBase > Issue Type: Umbrella >Reporter: Duo Zhang >Priority: Major > > In HBASE-21512, we removed the ShortCircultConnection as now we will only use > Interface at client side but our RpcServices only implement > BlockingInterface. We should change them to implement Interface instead. -- This message was sent by Atlassian JIRA (v7.6.3#76005)
[jira] [Commented] (HBASE-21879) Read HFile's block to ByteBuffer directly instead of to byte for reducing young gc purpose
[
https://issues.apache.org/jira/browse/HBASE-21879?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16866633#comment-16866633
]
Hudson commented on HBASE-21879:
Results for branch HBASE-21879
[build #149 on
builds.a.o|https://builds.apache.org/job/HBase%20Nightly/job/HBASE-21879/149/]:
(x) *{color:red}-1 overall{color}*
details (if available):
(x) {color:red}-1 general checks{color}
-- For more information [see general
report|https://builds.apache.org/job/HBase%20Nightly/job/HBASE-21879/149//General_Nightly_Build_Report/]
(x) {color:red}-1 jdk8 hadoop2 checks{color}
-- For more information [see jdk8 (hadoop2)
report|https://builds.apache.org/job/HBase%20Nightly/job/HBASE-21879/149//JDK8_Nightly_Build_Report_(Hadoop2)/]
(x) {color:red}-1 jdk8 hadoop3 checks{color}
-- For more information [see jdk8 (hadoop3)
report|https://builds.apache.org/job/HBase%20Nightly/job/HBASE-21879/149//JDK8_Nightly_Build_Report_(Hadoop3)/]
(/) {color:green}+1 source release artifact{color}
-- See build output for details.
(/) {color:green}+1 client integration test{color}
> Read HFile's block to ByteBuffer directly instead of to byte for reducing
> young gc purpose
> --
>
> Key: HBASE-21879
> URL: https://issues.apache.org/jira/browse/HBASE-21879
> Project: HBase
> Issue Type: Improvement
>Reporter: Zheng Hu
>Assignee: Zheng Hu
>Priority: Major
> Fix For: 3.0.0, 2.3.0
>
> Attachments: HBASE-21879.v1.patch, HBASE-21879.v1.patch,
> QPS-latencies-before-HBASE-21879.png, gc-data-before-HBASE-21879.png
>
>
> In HFileBlock#readBlockDataInternal, we have the following:
> {code}
> @VisibleForTesting
> protected HFileBlock readBlockDataInternal(FSDataInputStream is, long offset,
> long onDiskSizeWithHeaderL, boolean pread, boolean verifyChecksum,
> boolean updateMetrics)
> throws IOException {
> // .
> // TODO: Make this ByteBuffer-based. Will make it easier to go to HDFS with
> BBPool (offheap).
> byte [] onDiskBlock = new byte[onDiskSizeWithHeader + hdrSize];
> int nextBlockOnDiskSize = readAtOffset(is, onDiskBlock, preReadHeaderSize,
> onDiskSizeWithHeader - preReadHeaderSize, true, offset +
> preReadHeaderSize, pread);
> if (headerBuf != null) {
> // ...
> }
> // ...
> }
> {code}
> In the read path, we still read the block from hfile to on-heap byte[], then
> copy the on-heap byte[] to offheap bucket cache asynchronously, and in my
> 100% get performance test, I also observed some frequent young gc, The
> largest memory footprint in the young gen should be the on-heap block byte[].
> In fact, we can read HFile's block to ByteBuffer directly instead of to
> byte[] for reducing young gc purpose. we did not implement this before,
> because no ByteBuffer reading interface in the older HDFS client, but 2.7+
> has supported this now, so we can fix this now. I think.
> Will provide an patch and some perf-comparison for this.
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
[jira] [Commented] (HBASE-21512) Reimplement sync client based on async client
[ https://issues.apache.org/jira/browse/HBASE-21512?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16866623#comment-16866623 ] Duo Zhang commented on HBASE-21512: --- Merged to master. Thanks all for helping. > Reimplement sync client based on async client > - > > Key: HBASE-21512 > URL: https://issues.apache.org/jira/browse/HBASE-21512 > Project: HBase > Issue Type: Umbrella > Components: asyncclient, Client >Reporter: Duo Zhang >Assignee: Duo Zhang >Priority: Major > Fix For: 3.0.0 > > > At least for the RSProcedureDispatcher, with CompletableFuture we do not need > to set a delay and use a thread pool any more, which could reduce the > resource usage and also the latency. > Once this is done, I think we can remove the ClusterConnection completely, > and start to rewrite the old sync client based on the async client, which > could reduce the code base a lot for our client. -- This message was sent by Atlassian JIRA (v7.6.3#76005)
[jira] [Resolved] (HBASE-21512) Reimplement sync client based on async client
[ https://issues.apache.org/jira/browse/HBASE-21512?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Duo Zhang resolved HBASE-21512. --- Resolution: Fixed > Reimplement sync client based on async client > - > > Key: HBASE-21512 > URL: https://issues.apache.org/jira/browse/HBASE-21512 > Project: HBase > Issue Type: Umbrella > Components: asyncclient, Client >Reporter: Duo Zhang >Assignee: Duo Zhang >Priority: Major > Fix For: 3.0.0 > > > At least for the RSProcedureDispatcher, with CompletableFuture we do not need > to set a delay and use a thread pool any more, which could reduce the > resource usage and also the latency. > Once this is done, I think we can remove the ClusterConnection completely, > and start to rewrite the old sync client based on the async client, which > could reduce the code base a lot for our client. -- This message was sent by Atlassian JIRA (v7.6.3#76005)
[jira] [Assigned] (HBASE-21512) Reimplement sync client based on async client
[ https://issues.apache.org/jira/browse/HBASE-21512?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Duo Zhang reassigned HBASE-21512: - Assignee: Duo Zhang > Reimplement sync client based on async client > - > > Key: HBASE-21512 > URL: https://issues.apache.org/jira/browse/HBASE-21512 > Project: HBase > Issue Type: Umbrella > Components: asyncclient, Client >Reporter: Duo Zhang >Assignee: Duo Zhang >Priority: Major > Fix For: 3.0.0 > > > At least for the RSProcedureDispatcher, with CompletableFuture we do not need > to set a delay and use a thread pool any more, which could reduce the > resource usage and also the latency. > Once this is done, I think we can remove the ClusterConnection completely, > and start to rewrite the old sync client based on the async client, which > could reduce the code base a lot for our client. -- This message was sent by Atlassian JIRA (v7.6.3#76005)
[GitHub] [hbase] asfgit merged pull request #287: HBASE-21512 Reimplement sync client based on async client
asfgit merged pull request #287: HBASE-21512 Reimplement sync client based on async client URL: https://github.com/apache/hbase/pull/287 This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
[GitHub] [hbase] Apache-HBase commented on issue #320: HBASE-21879 Read HFile's block to ByteBuffer directly instead of to byte for reducing young gc purpose
Apache-HBase commented on issue #320: HBASE-21879 Read HFile's block to ByteBuffer directly instead of to byte for reducing young gc purpose URL: https://github.com/apache/hbase/pull/320#issuecomment-503140071 :broken_heart: **-1 overall** | Vote | Subsystem | Runtime | Comment | |::|--:|:|:| | 0 | reexec | 0 | Docker mode activated. | | -1 | patch | 7 | https://github.com/apache/hbase/pull/320 does not apply to master. Rebase required? Wrong Branch? See https://yetus.apache.org/documentation/in-progress/precommit-patchnames for help. | | Subsystem | Report/Notes | |--:|:-| | GITHUB PR | https://github.com/apache/hbase/pull/320 | | Console output | https://builds.apache.org/job/HBase-PreCommit-GitHub-PR/job/PR-320/1/console | | Powered by | Apache Yetus 0.9.0 http://yetus.apache.org | This message was automatically generated. This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
[GitHub] [hbase] openinx opened a new pull request #320: HBASE-21879 Read HFile's block to ByteBuffer directly instead of to byte for reducing young gc purpose
openinx opened a new pull request #320: HBASE-21879 Read HFile's block to ByteBuffer directly instead of to byte for reducing young gc purpose URL: https://github.com/apache/hbase/pull/320 This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
[GitHub] [hbase] openinx commented on a change in pull request #301: HBASE-22547 Document for offheap read in HBase Book
openinx commented on a change in pull request #301: HBASE-22547 Document for offheap read in HBase Book URL: https://github.com/apache/hbase/pull/301#discussion_r294823074 ## File path: src/main/asciidoc/_chapters/offheap_read_write.adoc ## @@ -0,0 +1,178 @@ + +/** + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +[[offheap_read_write]] += RegionServer Offheap Read/Write Path +:doctype: book +:numbered: +:toc: left +:icons: font +:experimental: + +[[regionserver.offheap.overview]] +== Overview + +For reducing the Java GC impact to P99/P999 RPC latency, HBase 2.x has made the offheap read and write path. The cells are +allocated from JVM offheap memory area, which won’t be garbage collected by JVM and need to be deallocated explicitly by +upstream callers. In the write path, the request packet received from client will be allocated offheap and retained +until those cells are successfully written to the WAL and Memstore. The memory data structure in Memstore does +not directly store the cell memory, but reference to cells which are encoded in multiple chunks in MSLAB, this is easier +to manage the offheap memory. Similarly, in the read path, we’ll try to read the cache firstly, if the cache +misses, go to the HFile and read the corresponding block. The workflow: from reading blocks to sending cells to +client, it's basically not involved in on-heap memory allocations. + +image::offheap-overview.png[] + + +[[regionserver.offheap.readpath]] +== Offheap read-path +In HBase-2.0.0, link:https://issues.apache.org/jira/browse/HBASE-11425[HBASE-11425] changed the HBase read path so it +could hold the read-data off-heap (from BucketCache) avoiding copying of cached data on to the java heap. +This reduces GC pauses given there is less garbage made and so less to clear. The off-heap read path has a performance +that is similar/better to that of the on-heap LRU cache. This feature is available since HBase 2.0.0. +If the BucketCache is in `file` mode, fetching will always be slower compared to the native on-heap LruBlockCache. +Refer to below blogs for more details and test results on off heaped read path +link:https://blogs.apache.org/hbase/entry/offheaping_the_read_path_in[Offheaping the Read Path in Apache HBase: Part 1 of 2] +and link:https://blogs.apache.org/hbase/entry/offheap-read-path-in-production[Offheap Read-Path in Production - The Alibaba story] + +For an end-to-end off-heaped read-path, first of all there should be an off-heap backed <>(BC). Configure 'hbase.bucketcache.ioengine' to off-heap in +_hbase-site.xml_. Also specify the total capacity of the BC using `hbase.bucketcache.size` config. Please remember to adjust value of 'HBASE_OFFHEAPSIZE' in +_hbase-env.sh_. This is how we specify the max possible off-heap memory allocation for the RegionServer java process. +This should be bigger than the off-heap BC size. Please keep in mind that there is no default for `hbase.bucketcache.ioengine` +which means the BC is turned OFF by default (See <>). + +Next thing to tune is the ByteBuffer pool on the RPC server side: + +NOTE: the config keys which starts with prefix hbase.ipc.server.reservoir are deprecated in HBase3.x. If you are still Review comment: > In all these new document, can we refer to new config at 1st place. Also say the old one and is deprecated in 3.0 @anoopsjohn, I've addressed your comment here. This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
[GitHub] [hbase] openinx commented on a change in pull request #301: HBASE-22547 Document for offheap read in HBase Book
openinx commented on a change in pull request #301: HBASE-22547 Document for offheap read in HBase Book URL: https://github.com/apache/hbase/pull/301#discussion_r294823074 ## File path: src/main/asciidoc/_chapters/offheap_read_write.adoc ## @@ -0,0 +1,178 @@ + +/** + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +[[offheap_read_write]] += RegionServer Offheap Read/Write Path +:doctype: book +:numbered: +:toc: left +:icons: font +:experimental: + +[[regionserver.offheap.overview]] +== Overview + +For reducing the Java GC impact to P99/P999 RPC latency, HBase 2.x has made the offheap read and write path. The cells are +allocated from JVM offheap memory area, which won’t be garbage collected by JVM and need to be deallocated explicitly by +upstream callers. In the write path, the request packet received from client will be allocated offheap and retained +until those cells are successfully written to the WAL and Memstore. The memory data structure in Memstore does +not directly store the cell memory, but reference to cells which are encoded in multiple chunks in MSLAB, this is easier +to manage the offheap memory. Similarly, in the read path, we’ll try to read the cache firstly, if the cache +misses, go to the HFile and read the corresponding block. The workflow: from reading blocks to sending cells to +client, it's basically not involved in on-heap memory allocations. + +image::offheap-overview.png[] + + +[[regionserver.offheap.readpath]] +== Offheap read-path +In HBase-2.0.0, link:https://issues.apache.org/jira/browse/HBASE-11425[HBASE-11425] changed the HBase read path so it +could hold the read-data off-heap (from BucketCache) avoiding copying of cached data on to the java heap. +This reduces GC pauses given there is less garbage made and so less to clear. The off-heap read path has a performance +that is similar/better to that of the on-heap LRU cache. This feature is available since HBase 2.0.0. +If the BucketCache is in `file` mode, fetching will always be slower compared to the native on-heap LruBlockCache. +Refer to below blogs for more details and test results on off heaped read path +link:https://blogs.apache.org/hbase/entry/offheaping_the_read_path_in[Offheaping the Read Path in Apache HBase: Part 1 of 2] +and link:https://blogs.apache.org/hbase/entry/offheap-read-path-in-production[Offheap Read-Path in Production - The Alibaba story] + +For an end-to-end off-heaped read-path, first of all there should be an off-heap backed <>(BC). Configure 'hbase.bucketcache.ioengine' to off-heap in +_hbase-site.xml_. Also specify the total capacity of the BC using `hbase.bucketcache.size` config. Please remember to adjust value of 'HBASE_OFFHEAPSIZE' in +_hbase-env.sh_. This is how we specify the max possible off-heap memory allocation for the RegionServer java process. +This should be bigger than the off-heap BC size. Please keep in mind that there is no default for `hbase.bucketcache.ioengine` +which means the BC is turned OFF by default (See <>). + +Next thing to tune is the ByteBuffer pool on the RPC server side: + +NOTE: the config keys which starts with prefix hbase.ipc.server.reservoir are deprecated in HBase3.x. If you are still Review comment: > In all these new document, can we refer to new config at 1st place. Also say the old one and is deprecated in 3.0 @anoopsjohn, I've addressed your comment here. This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
[jira] [Commented] (HBASE-21512) Reimplement sync client based on async client
[
https://issues.apache.org/jira/browse/HBASE-21512?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16866598#comment-16866598
]
Hudson commented on HBASE-21512:
Results for branch HBASE-21512
[build #298 on
builds.a.o|https://builds.apache.org/job/HBase%20Nightly/job/HBASE-21512/298/]:
(x) *{color:red}-1 overall{color}*
details (if available):
(/) {color:green}+1 general checks{color}
-- For more information [see general
report|https://builds.apache.org/job/HBase%20Nightly/job/HBASE-21512/298//General_Nightly_Build_Report/]
(x) {color:red}-1 jdk8 hadoop2 checks{color}
-- For more information [see jdk8 (hadoop2)
report|https://builds.apache.org/job/HBase%20Nightly/job/HBASE-21512/298//JDK8_Nightly_Build_Report_(Hadoop2)/]
(/) {color:green}+1 jdk8 hadoop3 checks{color}
-- For more information [see jdk8 (hadoop3)
report|https://builds.apache.org/job/HBase%20Nightly/job/HBASE-21512/298//JDK8_Nightly_Build_Report_(Hadoop3)/]
(/) {color:green}+1 source release artifact{color}
-- See build output for details.
(/) {color:green}+1 client integration test{color}
> Reimplement sync client based on async client
> -
>
> Key: HBASE-21512
> URL: https://issues.apache.org/jira/browse/HBASE-21512
> Project: HBase
> Issue Type: Umbrella
> Components: asyncclient, Client
>Reporter: Duo Zhang
>Priority: Major
> Fix For: 3.0.0
>
>
> At least for the RSProcedureDispatcher, with CompletableFuture we do not need
> to set a delay and use a thread pool any more, which could reduce the
> resource usage and also the latency.
> Once this is done, I think we can remove the ClusterConnection completely,
> and start to rewrite the old sync client based on the async client, which
> could reduce the code base a lot for our client.
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
[GitHub] [hbase] openinx commented on a change in pull request #301: HBASE-22547 Document for offheap read in HBase Book
openinx commented on a change in pull request #301: HBASE-22547 Document for offheap read in HBase Book URL: https://github.com/apache/hbase/pull/301#discussion_r294800257 ## File path: src/main/asciidoc/_chapters/offheap_read_write.adoc ## @@ -0,0 +1,168 @@ + +/** + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +[[offheap_read_write]] += RegionServer Offheap Read/Write Path +:doctype: book +:numbered: +:toc: left +:icons: font +:experimental: + +[[regionserver.offheap.overview]] +== Overview + +For reducing the Java GC impact to P99/P999 RPC latency, HBase 2.x has made the offheap read and write path. The cells are +allocated from JVM offheap memory area, which won’t be garbage collected by JVM and need to be deallocated explicitly by +upstream callers. In the write path, the request packet received from client will be allocated offheap and retained +until those cells are successfully written to the WAL and Memstore. The memory data structure in Memstore does +not directly store the cell memory, but reference to cells which are encoded in multiple chunks in MSLAB, this is easier +to manage the offheap memory. Similarly, in the read path, we’ll try to read the cache firstly, if the cache +misses, go to the HFile and read the corresponding block. The workflow: from reading blocks to sending cells to +client, it's basically not involved in on-heap memory allocations. + +image::offheap-overview.png[] + + +[[regionserver.offheap.readpath]] +== Offheap read-path +In HBase-2.0.0, link:https://issues.apache.org/jira/browse/HBASE-11425[HBASE-11425] changed the HBase read path so it +could hold the read-data off-heap (from BucketCache) avoiding copying of cached data on to the java heap. +This reduces GC pauses given there is less garbage made and so less to clear. The off-heap read path has a performance +that is similar/better to that of the on-heap LRU cache. This feature is available since HBase 2.0.0. +If the BucketCache is in `file` mode, fetching will always be slower compared to the native on-heap LruBlockCache. +Refer to below blogs for more details and test results on off heaped read path +link:https://blogs.apache.org/hbase/entry/offheaping_the_read_path_in[Offheaping the Read Path in Apache HBase: Part 1 of 2] +and link:https://blogs.apache.org/hbase/entry/offheap-read-path-in-production[Offheap Read-Path in Production - The Alibaba story] + +For an end-to-end off-heaped read-path, first of all there should be an off-heap backed <>(BC). Configure 'hbase.bucketcache.ioengine' to off-heap in +_hbase-site.xml_. Also specify the total capacity of the BC using `hbase.bucketcache.size` config. Please remember to adjust value of 'HBASE_OFFHEAPSIZE' in +_hbase-env.sh_. This is how we specify the max possible off-heap memory allocation for the RegionServer java process. +This should be bigger than the off-heap BC size. Please keep in mind that there is no default for `hbase.bucketcache.ioengine` +which means the BC is turned OFF by default (See <>). + +Next thing to tune is the ByteBuffer pool on the RPC server side. +The buffers from this pool will be used to accumulate the cell bytes and create a result cell block to send back to the client side. +`hbase.ipc.server.reservoir.enabled` can be used to turn this pool ON or OFF. By default this pool is ON and available. HBase will create off heap ByteBuffers +and pool them. Please make sure not to turn this OFF if you want end-to-end off-heaping in read path. +If this pool is turned off, the server will create temp buffers on heap to accumulate the cell bytes and make a result cell block. This can impact the GC on a highly read loaded server. +The user can tune this pool with respect to how many buffers are in the pool and what should be the size of each ByteBuffer. +Use the config `hbase.ipc.server.reservoir.initial.buffer.size` to tune each of the buffer sizes. Default is 64 KB for HBase2.x, while it will be changed to 65KB by default for HBase3.x Review comment: That's OK. This is an automated message from the Apache Git Service. To respond to the message,
[GitHub] [hbase] openinx commented on a change in pull request #301: HBASE-22547 Document for offheap read in HBase Book
openinx commented on a change in pull request #301: HBASE-22547 Document for offheap read in HBase Book URL: https://github.com/apache/hbase/pull/301#discussion_r294799549 ## File path: src/main/asciidoc/_chapters/offheap_read_write.adoc ## @@ -0,0 +1,168 @@ + +/** + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +[[offheap_read_write]] += RegionServer Offheap Read/Write Path +:doctype: book +:numbered: +:toc: left +:icons: font +:experimental: + +[[regionserver.offheap.overview]] +== Overview + +For reducing the Java GC impact to P99/P999 RPC latency, HBase 2.x has made the offheap read and write path. The cells are +allocated from JVM offheap memory area, which won’t be garbage collected by JVM and need to be deallocated explicitly by +upstream callers. In the write path, the request packet received from client will be allocated offheap and retained +until those cells are successfully written to the WAL and Memstore. The memory data structure in Memstore does +not directly store the cell memory, but reference to cells which are encoded in multiple chunks in MSLAB, this is easier +to manage the offheap memory. Similarly, in the read path, we’ll try to read the cache firstly, if the cache +misses, go to the HFile and read the corresponding block. The workflow: from reading blocks to sending cells to +client, it's basically not involved in on-heap memory allocations. + +image::offheap-overview.png[] + + +[[regionserver.offheap.readpath]] +== Offheap read-path +In HBase-2.0.0, link:https://issues.apache.org/jira/browse/HBASE-11425[HBASE-11425] changed the HBase read path so it +could hold the read-data off-heap (from BucketCache) avoiding copying of cached data on to the java heap. +This reduces GC pauses given there is less garbage made and so less to clear. The off-heap read path has a performance +that is similar/better to that of the on-heap LRU cache. This feature is available since HBase 2.0.0. +If the BucketCache is in `file` mode, fetching will always be slower compared to the native on-heap LruBlockCache. +Refer to below blogs for more details and test results on off heaped read path +link:https://blogs.apache.org/hbase/entry/offheaping_the_read_path_in[Offheaping the Read Path in Apache HBase: Part 1 of 2] +and link:https://blogs.apache.org/hbase/entry/offheap-read-path-in-production[Offheap Read-Path in Production - The Alibaba story] + +For an end-to-end off-heaped read-path, first of all there should be an off-heap backed <>(BC). Configure 'hbase.bucketcache.ioengine' to off-heap in +_hbase-site.xml_. Also specify the total capacity of the BC using `hbase.bucketcache.size` config. Please remember to adjust value of 'HBASE_OFFHEAPSIZE' in +_hbase-env.sh_. This is how we specify the max possible off-heap memory allocation for the RegionServer java process. +This should be bigger than the off-heap BC size. Please keep in mind that there is no default for `hbase.bucketcache.ioengine` +which means the BC is turned OFF by default (See <>). + +Next thing to tune is the ByteBuffer pool on the RPC server side. +The buffers from this pool will be used to accumulate the cell bytes and create a result cell block to send back to the client side. +`hbase.ipc.server.reservoir.enabled` can be used to turn this pool ON or OFF. By default this pool is ON and available. HBase will create off heap ByteBuffers Review comment: Em..seems we can refactor this config as hbase.server.allocator.enabled... the hbase.ipc.server.reservoir.enabled is some confusing ... This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
[GitHub] [hbase] openinx commented on a change in pull request #301: HBASE-22547 Document for offheap read in HBase Book
openinx commented on a change in pull request #301: HBASE-22547 Document for offheap read in HBase Book URL: https://github.com/apache/hbase/pull/301#discussion_r294666137 ## File path: hbase-common/src/main/java/org/apache/hadoop/hbase/io/ByteBuffAllocator.java ## @@ -114,7 +114,7 @@ public static final int DEFAULT_BUFFER_SIZE = 65 * 1024; public static final String MIN_ALLOCATE_SIZE_KEY = - "hbase.ipc.server.reservoir.minimal.allocating.size"; + "hbase.server.reservoir.minimal.allocating.size"; Review comment: I think I forget remove the ipc of this config in https://issues.apache.org/jira/browse/HBASE-22598 before. So remove here. This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
[jira] [Commented] (HBASE-19146) Hbase3.0 protobuf-maven-plugin do not support Arm64(only for x86)
[
https://issues.apache.org/jira/browse/HBASE-19146?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16866391#comment-16866391
]
RuiChen commented on HBASE-19146:
-
I face the same issue when I compile HBase on aarch64, and found there is no
any updates for this opened issue for long time.
Whether we can apply
[com.github.os72/protoc/2.6.1-build3|https://search.maven.org/artifact/com.github.os72/protoc/2.6.1-build3/pom]
in hbase side as option, this protoc have supported aarch64
com.github.os72
protoc
2.6.1-build3
pom
> Hbase3.0 protobuf-maven-plugin do not support Arm64(only for x86)
> --
>
> Key: HBASE-19146
> URL: https://issues.apache.org/jira/browse/HBASE-19146
> Project: HBase
> Issue Type: Bug
> Components: build, pom
>Affects Versions: 3.0.0
> Environment: OS: Ubuntu 16.04.3
> OpenJDK Runtime Environment (build 1.8.0_131-8u131-b11-2ubuntu1.16.04.3-b11)
> Hw platform: AARCH64
>Reporter: Yuqi Gu
>Priority: Major
>
> We are building the HBase 3.0.0-SNAPSHOT on AARCH64.
> It is noted that 'protobuf-maven-plugin' only support x86 shown as follows:
> {code:java}
>
>org.xolstice.maven.plugins
>protobuf-maven-plugin
>${protobuf.plugin.version}
>
> com.google.protobuf:protoc:${external.protobuf.version}:
> exe:${os.detected.classifier}
>
> com.google.protobuf:protoc:${external.protobuf.version}:exe:${os.detected.classifier}
>
>false
>true
>
>
> {code}
> So the build is failed.
> {code:java}
> [INFO] --- protobuf-maven-plugin:0.5.0:compile (compile-protoc) @
> hbase-protocol-shaded ---
> [INFO] Compiling 32 proto file(s) to
> /root/hbase/hbase-protocol-shaded/target/generated-sources/protobuf/java
> Failed to execute goal
> org.xolstice.maven.plugins:protobuf-maven-plugin:0.5.0:compile
> (compile-protoc) on project hbase-protocol-shaded: Missing:
> {code}
> Then I installed aarch64 protobuf 2.5.0 on the host and modify the pom:
> {code:java}
> -
> com.google.protobuf:protoc:${external.protobuf.version}:exe:${os.detected.classifier}
> + /usr/local/bin/protoc
> {code}
> The build is also failed:
> {code:java}
> [INFO] Compiling 32 proto file(s) to
> /root/hbase/hbase-protocol-shaded/target/generated-sources/protobuf/java
> [ERROR] PROTOC FAILED: google/protobuf/any.proto:31:10: Unrecognized syntax
> identifier "proto3". This parser only recognizes "proto2".
> {code}
> It seems that "internal.protobuf.version" in "hbase-protocol-shaded" is 3.3.0.
> How to fix it? Thanks!
>
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
[GitHub] [hbase] Apache-HBase commented on issue #163: HBASE-21995 Add a coprocessor to set HDFS ACL for hbase granted user
Apache-HBase commented on issue #163: HBASE-21995 Add a coprocessor to set HDFS ACL for hbase granted user URL: https://github.com/apache/hbase/pull/163#issuecomment-503024482 :broken_heart: **-1 overall** | Vote | Subsystem | Runtime | Comment | |::|--:|:|:| | 0 | reexec | 50 | Docker mode activated. | ||| _ Prechecks _ | | +1 | hbaseanti | 0 | Patch does not have any anti-patterns. | | +1 | @author | 0 | The patch does not contain any @author tags. | | +1 | test4tests | 0 | The patch appears to include 1 new or modified test files. | ||| _ master Compile Tests _ | | +1 | mvninstall | 272 | master passed | | +1 | compile | 56 | master passed | | +1 | checkstyle | 72 | master passed | | +1 | shadedjars | 282 | branch has no errors when building our shaded downstream artifacts. | | +1 | findbugs | 219 | master passed | | +1 | javadoc | 36 | master passed | ||| _ Patch Compile Tests _ | | +1 | mvninstall | 264 | the patch passed | | +1 | compile | 60 | the patch passed | | +1 | javac | 60 | the patch passed | | -1 | checkstyle | 68 | hbase-server: The patch generated 2 new + 51 unchanged - 2 fixed = 53 total (was 53) | | +1 | whitespace | 0 | The patch has no whitespace issues. | | +1 | shadedjars | 274 | patch has no errors when building our shaded downstream artifacts. | | +1 | hadoopcheck | 819 | Patch does not cause any errors with Hadoop 2.8.5 2.9.2 or 3.1.2. | | +1 | findbugs | 228 | the patch passed | | +1 | javadoc | 38 | the patch passed | ||| _ Other Tests _ | | -1 | unit | 17009 | hbase-server in the patch failed. | | +1 | asflicense | 22 | The patch does not generate ASF License warnings. | | | | 20129 | | | Reason | Tests | |---:|:--| | Failed junit tests | hadoop.hbase.regionserver.TestSplitTransactionOnCluster | | | hadoop.hbase.client.TestMobSnapshotCloneIndependence | | | hadoop.hbase.util.TestFromClientSide3WoUnsafe | | Subsystem | Report/Notes | |--:|:-| | Docker | Client=17.05.0-ce Server=17.05.0-ce base: https://builds.apache.org/job/HBase-PreCommit-GitHub-PR/job/PR-163/14/artifact/out/Dockerfile | | GITHUB PR | https://github.com/apache/hbase/pull/163 | | Optional Tests | dupname asflicense javac javadoc unit findbugs shadedjars hadoopcheck hbaseanti checkstyle compile | | uname | Linux 08362ce213a6 4.4.0-138-generic #164-Ubuntu SMP Tue Oct 2 17:16:02 UTC 2018 x86_64 GNU/Linux | | Build tool | maven | | Personality | /testptch/patchprocess/precommit/personality/provided.sh | | git revision | master / 5da5deb59d | | maven | version: Apache Maven 3.5.4 (1edded0938998edf8bf061f1ceb3cfdeccf443fe; 2018-06-17T18:33:14Z) | | Default Java | 1.8.0_181 | | findbugs | v3.1.11 | | checkstyle | https://builds.apache.org/job/HBase-PreCommit-GitHub-PR/job/PR-163/14/artifact/out/diff-checkstyle-hbase-server.txt | | unit | https://builds.apache.org/job/HBase-PreCommit-GitHub-PR/job/PR-163/14/artifact/out/patch-unit-hbase-server.txt | | Test Results | https://builds.apache.org/job/HBase-PreCommit-GitHub-PR/job/PR-163/14/testReport/ | | Max. process+thread count | 5031 (vs. ulimit of 1) | | modules | C: hbase-server U: hbase-server | | Console output | https://builds.apache.org/job/HBase-PreCommit-GitHub-PR/job/PR-163/14/console | | Powered by | Apache Yetus 0.9.0 http://yetus.apache.org | This message was automatically generated. This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
