[hbase] branch branch-2 updated: HBASE-19352 Port HADOOP-10379: Protect authentication cookies with the HttpOnly and Secure flags
This is an automated email from the ASF dual-hosted git repository.
esteban pushed a commit to branch branch-2
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2 by this push:
new 800a4d9 HBASE-19352 Port HADOOP-10379: Protect authentication cookies
with the HttpOnly and Secure flags
800a4d9 is described below
commit 800a4d9868a0f0eec4ca203e8f0f4d12a54f9d8e
Author: Esteban Gutierrez
AuthorDate: Thu Sep 3 13:20:44 2020 -0500
HBASE-19352 Port HADOOP-10379: Protect authentication cookies with the
HttpOnly and Secure flags
Signed-off-by: Sean Busbey
---
.../org/apache/hadoop/hbase/http/HttpServer.java | 2 +
.../hadoop/hbase/http/TestHttpCookieFlag.java | 191 +
2 files changed, 193 insertions(+)
diff --git
a/hbase-http/src/main/java/org/apache/hadoop/hbase/http/HttpServer.java
b/hbase-http/src/main/java/org/apache/hadoop/hbase/http/HttpServer.java
index 50a6fe5..8a47ca9 100644
--- a/hbase-http/src/main/java/org/apache/hadoop/hbase/http/HttpServer.java
+++ b/hbase-http/src/main/java/org/apache/hadoop/hbase/http/HttpServer.java
@@ -857,6 +857,8 @@ public class HttpServer implements FilterContainer {
fmap.setFilterName(AdminAuthorizedFilter.class.getSimpleName());
webAppContext.getServletHandler().addFilter(filter, fmap);
}
+
webAppContext.getSessionHandler().getSessionCookieConfig().setHttpOnly(true);
+webAppContext.getSessionHandler().getSessionCookieConfig().setSecure(true);
webAppContext.addServlet(holder, pathSpec);
}
diff --git
a/hbase-http/src/test/java/org/apache/hadoop/hbase/http/TestHttpCookieFlag.java
b/hbase-http/src/test/java/org/apache/hadoop/hbase/http/TestHttpCookieFlag.java
new file mode 100644
index 000..d373d60
--- /dev/null
+++
b/hbase-http/src/test/java/org/apache/hadoop/hbase/http/TestHttpCookieFlag.java
@@ -0,0 +1,191 @@
+/**
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License. See accompanying LICENSE file.
+ */
+package org.apache.hadoop.hbase.http;
+
+import java.util.List;
+import java.io.File;
+import java.io.IOException;
+import java.net.HttpURLConnection;
+import java.net.HttpCookie;
+import java.net.URI;
+import java.net.URL;
+import javax.net.ssl.HttpsURLConnection;
+import javax.servlet.Filter;
+import javax.servlet.FilterConfig;
+import javax.servlet.FilterChain;
+import javax.servlet.ServletRequest;
+import javax.servlet.ServletResponse;
+import javax.servlet.ServletException;
+import javax.servlet.http.HttpServletResponse;
+import java.security.GeneralSecurityException;
+import org.apache.hadoop.hbase.HBaseClassTestRule;
+import org.apache.hadoop.hbase.testclassification.MiscTests;
+import org.apache.hadoop.hbase.testclassification.SmallTests;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.security.authentication.server.AuthenticationFilter;
+import org.apache.hadoop.security.ssl.KeyStoreTestUtil;
+import org.apache.hadoop.security.ssl.SSLFactory;
+
+import org.junit.Assert;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.ClassRule;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+@Category({ MiscTests.class, SmallTests.class})
+public class TestHttpCookieFlag {
+ @ClassRule
+ public static final HBaseClassTestRule CLASS_RULE =
+ HBaseClassTestRule.forClass(TestHttpCookieFlag.class);
+
+ private static final String BASEDIR = System.getProperty("test.build.dir",
+ "target/test-dir") + "/" +
+ org.apache.hadoop.hbase.http.TestHttpCookieFlag.class.getSimpleName();
+ private static String keystoresDir;
+ private static String sslConfDir;
+ private static SSLFactory clientSslFactory;
+ private static HttpServer server;
+
+ public static class DummyAuthenticationFilter implements Filter {
+
+@Override
+public void init(FilterConfig filterConfig) throws ServletException {
+}
+
+@Override
+public void doFilter(ServletRequest request, ServletResponse response,
+ FilterChain chain) throws IOException,
+ ServletException {
+ HttpServletResponse resp = (HttpServletResponse) response;
+ boolean isHttps = "https".equals(request.getScheme());
+ Authent
[hbase] branch branch-2.3 updated: HBASE-19352 Port HADOOP-10379: Protect authentication cookies with the HttpOnly and Secure flags
This is an automated email from the ASF dual-hosted git repository.
esteban pushed a commit to branch branch-2.3
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2.3 by this push:
new b9fcd1c HBASE-19352 Port HADOOP-10379: Protect authentication cookies
with the HttpOnly and Secure flags
b9fcd1c is described below
commit b9fcd1c4f2b4b1bc959718d631e83c87f4d5deb2
Author: Esteban Gutierrez
AuthorDate: Thu Sep 3 13:20:44 2020 -0500
HBASE-19352 Port HADOOP-10379: Protect authentication cookies with the
HttpOnly and Secure flags
Signed-off-by: Sean Busbey
---
.../org/apache/hadoop/hbase/http/HttpServer.java | 4 +
.../hadoop/hbase/http/TestHttpCookieFlag.java | 191 +
2 files changed, 195 insertions(+)
diff --git
a/hbase-http/src/main/java/org/apache/hadoop/hbase/http/HttpServer.java
b/hbase-http/src/main/java/org/apache/hadoop/hbase/http/HttpServer.java
index d3176ca..f3c4a59 100644
--- a/hbase-http/src/main/java/org/apache/hadoop/hbase/http/HttpServer.java
+++ b/hbase-http/src/main/java/org/apache/hadoop/hbase/http/HttpServer.java
@@ -858,6 +858,10 @@ public class HttpServer implements FilterContainer {
fmap.setFilterName(AdminAuthorizedFilter.class.getSimpleName());
webAppContext.getServletHandler().addFilter(filter, fmap);
}
+webAppContext.getSessionHandler().getSessionManager().
+getSessionCookieConfig().setHttpOnly(true);
+webAppContext.getSessionHandler().getSessionManager().
+getSessionCookieConfig().setSecure(true);
webAppContext.addServlet(holder, pathSpec);
}
diff --git
a/hbase-http/src/test/java/org/apache/hadoop/hbase/http/TestHttpCookieFlag.java
b/hbase-http/src/test/java/org/apache/hadoop/hbase/http/TestHttpCookieFlag.java
new file mode 100644
index 000..d373d60
--- /dev/null
+++
b/hbase-http/src/test/java/org/apache/hadoop/hbase/http/TestHttpCookieFlag.java
@@ -0,0 +1,191 @@
+/**
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License. See accompanying LICENSE file.
+ */
+package org.apache.hadoop.hbase.http;
+
+import java.util.List;
+import java.io.File;
+import java.io.IOException;
+import java.net.HttpURLConnection;
+import java.net.HttpCookie;
+import java.net.URI;
+import java.net.URL;
+import javax.net.ssl.HttpsURLConnection;
+import javax.servlet.Filter;
+import javax.servlet.FilterConfig;
+import javax.servlet.FilterChain;
+import javax.servlet.ServletRequest;
+import javax.servlet.ServletResponse;
+import javax.servlet.ServletException;
+import javax.servlet.http.HttpServletResponse;
+import java.security.GeneralSecurityException;
+import org.apache.hadoop.hbase.HBaseClassTestRule;
+import org.apache.hadoop.hbase.testclassification.MiscTests;
+import org.apache.hadoop.hbase.testclassification.SmallTests;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.security.authentication.server.AuthenticationFilter;
+import org.apache.hadoop.security.ssl.KeyStoreTestUtil;
+import org.apache.hadoop.security.ssl.SSLFactory;
+
+import org.junit.Assert;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.ClassRule;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+@Category({ MiscTests.class, SmallTests.class})
+public class TestHttpCookieFlag {
+ @ClassRule
+ public static final HBaseClassTestRule CLASS_RULE =
+ HBaseClassTestRule.forClass(TestHttpCookieFlag.class);
+
+ private static final String BASEDIR = System.getProperty("test.build.dir",
+ "target/test-dir") + "/" +
+ org.apache.hadoop.hbase.http.TestHttpCookieFlag.class.getSimpleName();
+ private static String keystoresDir;
+ private static String sslConfDir;
+ private static SSLFactory clientSslFactory;
+ private static HttpServer server;
+
+ public static class DummyAuthenticationFilter implements Filter {
+
+@Override
+public void init(FilterConfig filterConfig) throws ServletException {
+}
+
+@Override
+public void doFilter(ServletRequest request, ServletResponse response,
+ FilterChain chain) throws IOException,
+ ServletException {
+ HttpServletResponse resp = (HttpServletResponse) response;
+
[hbase] branch branch-2.2 updated: HBASE-19352 Port HADOOP-10379: Protect authentication cookies with the HttpOnly and Secure flags
This is an automated email from the ASF dual-hosted git repository.
esteban pushed a commit to branch branch-2.2
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2.2 by this push:
new d234f1f HBASE-19352 Port HADOOP-10379: Protect authentication cookies
with the HttpOnly and Secure flags
d234f1f is described below
commit d234f1fbcc7a34bef57198742d0fc63e5ebf18c8
Author: Esteban Gutierrez
AuthorDate: Thu Sep 3 13:20:44 2020 -0500
HBASE-19352 Port HADOOP-10379: Protect authentication cookies with the
HttpOnly and Secure flags
Signed-off-by: Sean Busbey
---
.../org/apache/hadoop/hbase/http/HttpServer.java | 4 +
.../hadoop/hbase/http/TestHttpCookieFlag.java | 191 +
2 files changed, 195 insertions(+)
diff --git
a/hbase-http/src/main/java/org/apache/hadoop/hbase/http/HttpServer.java
b/hbase-http/src/main/java/org/apache/hadoop/hbase/http/HttpServer.java
index 788fcfb..1e4adee 100644
--- a/hbase-http/src/main/java/org/apache/hadoop/hbase/http/HttpServer.java
+++ b/hbase-http/src/main/java/org/apache/hadoop/hbase/http/HttpServer.java
@@ -823,6 +823,10 @@ public class HttpServer implements FilterContainer {
fmap.setFilterName(AdminAuthorizedFilter.class.getSimpleName());
webAppContext.getServletHandler().addFilter(filter, fmap);
}
+webAppContext.getSessionHandler().getSessionManager().
+getSessionCookieConfig().setHttpOnly(true);
+webAppContext.getSessionHandler().getSessionManager().
+getSessionCookieConfig().setSecure(true);
webAppContext.addServlet(holder, pathSpec);
}
diff --git
a/hbase-http/src/test/java/org/apache/hadoop/hbase/http/TestHttpCookieFlag.java
b/hbase-http/src/test/java/org/apache/hadoop/hbase/http/TestHttpCookieFlag.java
new file mode 100644
index 000..d373d60
--- /dev/null
+++
b/hbase-http/src/test/java/org/apache/hadoop/hbase/http/TestHttpCookieFlag.java
@@ -0,0 +1,191 @@
+/**
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License. See accompanying LICENSE file.
+ */
+package org.apache.hadoop.hbase.http;
+
+import java.util.List;
+import java.io.File;
+import java.io.IOException;
+import java.net.HttpURLConnection;
+import java.net.HttpCookie;
+import java.net.URI;
+import java.net.URL;
+import javax.net.ssl.HttpsURLConnection;
+import javax.servlet.Filter;
+import javax.servlet.FilterConfig;
+import javax.servlet.FilterChain;
+import javax.servlet.ServletRequest;
+import javax.servlet.ServletResponse;
+import javax.servlet.ServletException;
+import javax.servlet.http.HttpServletResponse;
+import java.security.GeneralSecurityException;
+import org.apache.hadoop.hbase.HBaseClassTestRule;
+import org.apache.hadoop.hbase.testclassification.MiscTests;
+import org.apache.hadoop.hbase.testclassification.SmallTests;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.security.authentication.server.AuthenticationFilter;
+import org.apache.hadoop.security.ssl.KeyStoreTestUtil;
+import org.apache.hadoop.security.ssl.SSLFactory;
+
+import org.junit.Assert;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.ClassRule;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+@Category({ MiscTests.class, SmallTests.class})
+public class TestHttpCookieFlag {
+ @ClassRule
+ public static final HBaseClassTestRule CLASS_RULE =
+ HBaseClassTestRule.forClass(TestHttpCookieFlag.class);
+
+ private static final String BASEDIR = System.getProperty("test.build.dir",
+ "target/test-dir") + "/" +
+ org.apache.hadoop.hbase.http.TestHttpCookieFlag.class.getSimpleName();
+ private static String keystoresDir;
+ private static String sslConfDir;
+ private static SSLFactory clientSslFactory;
+ private static HttpServer server;
+
+ public static class DummyAuthenticationFilter implements Filter {
+
+@Override
+public void init(FilterConfig filterConfig) throws ServletException {
+}
+
+@Override
+public void doFilter(ServletRequest request, ServletResponse response,
+ FilterChain chain) throws IOException,
+ ServletException {
+ HttpServletResponse resp = (HttpServletResponse) response;
+
[hbase] branch revert-2348-HBASE-19352 created (now 1c1a7be)
This is an automated email from the ASF dual-hosted git repository. esteban pushed a change to branch revert-2348-HBASE-19352 in repository https://gitbox.apache.org/repos/asf/hbase.git. at 1c1a7be Revert "HBASE-19352 Port HADOOP-10379: Protect authentication cookies with the HttpOnly and Secure flags (#2348)" This branch includes the following new commits: new 1c1a7be Revert "HBASE-19352 Port HADOOP-10379: Protect authentication cookies with the HttpOnly and Secure flags (#2348)" The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference.
[hbase] 01/01: Revert " HBASE-19352 Port HADOOP-10379: Protect authentication cookies with the HttpOnly and Secure flags (#2348)"
This is an automated email from the ASF dual-hosted git repository.
esteban pushed a commit to branch revert-2348-HBASE-19352
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit 1c1a7be1a63525468a20180a0bb554abb0c8be3f
Author: Esteban Gutierrez
AuthorDate: Thu Sep 3 13:38:45 2020 -0500
Revert "HBASE-19352 Port HADOOP-10379: Protect authentication cookies
with the HttpOnly and Secure flags (#2348)"
This reverts commit 19b8a2a64a63e9e546af3497871b5346ea5b6b5b.
---
.../org/apache/hadoop/hbase/http/HttpServer.java | 2 -
.../hadoop/hbase/http/TestHttpCookieFlag.java | 191 -
2 files changed, 193 deletions(-)
diff --git
a/hbase-http/src/main/java/org/apache/hadoop/hbase/http/HttpServer.java
b/hbase-http/src/main/java/org/apache/hadoop/hbase/http/HttpServer.java
index 8a47ca9..50a6fe5 100644
--- a/hbase-http/src/main/java/org/apache/hadoop/hbase/http/HttpServer.java
+++ b/hbase-http/src/main/java/org/apache/hadoop/hbase/http/HttpServer.java
@@ -857,8 +857,6 @@ public class HttpServer implements FilterContainer {
fmap.setFilterName(AdminAuthorizedFilter.class.getSimpleName());
webAppContext.getServletHandler().addFilter(filter, fmap);
}
-
webAppContext.getSessionHandler().getSessionCookieConfig().setHttpOnly(true);
-webAppContext.getSessionHandler().getSessionCookieConfig().setSecure(true);
webAppContext.addServlet(holder, pathSpec);
}
diff --git
a/hbase-http/src/test/java/org/apache/hadoop/hbase/http/TestHttpCookieFlag.java
b/hbase-http/src/test/java/org/apache/hadoop/hbase/http/TestHttpCookieFlag.java
deleted file mode 100644
index d373d60..000
---
a/hbase-http/src/test/java/org/apache/hadoop/hbase/http/TestHttpCookieFlag.java
+++ /dev/null
@@ -1,191 +0,0 @@
-/**
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License. See accompanying LICENSE file.
- */
-package org.apache.hadoop.hbase.http;
-
-import java.util.List;
-import java.io.File;
-import java.io.IOException;
-import java.net.HttpURLConnection;
-import java.net.HttpCookie;
-import java.net.URI;
-import java.net.URL;
-import javax.net.ssl.HttpsURLConnection;
-import javax.servlet.Filter;
-import javax.servlet.FilterConfig;
-import javax.servlet.FilterChain;
-import javax.servlet.ServletRequest;
-import javax.servlet.ServletResponse;
-import javax.servlet.ServletException;
-import javax.servlet.http.HttpServletResponse;
-import java.security.GeneralSecurityException;
-import org.apache.hadoop.hbase.HBaseClassTestRule;
-import org.apache.hadoop.hbase.testclassification.MiscTests;
-import org.apache.hadoop.hbase.testclassification.SmallTests;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileUtil;
-import org.apache.hadoop.net.NetUtils;
-import org.apache.hadoop.security.authentication.server.AuthenticationFilter;
-import org.apache.hadoop.security.ssl.KeyStoreTestUtil;
-import org.apache.hadoop.security.ssl.SSLFactory;
-
-import org.junit.Assert;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-import org.junit.ClassRule;
-import org.junit.Test;
-import org.junit.experimental.categories.Category;
-
-@Category({ MiscTests.class, SmallTests.class})
-public class TestHttpCookieFlag {
- @ClassRule
- public static final HBaseClassTestRule CLASS_RULE =
- HBaseClassTestRule.forClass(TestHttpCookieFlag.class);
-
- private static final String BASEDIR = System.getProperty("test.build.dir",
- "target/test-dir") + "/" +
- org.apache.hadoop.hbase.http.TestHttpCookieFlag.class.getSimpleName();
- private static String keystoresDir;
- private static String sslConfDir;
- private static SSLFactory clientSslFactory;
- private static HttpServer server;
-
- public static class DummyAuthenticationFilter implements Filter {
-
-@Override
-public void init(FilterConfig filterConfig) throws ServletException {
-}
-
-@Override
-public void doFilter(ServletRequest request, ServletResponse response,
- FilterChain chain) throws IOException,
- ServletException {
- HttpServletResponse resp = (HttpServletResponse) response;
- boolean isHttps = "https".equals(request.getScheme());
- AuthenticationFilter.createAuthCookie(resp, "token", null, null, -1,
- true, isHttps);
- chain.doFilter(request, resp);
-}
[hbase] branch master updated (a352706 -> 19b8a2a)
This is an automated email from the ASF dual-hosted git repository. esteban pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/hbase.git. from a352706 HBASE-24940: runCatalogJanitor() API should return -1 to indicate already running status add 19b8a2a HBASE-19352 Port HADOOP-10379: Protect authentication cookies with the HttpOnly and Secure flags (#2348) No new revisions were added by this update. Summary of changes: .../org/apache/hadoop/hbase/http/HttpServer.java | 2 + .../hadoop/hbase/http/TestHttpCookieFlag.java | 191 + 2 files changed, 193 insertions(+) create mode 100644 hbase-http/src/test/java/org/apache/hadoop/hbase/http/TestHttpCookieFlag.java
[hbase] branch branch-2.3 updated: HBASE-24041 [regression] Increase RESTServer buffer size back to 64k (#1339)
This is an automated email from the ASF dual-hosted git repository.
esteban pushed a commit to branch branch-2.3
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2.3 by this push:
new 11bc006 HBASE-24041 [regression] Increase RESTServer buffer size back
to 64k (#1339)
11bc006 is described below
commit 11bc006ff3567228a3a3790ae7c660e09e3a0a78
Author: Esteban Gutierrez
AuthorDate: Tue Mar 24 15:12:02 2020 -0500
HBASE-24041 [regression] Increase RESTServer buffer size back to 64k (#1339)
* HBASE-24041 [regression] Increase RESTServer buffer size back to 64k
* HBASE-24041 [regression] Increase RESTServer buffer size back to 64k
---
hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/RESTServer.java | 4
1 file changed, 4 insertions(+)
diff --git
a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/RESTServer.java
b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/RESTServer.java
index c3d16ee..3411ef2 100644
--- a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/RESTServer.java
+++ b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/RESTServer.java
@@ -90,6 +90,7 @@ public class RESTServer implements Constants {
static final String REST_CSRF_METHODS_TO_IGNORE_KEY =
"hbase.rest.csrf.methods.to.ignore";
static final String REST_CSRF_METHODS_TO_IGNORE_DEFAULT =
"GET,OPTIONS,HEAD,TRACE";
public static final String SKIP_LOGIN_KEY = "hbase.rest.skip.login";
+ static final int DEFAULT_HTTP_MAX_HEADER_SIZE = 64 * 1024; // 64k
private static final String PATH_SPEC_ANY = "/*";
@@ -293,6 +294,9 @@ public class RESTServer implements Constants {
HttpConfiguration httpConfig = new HttpConfiguration();
httpConfig.setSecureScheme("https");
httpConfig.setSecurePort(servicePort);
+httpConfig.setHeaderCacheSize(DEFAULT_HTTP_MAX_HEADER_SIZE);
+httpConfig.setRequestHeaderSize(DEFAULT_HTTP_MAX_HEADER_SIZE);
+httpConfig.setResponseHeaderSize(DEFAULT_HTTP_MAX_HEADER_SIZE);
httpConfig.setSendServerVersion(false);
httpConfig.setSendDateHeader(false);
[hbase] branch branch-2 updated: HBASE-24041 [regression] Increase RESTServer buffer size back to 64k (#1339)
This is an automated email from the ASF dual-hosted git repository.
esteban pushed a commit to branch branch-2
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2 by this push:
new a867464 HBASE-24041 [regression] Increase RESTServer buffer size back
to 64k (#1339)
a867464 is described below
commit a867464f5a473317fbc6ef2f6e91c5aea25f9c94
Author: Esteban Gutierrez
AuthorDate: Tue Mar 24 15:12:02 2020 -0500
HBASE-24041 [regression] Increase RESTServer buffer size back to 64k (#1339)
* HBASE-24041 [regression] Increase RESTServer buffer size back to 64k
* HBASE-24041 [regression] Increase RESTServer buffer size back to 64k
---
hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/RESTServer.java | 4
1 file changed, 4 insertions(+)
diff --git
a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/RESTServer.java
b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/RESTServer.java
index c3d16ee..3411ef2 100644
--- a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/RESTServer.java
+++ b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/RESTServer.java
@@ -90,6 +90,7 @@ public class RESTServer implements Constants {
static final String REST_CSRF_METHODS_TO_IGNORE_KEY =
"hbase.rest.csrf.methods.to.ignore";
static final String REST_CSRF_METHODS_TO_IGNORE_DEFAULT =
"GET,OPTIONS,HEAD,TRACE";
public static final String SKIP_LOGIN_KEY = "hbase.rest.skip.login";
+ static final int DEFAULT_HTTP_MAX_HEADER_SIZE = 64 * 1024; // 64k
private static final String PATH_SPEC_ANY = "/*";
@@ -293,6 +294,9 @@ public class RESTServer implements Constants {
HttpConfiguration httpConfig = new HttpConfiguration();
httpConfig.setSecureScheme("https");
httpConfig.setSecurePort(servicePort);
+httpConfig.setHeaderCacheSize(DEFAULT_HTTP_MAX_HEADER_SIZE);
+httpConfig.setRequestHeaderSize(DEFAULT_HTTP_MAX_HEADER_SIZE);
+httpConfig.setResponseHeaderSize(DEFAULT_HTTP_MAX_HEADER_SIZE);
httpConfig.setSendServerVersion(false);
httpConfig.setSendDateHeader(false);
[hbase] branch branch-2.2 updated: HBASE-24041 [regression] Increase RESTServer buffer size back to 64k (#1339)
This is an automated email from the ASF dual-hosted git repository.
esteban pushed a commit to branch branch-2.2
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2.2 by this push:
new 1777513 HBASE-24041 [regression] Increase RESTServer buffer size back
to 64k (#1339)
1777513 is described below
commit 177751345526fd55f8a2ee20f02154441909d06d
Author: Esteban Gutierrez
AuthorDate: Tue Mar 24 15:12:02 2020 -0500
HBASE-24041 [regression] Increase RESTServer buffer size back to 64k (#1339)
* HBASE-24041 [regression] Increase RESTServer buffer size back to 64k
* HBASE-24041 [regression] Increase RESTServer buffer size back to 64k
---
hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/RESTServer.java | 4
1 file changed, 4 insertions(+)
diff --git
a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/RESTServer.java
b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/RESTServer.java
index c3d16ee..3411ef2 100644
--- a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/RESTServer.java
+++ b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/RESTServer.java
@@ -90,6 +90,7 @@ public class RESTServer implements Constants {
static final String REST_CSRF_METHODS_TO_IGNORE_KEY =
"hbase.rest.csrf.methods.to.ignore";
static final String REST_CSRF_METHODS_TO_IGNORE_DEFAULT =
"GET,OPTIONS,HEAD,TRACE";
public static final String SKIP_LOGIN_KEY = "hbase.rest.skip.login";
+ static final int DEFAULT_HTTP_MAX_HEADER_SIZE = 64 * 1024; // 64k
private static final String PATH_SPEC_ANY = "/*";
@@ -293,6 +294,9 @@ public class RESTServer implements Constants {
HttpConfiguration httpConfig = new HttpConfiguration();
httpConfig.setSecureScheme("https");
httpConfig.setSecurePort(servicePort);
+httpConfig.setHeaderCacheSize(DEFAULT_HTTP_MAX_HEADER_SIZE);
+httpConfig.setRequestHeaderSize(DEFAULT_HTTP_MAX_HEADER_SIZE);
+httpConfig.setResponseHeaderSize(DEFAULT_HTTP_MAX_HEADER_SIZE);
httpConfig.setSendServerVersion(false);
httpConfig.setSendDateHeader(false);
[hbase] branch master updated (aa53493 -> 9a212ee)
This is an automated email from the ASF dual-hosted git repository. esteban pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/hbase.git. from aa53493 HBASE-24002 shadedjars check does not propagate --hadoop-profile add 9a212ee HBASE-24041 [regression] Increase RESTServer buffer size back to 64k (#1339) No new revisions were added by this update. Summary of changes: hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/RESTServer.java | 4 1 file changed, 4 insertions(+)
hbase git commit: HBASE-18025 CatalogJanitor should collect outdated RegionStates from the AM
Repository: hbase
Updated Branches:
refs/heads/branch-2 5940f4224 -> b627cfad3
HBASE-18025 CatalogJanitor should collect outdated RegionStates from the AM
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/b627cfad
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/b627cfad
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/b627cfad
Branch: refs/heads/branch-2
Commit: b627cfad35bb7d925506d043f62ff69b0d57869d
Parents: 5940f42
Author: Esteban Gutierrez <este...@apache.org>
Authored: Fri Jul 21 14:13:13 2017 -0500
Committer: Esteban Gutierrez <este...@apache.org>
Committed: Fri Aug 11 14:42:22 2017 -0500
--
.../hadoop/hbase/master/CatalogJanitor.java | 13 +-
.../hadoop/hbase/master/ServerManager.java | 7 +
.../hbase/master/assignment/RegionStates.java | 6 +
.../TestCatalogJanitorInMemoryStates.java | 185 +++
4 files changed, 209 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/b627cfad/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java
index ba92c76..8daa7db 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java
@@ -221,6 +221,11 @@ public class CatalogJanitor extends ScheduledChore {
ProcedureExecutor pe =
this.services.getMasterProcedureExecutor();
pe.submitProcedure(new GCMergedRegionsProcedure(pe.getEnvironment(),
mergedRegion, regionA, regionB));
+ // Remove from in-memory states
+
this.services.getAssignmentManager().getRegionStates().deleteRegion(regionA);
+
this.services.getAssignmentManager().getRegionStates().deleteRegion(regionB);
+ this.services.getServerManager().removeRegion(regionA);
+ this.services.getServerManager().removeRegion(regionB);
return true;
}
return false;
@@ -234,6 +239,7 @@ public class CatalogJanitor extends ScheduledChore {
*/
int scan() throws IOException {
int result = 0;
+
try {
if (!alreadyRunning.compareAndSet(false, true)) {
LOG.debug("CatalogJanitor already running");
@@ -281,8 +287,8 @@ public class CatalogJanitor extends ScheduledChore {
}
if (!parentNotCleaned.contains(e.getKey().getEncodedName()) &&
- cleanParent(e.getKey(), e.getValue())) {
-result++;
+cleanParent(e.getKey(), e.getValue())) {
+ result++;
} else {
// We could not clean the parent, so it's daughters should not be
// cleaned either (HBASE-6160)
@@ -355,6 +361,9 @@ public class CatalogJanitor extends ScheduledChore {
" -- no longer hold references");
ProcedureExecutor pe =
this.services.getMasterProcedureExecutor();
pe.submitProcedure(new GCRegionProcedure(pe.getEnvironment(), parent));
+ // Remove from in-memory states
+
this.services.getAssignmentManager().getRegionStates().deleteRegion(parent);
+ this.services.getServerManager().removeRegion(parent);
return true;
}
return false;
http://git-wip-us.apache.org/repos/asf/hbase/blob/b627cfad/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
index c9c792a..f0e9b88 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
@@ -1028,6 +1028,13 @@ public class ServerManager {
flushedSequenceIdByRegion.remove(encodedName);
}
+ @VisibleForTesting
+ public boolean isRegionInServerManagerStates(final HRegionInfo hri) {
+final byte[] encodedName = hri.getEncodedNameAsBytes();
+return (storeFlushedSequenceIdsByRegion.containsKey(encodedName)
+|| flushedSequenceIdByRegion.containsKey(encodedName));
+ }
+
/**
* Called by delete table and similar to notify the ServerManager that a
region was removed.
*/
http://git-wip-us.apache.org/repos/asf/hbase/blob/b627cfad/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionStates.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoo
hbase git commit: HBASE-18025 CatalogJanitor should collect outdated RegionStates from the AM
Repository: hbase
Updated Branches:
refs/heads/branch-1 b181f172e -> 578e29f96
HBASE-18025 CatalogJanitor should collect outdated RegionStates from the AM
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/578e29f9
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/578e29f9
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/578e29f9
Branch: refs/heads/branch-1
Commit: 578e29f96b37875cd8092f7c6b3baf2b511148d0
Parents: b181f17
Author: Esteban Gutierrez <este...@apache.org>
Authored: Fri Aug 11 12:56:20 2017 -0500
Committer: Esteban Gutierrez <este...@apache.org>
Committed: Fri Aug 11 14:39:16 2017 -0500
--
.../hadoop/hbase/master/CatalogJanitor.java | 4 +
.../hadoop/hbase/master/RegionStates.java | 6 +
.../hadoop/hbase/master/ServerManager.java | 7 +
.../TestCatalogJanitorInMemoryStates.java | 188 +++
4 files changed, 205 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/578e29f9/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java
index 17644eb..00dc4a5 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java
@@ -217,6 +217,8 @@ public class CatalogJanitor extends ScheduledChore {
HFileArchiver.archiveRegion(this.services.getConfiguration(), fs,
regionA);
HFileArchiver.archiveRegion(this.services.getConfiguration(), fs,
regionB);
MetaTableAccessor.deleteMergeQualifiers(services.getConnection(),
mergedRegion);
+ services.getAssignmentManager().getRegionStates().deleteRegion(regionA);
+ services.getAssignmentManager().getRegionStates().deleteRegion(regionB);
services.getServerManager().removeRegion(regionA);
services.getServerManager().removeRegion(regionB);
return true;
@@ -361,6 +363,8 @@ public class CatalogJanitor extends ScheduledChore {
if (LOG.isTraceEnabled()) LOG.trace("Archiving parent region: " +
parent);
HFileArchiver.archiveRegion(this.services.getConfiguration(), fs,
parent);
MetaTableAccessor.deleteRegion(this.connection, parent);
+ if (services.getAssignmentManager().getRegionStates() != null)
+services.getAssignmentManager().getRegionStates().deleteRegion(parent);
services.getServerManager().removeRegion(parent);
result = true;
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/578e29f9/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStates.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStates.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStates.java
index 082b5cc..599e649 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStates.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStates.java
@@ -899,6 +899,12 @@ public class RegionStates {
}
}
+ @VisibleForTesting
+ public boolean isRegionInRegionStates(final HRegionInfo hri) {
+return (getRegionState(hri) != null || isRegionOnline(hri)) ||
isRegionInTransition(hri)
+|| isRegionInState(hri, State.OFFLINE, State.CLOSED);
+ }
+
/**
* Checking if a region was assigned to a server which is not online now.
* If so, we should hold re-assign this region till SSH has split its wals.
http://git-wip-us.apache.org/repos/asf/hbase/blob/578e29f9/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
index 93e532b..040342f 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
@@ -1311,6 +1311,13 @@ public class ServerManager {
flushedSequenceIdByRegion.remove(encodedName);
}
+ @VisibleForTesting
+ public boolean isRegionInServerManagerStates(final HRegionInfo hri) {
+final byte[] encodedName = hri.getEncodedNameAsBytes();
+return (storeFlushedSequenceIdsByRegion.containsKey(encodedName)
+|| flushedSequenceIdByRegion.containsKey(encodedName));
+ }
+
/**
* Called by delete table and similar to notify the Ser
hbase git commit: HBASE-18025 CatalogJanitor should collect outdated RegionStates from the AM
Repository: hbase
Updated Branches:
refs/heads/master 043ec9b37 -> 71a9a9a94
HBASE-18025 CatalogJanitor should collect outdated RegionStates from the AM
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/71a9a9a9
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/71a9a9a9
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/71a9a9a9
Branch: refs/heads/master
Commit: 71a9a9a9440c9f2e2e9dd301dd372197e38e70c5
Parents: 043ec9b
Author: Esteban Gutierrez <este...@apache.org>
Authored: Fri Jul 21 14:13:13 2017 -0500
Committer: Esteban Gutierrez <este...@apache.org>
Committed: Fri Aug 11 13:36:38 2017 -0500
--
.../hadoop/hbase/master/CatalogJanitor.java | 13 +-
.../hadoop/hbase/master/ServerManager.java | 7 +
.../hbase/master/assignment/RegionStates.java | 6 +
.../TestCatalogJanitorInMemoryStates.java | 185 +++
4 files changed, 209 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/71a9a9a9/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java
index ba92c76..8daa7db 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java
@@ -221,6 +221,11 @@ public class CatalogJanitor extends ScheduledChore {
ProcedureExecutor pe =
this.services.getMasterProcedureExecutor();
pe.submitProcedure(new GCMergedRegionsProcedure(pe.getEnvironment(),
mergedRegion, regionA, regionB));
+ // Remove from in-memory states
+
this.services.getAssignmentManager().getRegionStates().deleteRegion(regionA);
+
this.services.getAssignmentManager().getRegionStates().deleteRegion(regionB);
+ this.services.getServerManager().removeRegion(regionA);
+ this.services.getServerManager().removeRegion(regionB);
return true;
}
return false;
@@ -234,6 +239,7 @@ public class CatalogJanitor extends ScheduledChore {
*/
int scan() throws IOException {
int result = 0;
+
try {
if (!alreadyRunning.compareAndSet(false, true)) {
LOG.debug("CatalogJanitor already running");
@@ -281,8 +287,8 @@ public class CatalogJanitor extends ScheduledChore {
}
if (!parentNotCleaned.contains(e.getKey().getEncodedName()) &&
- cleanParent(e.getKey(), e.getValue())) {
-result++;
+cleanParent(e.getKey(), e.getValue())) {
+ result++;
} else {
// We could not clean the parent, so it's daughters should not be
// cleaned either (HBASE-6160)
@@ -355,6 +361,9 @@ public class CatalogJanitor extends ScheduledChore {
" -- no longer hold references");
ProcedureExecutor pe =
this.services.getMasterProcedureExecutor();
pe.submitProcedure(new GCRegionProcedure(pe.getEnvironment(), parent));
+ // Remove from in-memory states
+
this.services.getAssignmentManager().getRegionStates().deleteRegion(parent);
+ this.services.getServerManager().removeRegion(parent);
return true;
}
return false;
http://git-wip-us.apache.org/repos/asf/hbase/blob/71a9a9a9/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
index c9c792a..f0e9b88 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
@@ -1028,6 +1028,13 @@ public class ServerManager {
flushedSequenceIdByRegion.remove(encodedName);
}
+ @VisibleForTesting
+ public boolean isRegionInServerManagerStates(final HRegionInfo hri) {
+final byte[] encodedName = hri.getEncodedNameAsBytes();
+return (storeFlushedSequenceIdsByRegion.containsKey(encodedName)
+|| flushedSequenceIdByRegion.containsKey(encodedName));
+ }
+
/**
* Called by delete table and similar to notify the ServerManager that a
region was removed.
*/
http://git-wip-us.apache.org/repos/asf/hbase/blob/71a9a9a9/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionStates.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hb
hbase git commit: HBASE-18563 Fix RAT License complaint about website jenkins scripts
Repository: hbase Updated Branches: refs/heads/master efd211deb -> c37432fef HBASE-18563 Fix RAT License complaint about website jenkins scripts Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/c37432fe Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/c37432fe Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/c37432fe Branch: refs/heads/master Commit: c37432fefbf4d1ff5bf80f5227c986f3bde281a1 Parents: efd211d Author: Esteban Gutierrez <este...@apache.org> Authored: Thu Aug 10 20:08:03 2017 -0500 Committer: Esteban Gutierrez <este...@apache.org> Committed: Thu Aug 10 20:08:03 2017 -0500 -- .../jenkins-scripts/check-website-links.sh | 20 +++- .../jenkins-scripts/generate-hbase-website.sh | 18 ++ 2 files changed, 37 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/c37432fe/dev-support/jenkins-scripts/check-website-links.sh -- diff --git a/dev-support/jenkins-scripts/check-website-links.sh b/dev-support/jenkins-scripts/check-website-links.sh index c23abbb..1f7cd1c 100755 --- a/dev-support/jenkins-scripts/check-website-links.sh +++ b/dev-support/jenkins-scripts/check-website-links.sh @@ -1,4 +1,22 @@ #!/bin/bash +# +#/** +# * Licensed to the Apache Software Foundation (ASF) under one +# * or more contributor license agreements. See the NOTICE file +# * distributed with this work for additional information +# * regarding copyright ownership. The ASF licenses this file +# * to you under the Apache License, Version 2.0 (the +# * "License"); you may not use this file except in compliance +# * with the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# */ # This script is designed to run as a Jenkins job, such as at # https://builds.apache.org/view/All/job/HBase%20Website%20Link%20Checker/ @@ -44,4 +62,4 @@ if ! grep -q 'ERROR' link_report/index.html; then exit 1 else echo "No errors found. Warnings might be present." -fi \ No newline at end of file +fi http://git-wip-us.apache.org/repos/asf/hbase/blob/c37432fe/dev-support/jenkins-scripts/generate-hbase-website.sh -- diff --git a/dev-support/jenkins-scripts/generate-hbase-website.sh b/dev-support/jenkins-scripts/generate-hbase-website.sh index a3f7823..06d160a 100644 --- a/dev-support/jenkins-scripts/generate-hbase-website.sh +++ b/dev-support/jenkins-scripts/generate-hbase-website.sh @@ -1,4 +1,22 @@ #!/bin/bash +# +#/** +# * Licensed to the Apache Software Foundation (ASF) under one +# * or more contributor license agreements. See the NOTICE file +# * distributed with this work for additional information +# * regarding copyright ownership. The ASF licenses this file +# * to you under the Apache License, Version 2.0 (the +# * "License"); you may not use this file except in compliance +# * with the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# */ # This script is meant to run as part of a Jenkins job such as # https://builds.apache.org/job/hbase_generate_website/
hbase git commit: HBASE-18024 HRegion#initializeRegionInternals should not re-create .hregioninfo file when the region directory no longer exists
Repository: hbase
Updated Branches:
refs/heads/branch-1 cab492d34 -> 15bad3c03
HBASE-18024 HRegion#initializeRegionInternals should not re-create .hregioninfo
file when the region directory no longer exists
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/15bad3c0
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/15bad3c0
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/15bad3c0
Branch: refs/heads/branch-1
Commit: 15bad3c0367717e3e79fe5b2ac42ab713fbeddb6
Parents: cab492d
Author: Esteban Gutierrez <este...@apache.org>
Authored: Fri Jul 21 13:13:00 2017 -0500
Committer: Esteban Gutierrez <este...@apache.org>
Committed: Thu Aug 10 18:54:01 2017 -0500
--
.../hadoop/hbase/regionserver/HRegion.java | 11 +++-
.../hbase/regionserver/HRegionFileSystem.java | 31 +--
.../hadoop/hbase/regionserver/TestHRegion.java | 7 ++-
.../hbase/regionserver/TestRegionOpen.java | 56 +++-
.../TestStoreFileRefresherChore.java| 2 +
5 files changed, 99 insertions(+), 8 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/15bad3c0/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index dfb7b71..1fac683 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -902,8 +902,15 @@ public class HRegion implements HeapSize,
PropagatingConfigurationObserver, Regi
}
// Write HRI to a file in case we need to recover hbase:meta
-status.setStatus("Writing region info on filesystem");
-fs.checkRegionInfoOnFilesystem();
+// Only the primary replica should write .regioninfo
+if (this.getRegionInfo().getReplicaId() == HRegionInfo.DEFAULT_REPLICA_ID)
{
+ status.setStatus("Writing region info on filesystem");
+ fs.checkRegionInfoOnFilesystem();
+} else {
+ if (LOG.isDebugEnabled()) {
+LOG.debug("Skipping creation of .regioninfo file for " +
this.getRegionInfo());
+ }
+}
// Initialize all the HStores
status.setStatus("Initializing all the Stores");
http://git-wip-us.apache.org/repos/asf/hbase/blob/15bad3c0/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java
index 33c03ca..3a0b30a 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java
@@ -858,9 +858,19 @@ public class HRegionFileSystem {
// only should be sufficient. I don't want to read the file every time to
check if it pb
// serialized.
byte[] content = getRegionInfoFileContent(regionInfoForFs);
+
+// Verify if the region directory exists before opening a region. We need
to do this since if
+// the region directory doesn't exist we will re-create the region
directory and a new HRI
+// when HRegion.openHRegion() is called.
try {
- Path regionInfoFile = new Path(getRegionDir(), REGION_INFO_FILE);
+ FileStatus status = fs.getFileStatus(getRegionDir());
+} catch (FileNotFoundException e) {
+ LOG.warn(getRegionDir() + " doesn't exist for region: " +
regionInfoForFs.getEncodedName() +
+ " on table " + regionInfo.getTable());
+}
+try {
+ Path regionInfoFile = new Path(getRegionDir(), REGION_INFO_FILE);
FileStatus status = fs.getFileStatus(regionInfoFile);
if (status != null && status.getLen() == content.length) {
// Then assume the content good and move on.
@@ -953,7 +963,13 @@ public class HRegionFileSystem {
}
// Write HRI to a file in case we need to recover hbase:meta
-regionFs.writeRegionInfoOnFilesystem(false);
+// Only primary replicas should write region info
+if (regionInfo.getReplicaId() == HRegionInfo.DEFAULT_REPLICA_ID) {
+ regionFs.writeRegionInfoOnFilesystem(false);
+} else {
+ if (LOG.isDebugEnabled())
+LOG.debug("Skipping creation of .regioninfo file for " + regionInfo);
+}
return regionFs;
}
@@ -983,8 +999,15 @@ public class HRegionFileSystem {
regi
hbase git commit: HBASE-18024 HRegion#initializeRegionInternals should not re-create .hregioninfo file when the region directory no longer exists
Repository: hbase
Updated Branches:
refs/heads/branch-1.2 97877df17 -> 11503fc6e
HBASE-18024 HRegion#initializeRegionInternals should not re-create .hregioninfo
file when the region directory no longer exists
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/11503fc6
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/11503fc6
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/11503fc6
Branch: refs/heads/branch-1.2
Commit: 11503fc6e65a6e350d4a17cd6031d6ae4d204bdc
Parents: 97877df
Author: Esteban Gutierrez <este...@apache.org>
Authored: Fri Jul 21 13:13:00 2017 -0500
Committer: Esteban Gutierrez <este...@apache.org>
Committed: Thu Aug 10 18:47:45 2017 -0500
--
.../hadoop/hbase/regionserver/HRegion.java | 11 +--
.../hbase/regionserver/HRegionFileSystem.java | 31 +---
.../hadoop/hbase/regionserver/TestHRegion.java | 7 -
.../TestStoreFileRefresherChore.java| 2 ++
4 files changed, 44 insertions(+), 7 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/11503fc6/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index 5bb6206..ef9a81a 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -811,8 +811,15 @@ public class HRegion implements HeapSize,
PropagatingConfigurationObserver, Regi
}
// Write HRI to a file in case we need to recover hbase:meta
-status.setStatus("Writing region info on filesystem");
-fs.checkRegionInfoOnFilesystem();
+// Only the primary replica should write .regioninfo
+if (this.getRegionInfo().getReplicaId() == HRegionInfo.DEFAULT_REPLICA_ID)
{
+ status.setStatus("Writing region info on filesystem");
+ fs.checkRegionInfoOnFilesystem();
+} else {
+ if (LOG.isDebugEnabled()) {
+LOG.debug("Skipping creation of .regioninfo file for " +
this.getRegionInfo());
+ }
+}
// Initialize all the HStores
status.setStatus("Initializing all the Stores");
http://git-wip-us.apache.org/repos/asf/hbase/blob/11503fc6/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java
index 19e8c45..619358c 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java
@@ -798,9 +798,19 @@ public class HRegionFileSystem {
// only should be sufficient. I don't want to read the file every time to
check if it pb
// serialized.
byte[] content = getRegionInfoFileContent(regionInfoForFs);
+
+// Verify if the region directory exists before opening a region. We need
to do this since if
+// the region directory doesn't exist we will re-create the region
directory and a new HRI
+// when HRegion.openHRegion() is called.
try {
- Path regionInfoFile = new Path(getRegionDir(), REGION_INFO_FILE);
+ FileStatus status = fs.getFileStatus(getRegionDir());
+} catch (FileNotFoundException e) {
+ LOG.warn(getRegionDir() + " doesn't exist for region: " +
regionInfoForFs.getEncodedName() +
+ " on table " + regionInfo.getTable());
+}
+try {
+ Path regionInfoFile = new Path(getRegionDir(), REGION_INFO_FILE);
FileStatus status = fs.getFileStatus(regionInfoFile);
if (status != null && status.getLen() == content.length) {
// Then assume the content good and move on.
@@ -893,7 +903,13 @@ public class HRegionFileSystem {
}
// Write HRI to a file in case we need to recover hbase:meta
-regionFs.writeRegionInfoOnFilesystem(false);
+// Only primary replicas should write region info
+if (regionInfo.getReplicaId() == HRegionInfo.DEFAULT_REPLICA_ID) {
+ regionFs.writeRegionInfoOnFilesystem(false);
+} else {
+ if (LOG.isDebugEnabled())
+LOG.debug("Skipping creation of .regioninfo file for " + regionInfo);
+}
return regionFs;
}
@@ -923,8 +939,15 @@ public class HRegionFileSystem {
regionFs.cleanupSplitsDir();
regionFs.cleanupM
hbase git commit: HBASE-18024 HRegion#initializeRegionInternals should not re-create .hregioninfo file when the region directory no longer exists
Repository: hbase
Updated Branches:
refs/heads/branch-1.3 52c2dcbaa -> 77847da92
HBASE-18024 HRegion#initializeRegionInternals should not re-create .hregioninfo
file when the region directory no longer exists
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/77847da9
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/77847da9
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/77847da9
Branch: refs/heads/branch-1.3
Commit: 77847da928f9ee79674f54f8dc517fdbde35287f
Parents: 52c2dcb
Author: Esteban Gutierrez <este...@apache.org>
Authored: Fri Jul 21 13:13:00 2017 -0500
Committer: Esteban Gutierrez <este...@apache.org>
Committed: Thu Aug 10 18:41:20 2017 -0500
--
.../hadoop/hbase/regionserver/HRegion.java | 11 +++-
.../hbase/regionserver/HRegionFileSystem.java | 31 +--
.../hadoop/hbase/regionserver/TestHRegion.java | 7 ++-
.../hbase/regionserver/TestRegionOpen.java | 56 +++-
.../TestStoreFileRefresherChore.java| 2 +
5 files changed, 99 insertions(+), 8 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/77847da9/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index 4ba66b7..718fe74 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -820,8 +820,15 @@ public class HRegion implements HeapSize,
PropagatingConfigurationObserver, Regi
}
// Write HRI to a file in case we need to recover hbase:meta
-status.setStatus("Writing region info on filesystem");
-fs.checkRegionInfoOnFilesystem();
+// Only the primary replica should write .regioninfo
+if (this.getRegionInfo().getReplicaId() == HRegionInfo.DEFAULT_REPLICA_ID)
{
+ status.setStatus("Writing region info on filesystem");
+ fs.checkRegionInfoOnFilesystem();
+} else {
+ if (LOG.isDebugEnabled()) {
+LOG.debug("Skipping creation of .regioninfo file for " +
this.getRegionInfo());
+ }
+}
// Initialize all the HStores
status.setStatus("Initializing all the Stores");
http://git-wip-us.apache.org/repos/asf/hbase/blob/77847da9/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java
index 19e8c45..619358c 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java
@@ -798,9 +798,19 @@ public class HRegionFileSystem {
// only should be sufficient. I don't want to read the file every time to
check if it pb
// serialized.
byte[] content = getRegionInfoFileContent(regionInfoForFs);
+
+// Verify if the region directory exists before opening a region. We need
to do this since if
+// the region directory doesn't exist we will re-create the region
directory and a new HRI
+// when HRegion.openHRegion() is called.
try {
- Path regionInfoFile = new Path(getRegionDir(), REGION_INFO_FILE);
+ FileStatus status = fs.getFileStatus(getRegionDir());
+} catch (FileNotFoundException e) {
+ LOG.warn(getRegionDir() + " doesn't exist for region: " +
regionInfoForFs.getEncodedName() +
+ " on table " + regionInfo.getTable());
+}
+try {
+ Path regionInfoFile = new Path(getRegionDir(), REGION_INFO_FILE);
FileStatus status = fs.getFileStatus(regionInfoFile);
if (status != null && status.getLen() == content.length) {
// Then assume the content good and move on.
@@ -893,7 +903,13 @@ public class HRegionFileSystem {
}
// Write HRI to a file in case we need to recover hbase:meta
-regionFs.writeRegionInfoOnFilesystem(false);
+// Only primary replicas should write region info
+if (regionInfo.getReplicaId() == HRegionInfo.DEFAULT_REPLICA_ID) {
+ regionFs.writeRegionInfoOnFilesystem(false);
+} else {
+ if (LOG.isDebugEnabled())
+LOG.debug("Skipping creation of .regioninfo file for " + regionInfo);
+}
return regionFs;
}
@@ -923,8 +939,15 @@ public class HRegionFileSystem {
hbase git commit: HBASE-18024 HRegion#initializeRegionInternals should not re-create .hregioninfo file when the region directory no longer exists
Repository: hbase
Updated Branches:
refs/heads/branch-1.4 5b10393c9 -> ee86d07f4
HBASE-18024 HRegion#initializeRegionInternals should not re-create .hregioninfo
file when the region directory no longer exists
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/ee86d07f
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/ee86d07f
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/ee86d07f
Branch: refs/heads/branch-1.4
Commit: ee86d07f406713d3a93bca15193dba12a880e2fa
Parents: 5b10393
Author: Esteban Gutierrez <este...@apache.org>
Authored: Fri Jul 21 13:13:00 2017 -0500
Committer: Esteban Gutierrez <este...@apache.org>
Committed: Thu Aug 10 18:40:36 2017 -0500
--
.../hadoop/hbase/regionserver/HRegion.java | 11 +++-
.../hbase/regionserver/HRegionFileSystem.java | 31 +--
.../hadoop/hbase/regionserver/TestHRegion.java | 7 ++-
.../hbase/regionserver/TestRegionOpen.java | 56 +++-
.../TestStoreFileRefresherChore.java| 2 +
5 files changed, 99 insertions(+), 8 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/ee86d07f/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index 985c374..71c7892 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -902,8 +902,15 @@ public class HRegion implements HeapSize,
PropagatingConfigurationObserver, Regi
}
// Write HRI to a file in case we need to recover hbase:meta
-status.setStatus("Writing region info on filesystem");
-fs.checkRegionInfoOnFilesystem();
+// Only the primary replica should write .regioninfo
+if (this.getRegionInfo().getReplicaId() == HRegionInfo.DEFAULT_REPLICA_ID)
{
+ status.setStatus("Writing region info on filesystem");
+ fs.checkRegionInfoOnFilesystem();
+} else {
+ if (LOG.isDebugEnabled()) {
+LOG.debug("Skipping creation of .regioninfo file for " +
this.getRegionInfo());
+ }
+}
// Initialize all the HStores
status.setStatus("Initializing all the Stores");
http://git-wip-us.apache.org/repos/asf/hbase/blob/ee86d07f/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java
index 33c03ca..3a0b30a 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java
@@ -858,9 +858,19 @@ public class HRegionFileSystem {
// only should be sufficient. I don't want to read the file every time to
check if it pb
// serialized.
byte[] content = getRegionInfoFileContent(regionInfoForFs);
+
+// Verify if the region directory exists before opening a region. We need
to do this since if
+// the region directory doesn't exist we will re-create the region
directory and a new HRI
+// when HRegion.openHRegion() is called.
try {
- Path regionInfoFile = new Path(getRegionDir(), REGION_INFO_FILE);
+ FileStatus status = fs.getFileStatus(getRegionDir());
+} catch (FileNotFoundException e) {
+ LOG.warn(getRegionDir() + " doesn't exist for region: " +
regionInfoForFs.getEncodedName() +
+ " on table " + regionInfo.getTable());
+}
+try {
+ Path regionInfoFile = new Path(getRegionDir(), REGION_INFO_FILE);
FileStatus status = fs.getFileStatus(regionInfoFile);
if (status != null && status.getLen() == content.length) {
// Then assume the content good and move on.
@@ -953,7 +963,13 @@ public class HRegionFileSystem {
}
// Write HRI to a file in case we need to recover hbase:meta
-regionFs.writeRegionInfoOnFilesystem(false);
+// Only primary replicas should write region info
+if (regionInfo.getReplicaId() == HRegionInfo.DEFAULT_REPLICA_ID) {
+ regionFs.writeRegionInfoOnFilesystem(false);
+} else {
+ if (LOG.isDebugEnabled())
+LOG.debug("Skipping creation of .regioninfo file for " + regionInfo);
+}
return regionFs;
}
@@ -983,8 +999,15 @@ public class HRegionFileSystem {
hbase git commit: HBASE-18024 HRegion#initializeRegionInternals should not re-create .hregioninfo file when the region directory no longer exists
Repository: hbase
Updated Branches:
refs/heads/branch-2 ad266a4b6 -> b3e7e31de
HBASE-18024 HRegion#initializeRegionInternals should not re-create .hregioninfo
file when the region directory no longer exists
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/b3e7e31d
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/b3e7e31d
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/b3e7e31d
Branch: refs/heads/branch-2
Commit: b3e7e31dee77afa0053c798bb45c105323930774
Parents: ad266a4
Author: Esteban Gutierrez <este...@apache.org>
Authored: Fri Jul 21 13:13:00 2017 -0500
Committer: Esteban Gutierrez <este...@apache.org>
Committed: Thu Aug 10 17:56:56 2017 -0500
--
.../hadoop/hbase/regionserver/HRegion.java | 11 +++-
.../hbase/regionserver/HRegionFileSystem.java | 31 ++--
.../hadoop/hbase/regionserver/TestHRegion.java | 7 ++-
.../hbase/regionserver/TestRegionOpen.java | 53 +++-
.../TestStoreFileRefresherChore.java| 1 +
.../TestWALMonotonicallyIncreasingSeqId.java| 5 +-
6 files changed, 97 insertions(+), 11 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/b3e7e31d/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index 483cb36..3b24f3d 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -888,8 +888,15 @@ public class HRegion implements HeapSize,
PropagatingConfigurationObserver, Regi
}
// Write HRI to a file in case we need to recover hbase:meta
-status.setStatus("Writing region info on filesystem");
-fs.checkRegionInfoOnFilesystem();
+// Only the primary replica should write .regioninfo
+if (this.getRegionInfo().getReplicaId() == HRegionInfo.DEFAULT_REPLICA_ID)
{
+ status.setStatus("Writing region info on filesystem");
+ fs.checkRegionInfoOnFilesystem();
+} else {
+ if (LOG.isDebugEnabled()) {
+LOG.debug("Skipping creation of .regioninfo file for " +
this.getRegionInfo());
+ }
+}
// Initialize all the HStores
status.setStatus("Initializing all the Stores");
http://git-wip-us.apache.org/repos/asf/hbase/blob/b3e7e31d/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java
index 1041260..9cb1316 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java
@@ -893,9 +893,19 @@ public class HRegionFileSystem {
// only should be sufficient. I don't want to read the file every time to
check if it pb
// serialized.
byte[] content = getRegionInfoFileContent(regionInfoForFs);
+
+// Verify if the region directory exists before opening a region. We need
to do this since if
+// the region directory doesn't exist we will re-create the region
directory and a new HRI
+// when HRegion.openHRegion() is called.
try {
- Path regionInfoFile = new Path(getRegionDir(), REGION_INFO_FILE);
+ FileStatus status = fs.getFileStatus(getRegionDir());
+} catch (FileNotFoundException e) {
+ LOG.warn(getRegionDir() + " doesn't exist for region: " +
regionInfoForFs.getEncodedName() +
+ " on table " + regionInfo.getTable());
+}
+try {
+ Path regionInfoFile = new Path(getRegionDir(), REGION_INFO_FILE);
FileStatus status = fs.getFileStatus(regionInfoFile);
if (status != null && status.getLen() == content.length) {
// Then assume the content good and move on.
@@ -988,7 +998,13 @@ public class HRegionFileSystem {
}
// Write HRI to a file in case we need to recover hbase:meta
-regionFs.writeRegionInfoOnFilesystem(false);
+// Only primary replicas should write region info
+if (regionInfo.getReplicaId() == HRegionInfo.DEFAULT_REPLICA_ID) {
+ regionFs.writeRegionInfoOnFilesystem(false);
+} else {
+ if (LOG.isDebugEnabled())
+LOG.debug("Skipping creation of .regioninfo file for " + regionInfo);
+}
return regionFs;
hbase git commit: HBASE-18024 HRegion#initializeRegionInternals should not re-create .hregioninfo file when the region directory no longer exists
Repository: hbase
Updated Branches:
refs/heads/master 5507150a1 -> efd211deb
HBASE-18024 HRegion#initializeRegionInternals should not re-create .hregioninfo
file when the region directory no longer exists
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/efd211de
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/efd211de
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/efd211de
Branch: refs/heads/master
Commit: efd211debd8a37f215b1a6fdb982aa3ca890bc40
Parents: 5507150
Author: Esteban Gutierrez <este...@apache.org>
Authored: Fri Jul 21 13:13:00 2017 -0500
Committer: Esteban Gutierrez <este...@apache.org>
Committed: Thu Aug 10 17:56:17 2017 -0500
--
.../hadoop/hbase/regionserver/HRegion.java | 11 +++-
.../hbase/regionserver/HRegionFileSystem.java | 31 ++--
.../hadoop/hbase/regionserver/TestHRegion.java | 7 ++-
.../hbase/regionserver/TestRegionOpen.java | 53 +++-
.../TestStoreFileRefresherChore.java| 1 +
.../TestWALMonotonicallyIncreasingSeqId.java| 5 +-
6 files changed, 97 insertions(+), 11 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/efd211de/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index 483cb36..3b24f3d 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -888,8 +888,15 @@ public class HRegion implements HeapSize,
PropagatingConfigurationObserver, Regi
}
// Write HRI to a file in case we need to recover hbase:meta
-status.setStatus("Writing region info on filesystem");
-fs.checkRegionInfoOnFilesystem();
+// Only the primary replica should write .regioninfo
+if (this.getRegionInfo().getReplicaId() == HRegionInfo.DEFAULT_REPLICA_ID)
{
+ status.setStatus("Writing region info on filesystem");
+ fs.checkRegionInfoOnFilesystem();
+} else {
+ if (LOG.isDebugEnabled()) {
+LOG.debug("Skipping creation of .regioninfo file for " +
this.getRegionInfo());
+ }
+}
// Initialize all the HStores
status.setStatus("Initializing all the Stores");
http://git-wip-us.apache.org/repos/asf/hbase/blob/efd211de/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java
index 1041260..9cb1316 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java
@@ -893,9 +893,19 @@ public class HRegionFileSystem {
// only should be sufficient. I don't want to read the file every time to
check if it pb
// serialized.
byte[] content = getRegionInfoFileContent(regionInfoForFs);
+
+// Verify if the region directory exists before opening a region. We need
to do this since if
+// the region directory doesn't exist we will re-create the region
directory and a new HRI
+// when HRegion.openHRegion() is called.
try {
- Path regionInfoFile = new Path(getRegionDir(), REGION_INFO_FILE);
+ FileStatus status = fs.getFileStatus(getRegionDir());
+} catch (FileNotFoundException e) {
+ LOG.warn(getRegionDir() + " doesn't exist for region: " +
regionInfoForFs.getEncodedName() +
+ " on table " + regionInfo.getTable());
+}
+try {
+ Path regionInfoFile = new Path(getRegionDir(), REGION_INFO_FILE);
FileStatus status = fs.getFileStatus(regionInfoFile);
if (status != null && status.getLen() == content.length) {
// Then assume the content good and move on.
@@ -988,7 +998,13 @@ public class HRegionFileSystem {
}
// Write HRI to a file in case we need to recover hbase:meta
-regionFs.writeRegionInfoOnFilesystem(false);
+// Only primary replicas should write region info
+if (regionInfo.getReplicaId() == HRegionInfo.DEFAULT_REPLICA_ID) {
+ regionFs.writeRegionInfoOnFilesystem(false);
+} else {
+ if (LOG.isDebugEnabled())
+LOG.debug("Skipping creation of .regioninfo file for " + regionInfo);
+}
return regionFs;
hbase git commit: HBASE-18362 hbck should not report split replica parent region from meta as errors (Huaxiang Sun)
Repository: hbase
Updated Branches:
refs/heads/branch-2 10573c6c9 -> 75789048c
HBASE-18362 hbck should not report split replica parent region from meta as
errors (Huaxiang Sun)
Signed-off-by: Esteban Gutierrez <este...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/75789048
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/75789048
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/75789048
Branch: refs/heads/branch-2
Commit: 75789048c13d925d9ee9b67ad8b8dc356e7547ff
Parents: 10573c6
Author: Esteban Gutierrez <este...@apache.org>
Authored: Thu Jul 27 15:58:16 2017 -0500
Committer: Esteban Gutierrez <este...@apache.org>
Committed: Thu Jul 27 16:04:45 2017 -0500
--
.../org/apache/hadoop/hbase/util/HBaseFsck.java | 10
.../hbase/util/TestHBaseFsckReplicas.java | 25
2 files changed, 35 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/75789048/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
index d4f8e45..ff5d482 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
@@ -2484,6 +2484,16 @@ public class HBaseFsck extends Configured implements
Closeable {
return;
}
}
+
+ // For Replica region, we need to do a similar check. If replica is not
split successfully,
+ // error is going to be reported against primary daughter region.
+ if (hbi.getReplicaId() != HRegionInfo.DEFAULT_REPLICA_ID) {
+LOG.info("Region " + descriptiveName + " is a split parent in META, in
HDFS, "
++ "and not deployed on any region server. This may be transient.");
+hbi.setSkipChecks(true);
+return;
+ }
+
errors.reportError(ERROR_CODE.LINGERING_SPLIT_PARENT, "Region "
+ descriptiveName + " is a split parent in META, in HDFS, "
+ "and not deployed on any region server. This could be transient, "
http://git-wip-us.apache.org/repos/asf/hbase/blob/75789048/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckReplicas.java
--
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckReplicas.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckReplicas.java
index 3d0647e..6e49b81 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckReplicas.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckReplicas.java
@@ -111,6 +111,31 @@ public class TestHBaseFsckReplicas extends
BaseTestHBaseFsck {
}
/*
+ * This creates a table with region_replica > 1, do a split, check
+ * that hbck will not report split replica parent as lingering split parent
+ */
+ @Test public void testHbckReportReplicaLingeringSplitParent() throws
Exception {
+TableName table =
TableName.valueOf("testHbckReportReplicaLingeringSplitParent");
+
+try {
+ setupTableWithRegionReplica(table, 2);
+ admin.flush(table);
+
+ // disable catalog janitor
+ admin.enableCatalogJanitor(false);
+ admin.split(table, Bytes.toBytes("A1"));
+
+ Thread.sleep(1000);
+ // run hbck again to make sure we don't see any errors
+ assertNoErrors(doFsck(conf, false));
+} finally {
+ cleanupTable(table);
+ // enable catalog janitor
+ admin.enableCatalogJanitor(true);
+}
+ }
+
+ /*
* This creates a table with region_replica > 1 and verifies hbck runs
* successfully
*/
hbase git commit: HBASE-18362 hbck should not report split replica parent region from meta as errors (Huaxiang Sun)
Repository: hbase
Updated Branches:
refs/heads/branch-1.4 9914d28df -> 615a67799
HBASE-18362 hbck should not report split replica parent region from meta as
errors (Huaxiang Sun)
Signed-off-by: Esteban Gutierrez <este...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/615a6779
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/615a6779
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/615a6779
Branch: refs/heads/branch-1.4
Commit: 615a677994d3491d4e6d8b445502a2d819d29cdb
Parents: 9914d28
Author: Esteban Gutierrez <este...@apache.org>
Authored: Thu Jul 27 15:59:07 2017 -0500
Committer: Esteban Gutierrez <este...@apache.org>
Committed: Thu Jul 27 16:00:14 2017 -0500
--
.../org/apache/hadoop/hbase/util/HBaseFsck.java | 10
.../apache/hadoop/hbase/util/TestHBaseFsck.java | 25
2 files changed, 35 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/615a6779/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
index 0c491e2..e5e4b5a 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
@@ -2440,6 +2440,16 @@ public class HBaseFsck extends Configured implements
Closeable {
return;
}
}
+
+ // For Replica region, we need to do a similar check. If replica is not
split successfully,
+ // error is going to be reported against primary daughter region.
+ if (hbi.getReplicaId() != HRegionInfo.DEFAULT_REPLICA_ID) {
+LOG.info("Region " + descriptiveName + " is a split parent in META, in
HDFS, "
++ "and not deployed on any region server. This may be transient.");
+hbi.setSkipChecks(true);
+return;
+ }
+
errors.reportError(ERROR_CODE.LINGERING_SPLIT_PARENT, "Region "
+ descriptiveName + " is a split parent in META, in HDFS, "
+ "and not deployed on any region server. This could be transient, "
http://git-wip-us.apache.org/repos/asf/hbase/blob/615a6779/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java
--
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java
index c0102de..f1dc1a2 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java
@@ -210,6 +210,31 @@ public class TestHBaseFsck {
EnvironmentEdgeManager.reset();
}
+ /*
+ * This creates a table with region_replica > 1, do a split, check
+ * that hbck will not report split replica parent as lingering split parent
+ */
+ @Test public void testHbckReportReplicaLingeringSplitParent() throws
Exception {
+TableName table =
TableName.valueOf("testHbckReportReplicaLingeringSplitParent");
+
+try {
+ setupTableWithRegionReplica(table, 2);
+ TEST_UTIL.getHBaseAdmin().flush(table.getName());
+
+ // disable catalog janitor
+ TEST_UTIL.getHBaseAdmin().enableCatalogJanitor(false);
+ admin.split(table, Bytes.toBytes("A1"));
+
+ Thread.sleep(1000);
+ // run hbck again to make sure we don't see any errors
+ assertNoErrors(doFsck(conf, false));
+} finally {
+ cleanupTable(table);
+ // enable catalog janitor
+ TEST_UTIL.getHBaseAdmin().enableCatalogJanitor(true);
+}
+ }
+
@Test (timeout=18)
public void testHBaseFsck() throws Exception {
assertNoErrors(doFsck(conf, false));
hbase git commit: HBASE-18362 hbck should not report split replica parent region from meta as errors (Huaxiang Sun)
Repository: hbase
Updated Branches:
refs/heads/master 3536c58af -> 9a1661832
HBASE-18362 hbck should not report split replica parent region from meta as
errors (Huaxiang Sun)
Signed-off-by: Esteban Gutierrez <este...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/9a166183
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/9a166183
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/9a166183
Branch: refs/heads/master
Commit: 9a1661832d5f515df2addf706d0e249a8a2bb8cb
Parents: 3536c58
Author: Esteban Gutierrez <este...@apache.org>
Authored: Thu Jul 27 15:58:16 2017 -0500
Committer: Esteban Gutierrez <este...@apache.org>
Committed: Thu Jul 27 15:58:16 2017 -0500
--
.../org/apache/hadoop/hbase/util/HBaseFsck.java | 10
.../hbase/util/TestHBaseFsckReplicas.java | 25
2 files changed, 35 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/9a166183/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
index d4f8e45..ff5d482 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
@@ -2484,6 +2484,16 @@ public class HBaseFsck extends Configured implements
Closeable {
return;
}
}
+
+ // For Replica region, we need to do a similar check. If replica is not
split successfully,
+ // error is going to be reported against primary daughter region.
+ if (hbi.getReplicaId() != HRegionInfo.DEFAULT_REPLICA_ID) {
+LOG.info("Region " + descriptiveName + " is a split parent in META, in
HDFS, "
++ "and not deployed on any region server. This may be transient.");
+hbi.setSkipChecks(true);
+return;
+ }
+
errors.reportError(ERROR_CODE.LINGERING_SPLIT_PARENT, "Region "
+ descriptiveName + " is a split parent in META, in HDFS, "
+ "and not deployed on any region server. This could be transient, "
http://git-wip-us.apache.org/repos/asf/hbase/blob/9a166183/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckReplicas.java
--
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckReplicas.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckReplicas.java
index 3d0647e..6e49b81 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckReplicas.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckReplicas.java
@@ -111,6 +111,31 @@ public class TestHBaseFsckReplicas extends
BaseTestHBaseFsck {
}
/*
+ * This creates a table with region_replica > 1, do a split, check
+ * that hbck will not report split replica parent as lingering split parent
+ */
+ @Test public void testHbckReportReplicaLingeringSplitParent() throws
Exception {
+TableName table =
TableName.valueOf("testHbckReportReplicaLingeringSplitParent");
+
+try {
+ setupTableWithRegionReplica(table, 2);
+ admin.flush(table);
+
+ // disable catalog janitor
+ admin.enableCatalogJanitor(false);
+ admin.split(table, Bytes.toBytes("A1"));
+
+ Thread.sleep(1000);
+ // run hbck again to make sure we don't see any errors
+ assertNoErrors(doFsck(conf, false));
+} finally {
+ cleanupTable(table);
+ // enable catalog janitor
+ admin.enableCatalogJanitor(true);
+}
+ }
+
+ /*
* This creates a table with region_replica > 1 and verifies hbck runs
* successfully
*/
hbase git commit: HBASE-18362 hbck should not report split replica parent region from meta as errors (Huaxiang Sun)
Repository: hbase
Updated Branches:
refs/heads/branch-1.2 30d58a142 -> 34120e896
HBASE-18362 hbck should not report split replica parent region from meta as
errors (Huaxiang Sun)
Signed-off-by: Esteban Gutierrez <este...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/34120e89
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/34120e89
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/34120e89
Branch: refs/heads/branch-1.2
Commit: 34120e896956c3e08ff51a7cd12013888f9bf93e
Parents: 30d58a1
Author: Esteban Gutierrez <este...@apache.org>
Authored: Thu Jul 27 15:59:07 2017 -0500
Committer: Esteban Gutierrez <este...@apache.org>
Committed: Thu Jul 27 15:59:53 2017 -0500
--
.../org/apache/hadoop/hbase/util/HBaseFsck.java | 10
.../apache/hadoop/hbase/util/TestHBaseFsck.java | 25
2 files changed, 35 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/34120e89/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
index 0fc4d15..212b12a 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
@@ -2288,6 +2288,16 @@ public class HBaseFsck extends Configured implements
Closeable {
return;
}
}
+
+ // For Replica region, we need to do a similar check. If replica is not
split successfully,
+ // error is going to be reported against primary daughter region.
+ if (hbi.getReplicaId() != HRegionInfo.DEFAULT_REPLICA_ID) {
+LOG.info("Region " + descriptiveName + " is a split parent in META, in
HDFS, "
++ "and not deployed on any region server. This may be transient.");
+hbi.setSkipChecks(true);
+return;
+ }
+
errors.reportError(ERROR_CODE.LINGERING_SPLIT_PARENT, "Region "
+ descriptiveName + " is a split parent in META, in HDFS, "
+ "and not deployed on any region server. This could be transient, "
http://git-wip-us.apache.org/repos/asf/hbase/blob/34120e89/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java
--
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java
index 09692db..e74bbc1 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java
@@ -206,6 +206,31 @@ public class TestHBaseFsck {
EnvironmentEdgeManager.reset();
}
+ /*
+ * This creates a table with region_replica > 1, do a split, check
+ * that hbck will not report split replica parent as lingering split parent
+ */
+ @Test public void testHbckReportReplicaLingeringSplitParent() throws
Exception {
+TableName table =
TableName.valueOf("testHbckReportReplicaLingeringSplitParent");
+
+try {
+ setupTableWithRegionReplica(table, 2);
+ TEST_UTIL.getHBaseAdmin().flush(table.getName());
+
+ // disable catalog janitor
+ TEST_UTIL.getHBaseAdmin().enableCatalogJanitor(false);
+ admin.split(table, Bytes.toBytes("A1"));
+
+ Thread.sleep(1000);
+ // run hbck again to make sure we don't see any errors
+ assertNoErrors(doFsck(conf, false));
+} finally {
+ cleanupTable(table);
+ // enable catalog janitor
+ TEST_UTIL.getHBaseAdmin().enableCatalogJanitor(true);
+}
+ }
+
@Test (timeout=18)
public void testHBaseFsck() throws Exception {
assertNoErrors(doFsck(conf, false));
hbase git commit: HBASE-18362 hbck should not report split replica parent region from meta as errors (Huaxiang Sun)
Repository: hbase
Updated Branches:
refs/heads/branch-1.3 22233a28c -> c23b7c9c8
HBASE-18362 hbck should not report split replica parent region from meta as
errors (Huaxiang Sun)
Signed-off-by: Esteban Gutierrez <este...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/c23b7c9c
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/c23b7c9c
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/c23b7c9c
Branch: refs/heads/branch-1.3
Commit: c23b7c9c80d7663741b598f221cca8b4eb74033c
Parents: 22233a2
Author: Esteban Gutierrez <este...@apache.org>
Authored: Thu Jul 27 15:59:07 2017 -0500
Committer: Esteban Gutierrez <este...@apache.org>
Committed: Thu Jul 27 16:00:03 2017 -0500
--
.../org/apache/hadoop/hbase/util/HBaseFsck.java | 10
.../apache/hadoop/hbase/util/TestHBaseFsck.java | 25
2 files changed, 35 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/c23b7c9c/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
index a5b63ae..eae8a64 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
@@ -2311,6 +2311,16 @@ public class HBaseFsck extends Configured implements
Closeable {
return;
}
}
+
+ // For Replica region, we need to do a similar check. If replica is not
split successfully,
+ // error is going to be reported against primary daughter region.
+ if (hbi.getReplicaId() != HRegionInfo.DEFAULT_REPLICA_ID) {
+LOG.info("Region " + descriptiveName + " is a split parent in META, in
HDFS, "
++ "and not deployed on any region server. This may be transient.");
+hbi.setSkipChecks(true);
+return;
+ }
+
errors.reportError(ERROR_CODE.LINGERING_SPLIT_PARENT, "Region "
+ descriptiveName + " is a split parent in META, in HDFS, "
+ "and not deployed on any region server. This could be transient, "
http://git-wip-us.apache.org/repos/asf/hbase/blob/c23b7c9c/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java
--
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java
index 98aecce..9b628ee 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java
@@ -210,6 +210,31 @@ public class TestHBaseFsck {
EnvironmentEdgeManager.reset();
}
+ /*
+ * This creates a table with region_replica > 1, do a split, check
+ * that hbck will not report split replica parent as lingering split parent
+ */
+ @Test public void testHbckReportReplicaLingeringSplitParent() throws
Exception {
+TableName table =
TableName.valueOf("testHbckReportReplicaLingeringSplitParent");
+
+try {
+ setupTableWithRegionReplica(table, 2);
+ TEST_UTIL.getHBaseAdmin().flush(table.getName());
+
+ // disable catalog janitor
+ TEST_UTIL.getHBaseAdmin().enableCatalogJanitor(false);
+ admin.split(table, Bytes.toBytes("A1"));
+
+ Thread.sleep(1000);
+ // run hbck again to make sure we don't see any errors
+ assertNoErrors(doFsck(conf, false));
+} finally {
+ cleanupTable(table);
+ // enable catalog janitor
+ TEST_UTIL.getHBaseAdmin().enableCatalogJanitor(true);
+}
+ }
+
@Test (timeout=18)
public void testHBaseFsck() throws Exception {
assertNoErrors(doFsck(conf, false));
hbase git commit: HBASE-18362 hbck should not report split replica parent region from meta as errors (Huaxiang Sun)
Repository: hbase
Updated Branches:
refs/heads/branch-1 aca8a124c -> e662d0432
HBASE-18362 hbck should not report split replica parent region from meta as
errors (Huaxiang Sun)
Signed-off-by: Esteban Gutierrez <este...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/e662d043
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/e662d043
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/e662d043
Branch: refs/heads/branch-1
Commit: e662d04321f2c4cec29a775a220752a50168026d
Parents: aca8a12
Author: Esteban Gutierrez <este...@apache.org>
Authored: Thu Jul 27 15:59:07 2017 -0500
Committer: Esteban Gutierrez <este...@apache.org>
Committed: Thu Jul 27 15:59:07 2017 -0500
--
.../org/apache/hadoop/hbase/util/HBaseFsck.java | 10
.../apache/hadoop/hbase/util/TestHBaseFsck.java | 25
2 files changed, 35 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/e662d043/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
index 0c491e2..e5e4b5a 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
@@ -2440,6 +2440,16 @@ public class HBaseFsck extends Configured implements
Closeable {
return;
}
}
+
+ // For Replica region, we need to do a similar check. If replica is not
split successfully,
+ // error is going to be reported against primary daughter region.
+ if (hbi.getReplicaId() != HRegionInfo.DEFAULT_REPLICA_ID) {
+LOG.info("Region " + descriptiveName + " is a split parent in META, in
HDFS, "
++ "and not deployed on any region server. This may be transient.");
+hbi.setSkipChecks(true);
+return;
+ }
+
errors.reportError(ERROR_CODE.LINGERING_SPLIT_PARENT, "Region "
+ descriptiveName + " is a split parent in META, in HDFS, "
+ "and not deployed on any region server. This could be transient, "
http://git-wip-us.apache.org/repos/asf/hbase/blob/e662d043/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java
--
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java
index c0102de..f1dc1a2 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java
@@ -210,6 +210,31 @@ public class TestHBaseFsck {
EnvironmentEdgeManager.reset();
}
+ /*
+ * This creates a table with region_replica > 1, do a split, check
+ * that hbck will not report split replica parent as lingering split parent
+ */
+ @Test public void testHbckReportReplicaLingeringSplitParent() throws
Exception {
+TableName table =
TableName.valueOf("testHbckReportReplicaLingeringSplitParent");
+
+try {
+ setupTableWithRegionReplica(table, 2);
+ TEST_UTIL.getHBaseAdmin().flush(table.getName());
+
+ // disable catalog janitor
+ TEST_UTIL.getHBaseAdmin().enableCatalogJanitor(false);
+ admin.split(table, Bytes.toBytes("A1"));
+
+ Thread.sleep(1000);
+ // run hbck again to make sure we don't see any errors
+ assertNoErrors(doFsck(conf, false));
+} finally {
+ cleanupTable(table);
+ // enable catalog janitor
+ TEST_UTIL.getHBaseAdmin().enableCatalogJanitor(true);
+}
+ }
+
@Test (timeout=18)
public void testHBaseFsck() throws Exception {
assertNoErrors(doFsck(conf, false));
hbase git commit: HBASE-17761: Test TestRemoveRegionMetrics.testMoveRegion fails intermittently because of race condition
Repository: hbase
Updated Branches:
refs/heads/branch-1 c70769ea8 -> 3d9520b14
HBASE-17761: Test TestRemoveRegionMetrics.testMoveRegion fails intermittently
because of race condition
Signed-off-by: Esteban Gutierrez <este...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/3d9520b1
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/3d9520b1
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/3d9520b1
Branch: refs/heads/branch-1
Commit: 3d9520b1403346cbe7a6322cf6c2632197d79f7a
Parents: c70769e
Author: Umesh Agashe <uaga...@cloudera.com>
Authored: Wed Mar 8 13:25:28 2017 -0800
Committer: Esteban Gutierrez <este...@apache.org>
Committed: Thu Mar 9 14:26:10 2017 -0800
--
.../hadoop/hbase/HBaseTestingUtility.java | 24
.../apache/hadoop/hbase/client/TestAdmin1.java | 21 +
.../regionserver/TestRemoveRegionMetrics.java | 5 +---
.../hbase/regionserver/wal/TestWALReplay.java | 23 ++-
4 files changed, 28 insertions(+), 45 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/3d9520b1/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
--
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
index 8151759..58af51a 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
@@ -3467,6 +3467,30 @@ public class HBaseTestingUtility extends
HBaseCommonTestingUtility {
}
/**
+ * Move region to destination server and wait till region is completely
moved and online
+ *
+ * @param destRegion region to move
+ * @param destServer destination server of the region
+ * @throws InterruptedException
+ * @throws IOException
+ */
+ public void moveRegionAndWait(HRegionInfo destRegion, ServerName destServer)
+ throws InterruptedException, IOException {
+HMaster master = getMiniHBaseCluster().getMaster();
+getHBaseAdmin().move(destRegion.getEncodedNameAsBytes(),
+Bytes.toBytes(destServer.getServerName()));
+while (true) {
+ ServerName serverName = master.getAssignmentManager().getRegionStates()
+ .getRegionServerOfRegion(destRegion);
+ if (serverName != null && serverName.equals(destServer)) {
+assertRegionOnServer(destRegion, serverName, 200);
+break;
+ }
+ Thread.sleep(10);
+}
+ }
+
+ /**
* Wait until all regions for a table in hbase:meta have a non-empty
* info:server, up to a configuable timeout value (default is 60 seconds)
* This means all regions have been deployed,
http://git-wip-us.apache.org/repos/asf/hbase/blob/3d9520b1/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin1.java
--
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin1.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin1.java
index d153a85..43171c3 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin1.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin1.java
@@ -1251,7 +1251,7 @@ public class TestAdmin1 {
// Try going to the regionservers directly
// first move the region to the same regionserver
if (!regions.get(2).getSecond().equals(regions.get(1).getSecond())) {
- moveRegionAndWait(regions.get(2).getFirst(), regions.get(1).getSecond());
+ TEST_UTIL.moveRegionAndWait(regions.get(2).getFirst(),
regions.get(1).getSecond());
}
try {
AdminService.BlockingInterface admin =
TEST_UTIL.getHBaseAdmin().getConnection()
@@ -1264,25 +1264,6 @@ public class TestAdmin1 {
assertTrue(gotException);
}
- private void moveRegionAndWait(HRegionInfo destRegion, ServerName destServer)
- throws InterruptedException, MasterNotRunningException,
- ZooKeeperConnectionException, IOException {
-HMaster master = TEST_UTIL.getMiniHBaseCluster().getMaster();
-TEST_UTIL.getHBaseAdmin().move(
-destRegion.getEncodedNameAsBytes(),
-Bytes.toBytes(destServer.getServerName()));
-while (true) {
- ServerName serverName = master.getAssignmentManager()
- .getRegionStates().getRegionServerOfRegion(destRegion);
- if (serverName != null && serverName.equals(destServer)) {
-TEST_UTIL.assertRegionOnServer(
-destRegion, serverName, 200);
-break;
- }
- Thread.sleep(10);
-}
- }
-
/**
*
hbase git commit: HBASE-15941 HBCK repair should not unsplit healthy splitted region
Repository: hbase
Updated Branches:
refs/heads/branch-1.3 3b09e1853 -> 0f011a9d9
HBASE-15941 HBCK repair should not unsplit healthy splitted region
Signed-off-by: Michael Stack <st...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/0f011a9d
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/0f011a9d
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/0f011a9d
Branch: refs/heads/branch-1.3
Commit: 0f011a9d928999186ad371f6cd02b94f49a5c631
Parents: 3b09e18
Author: Esteban Gutierrez <este...@apache.org>
Authored: Tue Mar 7 01:00:48 2017 -0800
Committer: Esteban Gutierrez <este...@apache.org>
Committed: Wed Mar 8 22:41:44 2017 -0800
--
.../org/apache/hadoop/hbase/util/HBaseFsck.java | 153 ++-
.../hadoop/hbase/util/HBaseFsckRepair.java | 8 +
.../util/hbck/TableIntegrityErrorHandler.java | 8 +
.../apache/hadoop/hbase/util/TestHBaseFsck.java | 138 +
4 files changed, 304 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/0f011a9d/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
index 8d6f545..a5b63ae 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
@@ -247,6 +247,7 @@ public class HBaseFsck extends Configured implements
Closeable {
private boolean fixTableOrphans = false; // fix fs holes (missing .tableinfo)
private boolean fixVersionFile = false; // fix missing hbase.version file in
hdfs
private boolean fixSplitParents = false; // fix lingering split parents
+ private boolean removeParents = false; // remove split parents
private boolean fixReferenceFiles = false; // fix lingering reference store
file
private boolean fixEmptyMetaCells = false; // fix (remove) empty
REGIONINFO_QUALIFIER rows
private boolean fixTableLocks = false; // fix table locks which are expired
@@ -1067,6 +1068,8 @@ public class HBaseFsck extends Configured implements
Closeable {
setShouldRerun();
success = fs.rename(path, dst);
+debugLsr(dst);
+
}
if (!success) {
LOG.error("Failed to sideline reference file " + path);
@@ -2310,7 +2313,8 @@ public class HBaseFsck extends Configured implements
Closeable {
}
errors.reportError(ERROR_CODE.LINGERING_SPLIT_PARENT, "Region "
+ descriptiveName + " is a split parent in META, in HDFS, "
- + "and not deployed on any region server. This could be transient.");
+ + "and not deployed on any region server. This could be transient, "
+ + "consider to run the catalog janitor first!");
if (shouldFixSplitParents()) {
setShouldRerun();
resetSplitParent(hbi);
@@ -2708,6 +2712,18 @@ public class HBaseFsck extends Configured implements
Closeable {
}
@Override
+ public void handleSplit(HbckInfo r1, HbckInfo r2) throws IOException{
+byte[] key = r1.getStartKey();
+// dup start key
+errors.reportError(ERROR_CODE.DUPE_ENDKEYS,
+ "Multiple regions have the same regionID: "
++ Bytes.toStringBinary(key), getTableInfo(), r1);
+errors.reportError(ERROR_CODE.DUPE_ENDKEYS,
+ "Multiple regions have the same regionID: "
++ Bytes.toStringBinary(key), getTableInfo(), r2);
+ }
+
+ @Override
public void handleOverlapInRegionChain(HbckInfo hi1, HbckInfo hi2)
throws IOException{
errors.reportError(ERROR_CODE.OVERLAP_IN_REGION_CHAIN,
"There is an overlap in the region chain.",
@@ -2841,10 +2857,124 @@ public class HBaseFsck extends Configured implements
Closeable {
}
return;
}
-
+if (shouldRemoveParents()) {
+ removeParentsAndFixSplits(overlap);
+}
mergeOverlaps(overlap);
}
+ void removeParentsAndFixSplits(Collection overlap) throws
IOException {
+Pair<byte[], byte[]> range = null;
+HbckInfo parent = null;
+HbckInfo daughterA = null;
+HbckInfo daughterB = null;
+Collection daughters = new ArrayList(overlap);
+
+String thread = Thread.currentThread().getName();
+LOG.info("== [" + thread + "] Attempting fix splits in overlap
state.");
+
+// we only can handle a single split per group at th
hbase git commit: HBASE-15941 HBCK repair should not unsplit healthy splitted region
Repository: hbase
Updated Branches:
refs/heads/branch-1.2 d46e9613b -> c368c8587
HBASE-15941 HBCK repair should not unsplit healthy splitted region
Signed-off-by: Michael Stack <st...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/c368c858
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/c368c858
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/c368c858
Branch: refs/heads/branch-1.2
Commit: c368c8587a85261ac9eefe56a2d5f30ee8fe3f33
Parents: d46e961
Author: Esteban Gutierrez <este...@apache.org>
Authored: Tue Mar 7 01:00:48 2017 -0800
Committer: Esteban Gutierrez <este...@apache.org>
Committed: Wed Mar 8 22:34:52 2017 -0800
--
.../org/apache/hadoop/hbase/util/HBaseFsck.java | 154 ++-
.../hadoop/hbase/util/HBaseFsckRepair.java | 8 +
.../util/hbck/TableIntegrityErrorHandler.java | 8 +
.../apache/hadoop/hbase/util/TestHBaseFsck.java | 138 +
4 files changed, 305 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/c368c858/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
index c1778f7..0fc4d15 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
@@ -245,6 +245,7 @@ public class HBaseFsck extends Configured implements
Closeable {
private boolean fixTableOrphans = false; // fix fs holes (missing .tableinfo)
private boolean fixVersionFile = false; // fix missing hbase.version file in
hdfs
private boolean fixSplitParents = false; // fix lingering split parents
+ private boolean removeParents = false; // remove split parents
private boolean fixReferenceFiles = false; // fix lingering reference store
file
private boolean fixEmptyMetaCells = false; // fix (remove) empty
REGIONINFO_QUALIFIER rows
private boolean fixTableLocks = false; // fix table locks which are expired
@@ -1044,6 +1045,8 @@ public class HBaseFsck extends Configured implements
Closeable {
setShouldRerun();
success = fs.rename(path, dst);
+debugLsr(dst);
+
}
if (!success) {
LOG.error("Failed to sideline reference file " + path);
@@ -2287,7 +2290,8 @@ public class HBaseFsck extends Configured implements
Closeable {
}
errors.reportError(ERROR_CODE.LINGERING_SPLIT_PARENT, "Region "
+ descriptiveName + " is a split parent in META, in HDFS, "
- + "and not deployed on any region server. This could be transient.");
+ + "and not deployed on any region server. This could be transient, "
+ + "consider to run the catalog janitor first!");
if (shouldFixSplitParents()) {
setShouldRerun();
resetSplitParent(hbi);
@@ -2685,6 +2689,18 @@ public class HBaseFsck extends Configured implements
Closeable {
}
@Override
+ public void handleSplit(HbckInfo r1, HbckInfo r2) throws IOException{
+byte[] key = r1.getStartKey();
+// dup start key
+errors.reportError(ERROR_CODE.DUPE_ENDKEYS,
+ "Multiple regions have the same regionID: "
++ Bytes.toStringBinary(key), getTableInfo(), r1);
+errors.reportError(ERROR_CODE.DUPE_ENDKEYS,
+ "Multiple regions have the same regionID: "
++ Bytes.toStringBinary(key), getTableInfo(), r2);
+ }
+
+ @Override
public void handleOverlapInRegionChain(HbckInfo hi1, HbckInfo hi2)
throws IOException{
errors.reportError(ERROR_CODE.OVERLAP_IN_REGION_CHAIN,
"There is an overlap in the region chain.",
@@ -2818,10 +2834,124 @@ public class HBaseFsck extends Configured implements
Closeable {
}
return;
}
-
+if (shouldRemoveParents()) {
+ removeParentsAndFixSplits(overlap);
+}
mergeOverlaps(overlap);
}
+ void removeParentsAndFixSplits(Collection overlap) throws
IOException {
+Pair<byte[], byte[]> range = null;
+HbckInfo parent = null;
+HbckInfo daughterA = null;
+HbckInfo daughterB = null;
+Collection daughters = new ArrayList(overlap);
+
+String thread = Thread.currentThread().getName();
+LOG.info("== [" + thread + "] Attempting fix splits in overlap
state.");
+
+// we only can handle a single split per group at th
hbase git commit: HBASE-15941 HBCK repair should not unsplit healthy splitted region
Repository: hbase
Updated Branches:
refs/heads/master 2b8974843 -> 1160315e2
HBASE-15941 HBCK repair should not unsplit healthy splitted region
Signed-off-by: Michael Stack <st...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/1160315e
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/1160315e
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/1160315e
Branch: refs/heads/master
Commit: 1160315e2f666fd5c51505f55f77399e302885f2
Parents: 2b89748
Author: Esteban Gutierrez <este...@apache.org>
Authored: Tue Mar 7 01:00:48 2017 -0800
Committer: Esteban Gutierrez <este...@apache.org>
Committed: Wed Mar 8 22:02:14 2017 -0800
--
.../org/apache/hadoop/hbase/util/HBaseFsck.java | 153 ++-
.../hadoop/hbase/util/HBaseFsckRepair.java | 8 +
.../util/hbck/TableIntegrityErrorHandler.java | 8 +
.../hbase/util/TestHBaseFsckReplicas.java | 73 +
.../hadoop/hbase/util/TestHBaseFsckTwoRS.java | 87 +--
5 files changed, 311 insertions(+), 18 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/1160315e/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
index 7b3b25b..4eab62b 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
@@ -248,6 +248,7 @@ public class HBaseFsck extends Configured implements
Closeable {
private boolean fixTableOrphans = false; // fix fs holes (missing .tableinfo)
private boolean fixVersionFile = false; // fix missing hbase.version file in
hdfs
private boolean fixSplitParents = false; // fix lingering split parents
+ private boolean removeParents = false; // remove split parents
private boolean fixReferenceFiles = false; // fix lingering reference store
file
private boolean fixHFileLinks = false; // fix lingering HFileLinks
private boolean fixEmptyMetaCells = false; // fix (remove) empty
REGIONINFO_QUALIFIER rows
@@ -1105,6 +1106,8 @@ public class HBaseFsck extends Configured implements
Closeable {
setShouldRerun();
success = fs.rename(path, dst);
+debugLsr(dst);
+
}
if (!success) {
LOG.error("Failed to sideline reference file " + path);
@@ -2483,7 +2486,8 @@ public class HBaseFsck extends Configured implements
Closeable {
}
errors.reportError(ERROR_CODE.LINGERING_SPLIT_PARENT, "Region "
+ descriptiveName + " is a split parent in META, in HDFS, "
- + "and not deployed on any region server. This could be transient.");
+ + "and not deployed on any region server. This could be transient, "
+ + "consider to run the catalog janitor first!");
if (shouldFixSplitParents()) {
setShouldRerun();
resetSplitParent(hbi);
@@ -2880,6 +2884,18 @@ public class HBaseFsck extends Configured implements
Closeable {
}
@Override
+ public void handleSplit(HbckInfo r1, HbckInfo r2) throws IOException{
+byte[] key = r1.getStartKey();
+// dup start key
+errors.reportError(ERROR_CODE.DUPE_ENDKEYS,
+ "Multiple regions have the same regionID: "
++ Bytes.toStringBinary(key), getTableInfo(), r1);
+errors.reportError(ERROR_CODE.DUPE_ENDKEYS,
+ "Multiple regions have the same regionID: "
++ Bytes.toStringBinary(key), getTableInfo(), r2);
+ }
+
+ @Override
public void handleOverlapInRegionChain(HbckInfo hi1, HbckInfo hi2)
throws IOException{
errors.reportError(ERROR_CODE.OVERLAP_IN_REGION_CHAIN,
"There is an overlap in the region chain.",
@@ -3013,10 +3029,124 @@ public class HBaseFsck extends Configured implements
Closeable {
}
return;
}
-
+if (shouldRemoveParents()) {
+ removeParentsAndFixSplits(overlap);
+}
mergeOverlaps(overlap);
}
+ void removeParentsAndFixSplits(Collection overlap) throws
IOException {
+Pair<byte[], byte[]> range = null;
+HbckInfo parent = null;
+HbckInfo daughterA = null;
+HbckInfo daughterB = null;
+Collection daughters = new ArrayList(overlap);
+
+String thread = Thread.currentThread().getName();
+LOG.info("== [" + thread + "] Attempting fix splits in overlap
state.");
+
+// we only can handl
hbase git commit: HBASE-17761: Test TestRemoveRegionMetrics.testMoveRegion fails intermittently because of race condition
Repository: hbase
Updated Branches:
refs/heads/master 4ff838df1 -> 2b8974843
HBASE-17761: Test TestRemoveRegionMetrics.testMoveRegion fails intermittently
because of race condition
Signed-off-by: Esteban Gutierrez <este...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/2b897484
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/2b897484
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/2b897484
Branch: refs/heads/master
Commit: 2b897484327a0afb1ad56101b43b811c851da403
Parents: 4ff838d
Author: Umesh Agashe <uaga...@cloudera.com>
Authored: Wed Mar 8 13:25:28 2017 -0800
Committer: Esteban Gutierrez <este...@apache.org>
Committed: Wed Mar 8 21:18:05 2017 -0800
--
.../hadoop/hbase/HBaseTestingUtility.java | 24
.../regionserver/TestRemoveRegionMetrics.java | 5 +---
.../regionserver/wal/AbstractTestWALReplay.java | 23 ++-
3 files changed, 27 insertions(+), 25 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/2b897484/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
--
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
index 47170b1..a6c7f68 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
@@ -3298,6 +3298,30 @@ public class HBaseTestingUtility extends
HBaseCommonTestingUtility {
}
/**
+ * Move region to destination server and wait till region is completely
moved and online
+ *
+ * @param destRegion region to move
+ * @param destServer destination server of the region
+ * @throws InterruptedException
+ * @throws IOException
+ */
+ public void moveRegionAndWait(HRegionInfo destRegion, ServerName destServer)
+ throws InterruptedException, IOException {
+HMaster master = getMiniHBaseCluster().getMaster();
+getHBaseAdmin().move(destRegion.getEncodedNameAsBytes(),
+Bytes.toBytes(destServer.getServerName()));
+while (true) {
+ ServerName serverName = master.getAssignmentManager().getRegionStates()
+ .getRegionServerOfRegion(destRegion);
+ if (serverName != null && serverName.equals(destServer)) {
+assertRegionOnServer(destRegion, serverName, 200);
+break;
+ }
+ Thread.sleep(10);
+}
+ }
+
+ /**
* Wait until all regions for a table in hbase:meta have a non-empty
* info:server, up to a configuable timeout value (default is 60 seconds)
* This means all regions have been deployed,
http://git-wip-us.apache.org/repos/asf/hbase/blob/2b897484/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRemoveRegionMetrics.java
--
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRemoveRegionMetrics.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRemoveRegionMetrics.java
index eb7a1a1..85699ef 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRemoveRegionMetrics.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRemoveRegionMetrics.java
@@ -96,7 +96,6 @@ public class TestRemoveRegionMetrics {
int destServerIdx = (currentServerIdx +1)%
cluster.getLiveRegionServerThreads().size();
HRegionServer currentServer = cluster.getRegionServer(currentServerIdx);
HRegionServer destServer = cluster.getRegionServer(destServerIdx);
- byte[] destServerName =
Bytes.toBytes(destServer.getServerName().getServerName());
// Do a put. The counters should be non-zero now
@@ -119,13 +118,11 @@ public class TestRemoveRegionMetrics {
try {
-admin.move(regionInfo.getEncodedNameAsBytes(), destServerName);
+TEST_UTIL.moveRegionAndWait(regionInfo, destServer.getServerName());
moved = true;
-Thread.sleep(5000);
} catch (IOException ioe) {
moved = false;
}
- TEST_UTIL.waitUntilAllRegionsAssigned(t.getName());
if (moved) {
MetricsRegionAggregateSource destAgg =
destServer.getRegion(regionInfo.getRegionName())
http://git-wip-us.apache.org/repos/asf/hbase/blob/2b897484/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/AbstractTestWALReplay.java
--
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/AbstractTestWALReplay.java
b/hbase-server/src/test
hbase git commit: HBASE-17622 Add hbase-metrics package to TableMapReduceUtil
Repository: hbase
Updated Branches:
refs/heads/master 9d5d25c88 -> 054acec84
HBASE-17622 Add hbase-metrics package to TableMapReduceUtil
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/054acec8
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/054acec8
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/054acec8
Branch: refs/heads/master
Commit: 054acec84009feed94ca6cf3bf20f5af4f6fecac
Parents: 9d5d25c
Author: Esteban Gutierrez <este...@apache.org>
Authored: Thu Feb 9 17:37:14 2017 -0800
Committer: Esteban Gutierrez <este...@apache.org>
Committed: Fri Feb 10 09:32:50 2017 -0800
--
.../java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java | 1 +
1 file changed, 1 insertion(+)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/054acec8/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java
index 7de91e9..98f39da 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java
@@ -788,6 +788,7 @@ public class TableMapReduceUtil {
org.apache.hadoop.hbase.client.Put.class, //
hbase-client
org.apache.hadoop.hbase.CompatibilityFactory.class,//
hbase-hadoop-compat
org.apache.hadoop.hbase.mapreduce.TableMapper.class, //
hbase-server
+ org.apache.hadoop.hbase.metrics.impl.FastLongHistogram.class, //
hbase-metrics
prefixTreeCodecClass, // hbase-prefix-tree (if null will be skipped)
// pull necessary dependencies
org.apache.zookeeper.ZooKeeper.class,
hbase git commit: HBASE-17351 Unable to generate tar ball for master branch
Repository: hbase Updated Branches: refs/heads/master 5f631b965 -> 910e885a7 HBASE-17351 Unable to generate tar ball for master branch Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/910e885a Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/910e885a Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/910e885a Branch: refs/heads/master Commit: 910e885a75e2092e49ae577d1481d58845fffae5 Parents: 5f631b9 Author: Esteban Gutierrez <este...@apache.org> Authored: Tue Dec 20 15:18:28 2016 -0800 Committer: Esteban Gutierrez <este...@apache.org> Committed: Fri Jan 6 08:42:20 2017 -0800 -- pom.xml | 1 + 1 file changed, 1 insertion(+) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/910e885a/pom.xml -- diff --git a/pom.xml b/pom.xml index 02320dc..285e358 100644 --- a/pom.xml +++ b/pom.xml @@ -876,6 +876,7 @@ org.apache.maven.plugins maven-enforcer-plugin +1.4 org.codehaus.mojo
hbase git commit: HBASE-17058 Lower epsilon used for jitter verification from HBASE-15324
Repository: hbase
Updated Branches:
refs/heads/branch-1.1 a8628ee9a -> 7c58547a3
HBASE-17058 Lower epsilon used for jitter verification from HBASE-15324
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/7c58547a
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/7c58547a
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/7c58547a
Branch: refs/heads/branch-1.1
Commit: 7c58547a37c85a148f481398819badd7c26129bc
Parents: a8628ee
Author: Esteban Gutierrez <este...@apache.org>
Authored: Thu Nov 17 11:11:30 2016 -0800
Committer: Esteban Gutierrez <este...@apache.org>
Committed: Thu Nov 17 12:30:28 2016 -0800
--
.../hbase/regionserver/ConstantSizeRegionSplitPolicy.java | 5 ++---
1 file changed, 2 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/7c58547a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ConstantSizeRegionSplitPolicy.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ConstantSizeRegionSplitPolicy.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ConstantSizeRegionSplitPolicy.java
index 836cec5..d915f2e 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ConstantSizeRegionSplitPolicy.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ConstantSizeRegionSplitPolicy.java
@@ -39,7 +39,6 @@ import
org.apache.hadoop.hbase.classification.InterfaceAudience;
@InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.CONFIG)
public class ConstantSizeRegionSplitPolicy extends RegionSplitPolicy {
private static final Random RANDOM = new Random();
- private static final double EPSILON = 1E-6;
private long desiredMaxFileSize;
private double jitterRate;
@@ -60,7 +59,7 @@ public class ConstantSizeRegionSplitPolicy extends
RegionSplitPolicy {
this.jitterRate = (RANDOM.nextFloat() - 0.5D) * jitter;
long jitterValue = (long) (this.desiredMaxFileSize * this.jitterRate);
// make sure the long value won't overflow with jitter
-if (this.jitterRate > EPSILON && jitterValue > (Long.MAX_VALUE -
this.desiredMaxFileSize)) {
+if (this.jitterRate > 0 && jitterValue > (Long.MAX_VALUE -
this.desiredMaxFileSize)) {
this.desiredMaxFileSize = Long.MAX_VALUE;
} else {
this.desiredMaxFileSize += jitterValue;
@@ -94,6 +93,6 @@ public class ConstantSizeRegionSplitPolicy extends
RegionSplitPolicy {
@VisibleForTesting
public boolean positiveJitterRate() {
-return this.jitterRate > EPSILON;
+return this.jitterRate > 0;
}
}
hbase git commit: HBASE-17058 Lower epsilon used for jitter verification from HBASE-15324
Repository: hbase
Updated Branches:
refs/heads/branch-1.2 e7b310e68 -> bb891c683
HBASE-17058 Lower epsilon used for jitter verification from HBASE-15324
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/bb891c68
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/bb891c68
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/bb891c68
Branch: refs/heads/branch-1.2
Commit: bb891c6834a0691302b958d78e0c009b3601c442
Parents: e7b310e
Author: Esteban Gutierrez <este...@apache.org>
Authored: Thu Nov 17 11:11:30 2016 -0800
Committer: Esteban Gutierrez <este...@apache.org>
Committed: Thu Nov 17 12:30:03 2016 -0800
--
.../hbase/regionserver/ConstantSizeRegionSplitPolicy.java | 5 ++---
1 file changed, 2 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/bb891c68/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ConstantSizeRegionSplitPolicy.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ConstantSizeRegionSplitPolicy.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ConstantSizeRegionSplitPolicy.java
index 836cec5..d915f2e 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ConstantSizeRegionSplitPolicy.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ConstantSizeRegionSplitPolicy.java
@@ -39,7 +39,6 @@ import
org.apache.hadoop.hbase.classification.InterfaceAudience;
@InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.CONFIG)
public class ConstantSizeRegionSplitPolicy extends RegionSplitPolicy {
private static final Random RANDOM = new Random();
- private static final double EPSILON = 1E-6;
private long desiredMaxFileSize;
private double jitterRate;
@@ -60,7 +59,7 @@ public class ConstantSizeRegionSplitPolicy extends
RegionSplitPolicy {
this.jitterRate = (RANDOM.nextFloat() - 0.5D) * jitter;
long jitterValue = (long) (this.desiredMaxFileSize * this.jitterRate);
// make sure the long value won't overflow with jitter
-if (this.jitterRate > EPSILON && jitterValue > (Long.MAX_VALUE -
this.desiredMaxFileSize)) {
+if (this.jitterRate > 0 && jitterValue > (Long.MAX_VALUE -
this.desiredMaxFileSize)) {
this.desiredMaxFileSize = Long.MAX_VALUE;
} else {
this.desiredMaxFileSize += jitterValue;
@@ -94,6 +93,6 @@ public class ConstantSizeRegionSplitPolicy extends
RegionSplitPolicy {
@VisibleForTesting
public boolean positiveJitterRate() {
-return this.jitterRate > EPSILON;
+return this.jitterRate > 0;
}
}
hbase git commit: HBASE-17058 Lower epsilon used for jitter verification from HBASE-15324
Repository: hbase
Updated Branches:
refs/heads/branch-1 c6f1b6e62 -> 19441937e
HBASE-17058 Lower epsilon used for jitter verification from HBASE-15324
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/19441937
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/19441937
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/19441937
Branch: refs/heads/branch-1
Commit: 19441937ea688b6798675993c6af4a961f931c3a
Parents: c6f1b6e
Author: Esteban Gutierrez <este...@apache.org>
Authored: Thu Nov 17 11:11:30 2016 -0800
Committer: Esteban Gutierrez <este...@apache.org>
Committed: Thu Nov 17 12:29:08 2016 -0800
--
.../hbase/regionserver/ConstantSizeRegionSplitPolicy.java | 5 ++---
1 file changed, 2 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/19441937/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ConstantSizeRegionSplitPolicy.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ConstantSizeRegionSplitPolicy.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ConstantSizeRegionSplitPolicy.java
index 836cec5..d915f2e 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ConstantSizeRegionSplitPolicy.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ConstantSizeRegionSplitPolicy.java
@@ -39,7 +39,6 @@ import
org.apache.hadoop.hbase.classification.InterfaceAudience;
@InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.CONFIG)
public class ConstantSizeRegionSplitPolicy extends RegionSplitPolicy {
private static final Random RANDOM = new Random();
- private static final double EPSILON = 1E-6;
private long desiredMaxFileSize;
private double jitterRate;
@@ -60,7 +59,7 @@ public class ConstantSizeRegionSplitPolicy extends
RegionSplitPolicy {
this.jitterRate = (RANDOM.nextFloat() - 0.5D) * jitter;
long jitterValue = (long) (this.desiredMaxFileSize * this.jitterRate);
// make sure the long value won't overflow with jitter
-if (this.jitterRate > EPSILON && jitterValue > (Long.MAX_VALUE -
this.desiredMaxFileSize)) {
+if (this.jitterRate > 0 && jitterValue > (Long.MAX_VALUE -
this.desiredMaxFileSize)) {
this.desiredMaxFileSize = Long.MAX_VALUE;
} else {
this.desiredMaxFileSize += jitterValue;
@@ -94,6 +93,6 @@ public class ConstantSizeRegionSplitPolicy extends
RegionSplitPolicy {
@VisibleForTesting
public boolean positiveJitterRate() {
-return this.jitterRate > EPSILON;
+return this.jitterRate > 0;
}
}
hbase git commit: HBASE-17058 Lower epsilon used for jitter verification from HBASE-15324
Repository: hbase
Updated Branches:
refs/heads/branch-1.3 c3cb42039 -> f4ed43e06
HBASE-17058 Lower epsilon used for jitter verification from HBASE-15324
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/f4ed43e0
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/f4ed43e0
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/f4ed43e0
Branch: refs/heads/branch-1.3
Commit: f4ed43e06108687488ebb161086b00274d172bc0
Parents: c3cb420
Author: Esteban Gutierrez <este...@apache.org>
Authored: Thu Nov 17 11:11:30 2016 -0800
Committer: Esteban Gutierrez <este...@apache.org>
Committed: Thu Nov 17 12:28:22 2016 -0800
--
.../hbase/regionserver/ConstantSizeRegionSplitPolicy.java | 5 ++---
1 file changed, 2 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/f4ed43e0/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ConstantSizeRegionSplitPolicy.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ConstantSizeRegionSplitPolicy.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ConstantSizeRegionSplitPolicy.java
index 836cec5..d915f2e 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ConstantSizeRegionSplitPolicy.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ConstantSizeRegionSplitPolicy.java
@@ -39,7 +39,6 @@ import
org.apache.hadoop.hbase.classification.InterfaceAudience;
@InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.CONFIG)
public class ConstantSizeRegionSplitPolicy extends RegionSplitPolicy {
private static final Random RANDOM = new Random();
- private static final double EPSILON = 1E-6;
private long desiredMaxFileSize;
private double jitterRate;
@@ -60,7 +59,7 @@ public class ConstantSizeRegionSplitPolicy extends
RegionSplitPolicy {
this.jitterRate = (RANDOM.nextFloat() - 0.5D) * jitter;
long jitterValue = (long) (this.desiredMaxFileSize * this.jitterRate);
// make sure the long value won't overflow with jitter
-if (this.jitterRate > EPSILON && jitterValue > (Long.MAX_VALUE -
this.desiredMaxFileSize)) {
+if (this.jitterRate > 0 && jitterValue > (Long.MAX_VALUE -
this.desiredMaxFileSize)) {
this.desiredMaxFileSize = Long.MAX_VALUE;
} else {
this.desiredMaxFileSize += jitterValue;
@@ -94,6 +93,6 @@ public class ConstantSizeRegionSplitPolicy extends
RegionSplitPolicy {
@VisibleForTesting
public boolean positiveJitterRate() {
-return this.jitterRate > EPSILON;
+return this.jitterRate > 0;
}
}
hbase git commit: HBASE-17058 Lower epsilon used for jitter verification from HBASE-15324
Repository: hbase
Updated Branches:
refs/heads/master 5753d18c7 -> 7c6e839f6
HBASE-17058 Lower epsilon used for jitter verification from HBASE-15324
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/7c6e839f
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/7c6e839f
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/7c6e839f
Branch: refs/heads/master
Commit: 7c6e839f6a98cf2c3ed37109318632db13b4a0df
Parents: 5753d18
Author: Esteban Gutierrez <este...@apache.org>
Authored: Thu Nov 17 11:11:30 2016 -0800
Committer: Esteban Gutierrez <este...@apache.org>
Committed: Thu Nov 17 11:11:30 2016 -0800
--
.../hbase/regionserver/ConstantSizeRegionSplitPolicy.java | 5 ++---
1 file changed, 2 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/7c6e839f/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ConstantSizeRegionSplitPolicy.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ConstantSizeRegionSplitPolicy.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ConstantSizeRegionSplitPolicy.java
index 836cec5..d915f2e 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ConstantSizeRegionSplitPolicy.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ConstantSizeRegionSplitPolicy.java
@@ -39,7 +39,6 @@ import
org.apache.hadoop.hbase.classification.InterfaceAudience;
@InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.CONFIG)
public class ConstantSizeRegionSplitPolicy extends RegionSplitPolicy {
private static final Random RANDOM = new Random();
- private static final double EPSILON = 1E-6;
private long desiredMaxFileSize;
private double jitterRate;
@@ -60,7 +59,7 @@ public class ConstantSizeRegionSplitPolicy extends
RegionSplitPolicy {
this.jitterRate = (RANDOM.nextFloat() - 0.5D) * jitter;
long jitterValue = (long) (this.desiredMaxFileSize * this.jitterRate);
// make sure the long value won't overflow with jitter
-if (this.jitterRate > EPSILON && jitterValue > (Long.MAX_VALUE -
this.desiredMaxFileSize)) {
+if (this.jitterRate > 0 && jitterValue > (Long.MAX_VALUE -
this.desiredMaxFileSize)) {
this.desiredMaxFileSize = Long.MAX_VALUE;
} else {
this.desiredMaxFileSize += jitterValue;
@@ -94,6 +93,6 @@ public class ConstantSizeRegionSplitPolicy extends
RegionSplitPolicy {
@VisibleForTesting
public boolean positiveJitterRate() {
-return this.jitterRate > EPSILON;
+return this.jitterRate > 0;
}
}
hbase git commit: HBASE-15324 Jitter may cause desiredMaxFileSize overflow in ConstantSizeRegionSplitPolicy and trigger unexpected split (Yu Li)
Repository: hbase
Updated Branches:
refs/heads/branch-1.1 9e90285e1 -> 9fb5a8608
HBASE-15324 Jitter may cause desiredMaxFileSize overflow in
ConstantSizeRegionSplitPolicy and trigger unexpected split (Yu Li)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/9fb5a860
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/9fb5a860
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/9fb5a860
Branch: refs/heads/branch-1.1
Commit: 9fb5a8608dff761160e217dc3dd5b2dfa9b4875d
Parents: 9e90285
Author: stack <st...@apache.org>
Authored: Wed Mar 30 13:31:09 2016 -0700
Committer: Esteban Gutierrez <este...@apache.org>
Committed: Tue Nov 8 15:59:13 2016 -0800
--
.../ConstantSizeRegionSplitPolicy.java | 24
.../regionserver/TestRegionSplitPolicy.java | 20
2 files changed, 40 insertions(+), 4 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/9fb5a860/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ConstantSizeRegionSplitPolicy.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ConstantSizeRegionSplitPolicy.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ConstantSizeRegionSplitPolicy.java
index 66ef712..836cec5 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ConstantSizeRegionSplitPolicy.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ConstantSizeRegionSplitPolicy.java
@@ -17,13 +17,15 @@
*/
package org.apache.hadoop.hbase.regionserver;
-import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import com.google.common.annotations.VisibleForTesting;
+
+import java.util.Random;
+
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseInterfaceAudience;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HTableDescriptor;
-
-import java.util.Random;
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
/**
* A {@link RegionSplitPolicy} implementation which splits a region
@@ -37,8 +39,10 @@ import java.util.Random;
@InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.CONFIG)
public class ConstantSizeRegionSplitPolicy extends RegionSplitPolicy {
private static final Random RANDOM = new Random();
+ private static final double EPSILON = 1E-6;
private long desiredMaxFileSize;
+ private double jitterRate;
@Override
protected void configureForRegion(HRegion region) {
@@ -53,7 +57,14 @@ public class ConstantSizeRegionSplitPolicy extends
RegionSplitPolicy {
HConstants.DEFAULT_MAX_FILE_SIZE);
}
double jitter = conf.getDouble("hbase.hregion.max.filesize.jitter", 0.25D);
-this.desiredMaxFileSize += (long)(desiredMaxFileSize * (RANDOM.nextFloat()
- 0.5D) * jitter);
+this.jitterRate = (RANDOM.nextFloat() - 0.5D) * jitter;
+long jitterValue = (long) (this.desiredMaxFileSize * this.jitterRate);
+// make sure the long value won't overflow with jitter
+if (this.jitterRate > EPSILON && jitterValue > (Long.MAX_VALUE -
this.desiredMaxFileSize)) {
+ this.desiredMaxFileSize = Long.MAX_VALUE;
+} else {
+ this.desiredMaxFileSize += jitterValue;
+}
}
@Override
@@ -80,4 +91,9 @@ public class ConstantSizeRegionSplitPolicy extends
RegionSplitPolicy {
long getDesiredMaxFileSize() {
return desiredMaxFileSize;
}
+
+ @VisibleForTesting
+ public boolean positiveJitterRate() {
+return this.jitterRate > EPSILON;
+ }
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/9fb5a860/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionSplitPolicy.java
--
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionSplitPolicy.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionSplitPolicy.java
index 1168a3e..624f27d 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionSplitPolicy.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionSplitPolicy.java
@@ -320,4 +320,24 @@ public class TestRegionSplitPolicy {
assertEquals("ijk", Bytes.toString(policy.getSplitPoint()));
}
+ @Test
+ public void testConstantSizePolicyWithJitter() throws IOException {
+conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
+ ConstantSizeRegionSplitPolicy.class.getName());
+htd.setMaxFileSize(Long.MAX_VALUE);
+boolean positiveJitter = false;
+ConstantSizeRegionSplitPolicy policy = null;
+while (!positiveJitt
hbase git commit: HBASE-15324 Jitter may cause desiredMaxFileSize overflow in ConstantSizeRegionSplitPolicy and trigger unexpected split (Yu Li)
Repository: hbase
Updated Branches:
refs/heads/branch-1.2 a3d21ecb3 -> 6b8472038
HBASE-15324 Jitter may cause desiredMaxFileSize overflow in
ConstantSizeRegionSplitPolicy and trigger unexpected split (Yu Li)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/6b847203
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/6b847203
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/6b847203
Branch: refs/heads/branch-1.2
Commit: 6b8472038dd2632d20fdf8d3216c2fe86ee68580
Parents: a3d21ec
Author: stack <st...@apache.org>
Authored: Wed Mar 30 13:31:09 2016 -0700
Committer: Esteban Gutierrez <este...@apache.org>
Committed: Tue Nov 8 15:12:38 2016 -0800
--
.../ConstantSizeRegionSplitPolicy.java | 24
.../regionserver/TestRegionSplitPolicy.java | 20
2 files changed, 40 insertions(+), 4 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/6b847203/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ConstantSizeRegionSplitPolicy.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ConstantSizeRegionSplitPolicy.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ConstantSizeRegionSplitPolicy.java
index 66ef712..836cec5 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ConstantSizeRegionSplitPolicy.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ConstantSizeRegionSplitPolicy.java
@@ -17,13 +17,15 @@
*/
package org.apache.hadoop.hbase.regionserver;
-import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import com.google.common.annotations.VisibleForTesting;
+
+import java.util.Random;
+
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseInterfaceAudience;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HTableDescriptor;
-
-import java.util.Random;
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
/**
* A {@link RegionSplitPolicy} implementation which splits a region
@@ -37,8 +39,10 @@ import java.util.Random;
@InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.CONFIG)
public class ConstantSizeRegionSplitPolicy extends RegionSplitPolicy {
private static final Random RANDOM = new Random();
+ private static final double EPSILON = 1E-6;
private long desiredMaxFileSize;
+ private double jitterRate;
@Override
protected void configureForRegion(HRegion region) {
@@ -53,7 +57,14 @@ public class ConstantSizeRegionSplitPolicy extends
RegionSplitPolicy {
HConstants.DEFAULT_MAX_FILE_SIZE);
}
double jitter = conf.getDouble("hbase.hregion.max.filesize.jitter", 0.25D);
-this.desiredMaxFileSize += (long)(desiredMaxFileSize * (RANDOM.nextFloat()
- 0.5D) * jitter);
+this.jitterRate = (RANDOM.nextFloat() - 0.5D) * jitter;
+long jitterValue = (long) (this.desiredMaxFileSize * this.jitterRate);
+// make sure the long value won't overflow with jitter
+if (this.jitterRate > EPSILON && jitterValue > (Long.MAX_VALUE -
this.desiredMaxFileSize)) {
+ this.desiredMaxFileSize = Long.MAX_VALUE;
+} else {
+ this.desiredMaxFileSize += jitterValue;
+}
}
@Override
@@ -80,4 +91,9 @@ public class ConstantSizeRegionSplitPolicy extends
RegionSplitPolicy {
long getDesiredMaxFileSize() {
return desiredMaxFileSize;
}
+
+ @VisibleForTesting
+ public boolean positiveJitterRate() {
+return this.jitterRate > EPSILON;
+ }
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/6b847203/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionSplitPolicy.java
--
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionSplitPolicy.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionSplitPolicy.java
index 1168a3e..624f27d 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionSplitPolicy.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionSplitPolicy.java
@@ -320,4 +320,24 @@ public class TestRegionSplitPolicy {
assertEquals("ijk", Bytes.toString(policy.getSplitPoint()));
}
+ @Test
+ public void testConstantSizePolicyWithJitter() throws IOException {
+conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
+ ConstantSizeRegionSplitPolicy.class.getName());
+htd.setMaxFileSize(Long.MAX_VALUE);
+boolean positiveJitter = false;
+ConstantSizeRegionSplitPolicy policy = null;
+while (!positiveJitt
hbase git commit: HBASE-16345 RpcRetryingCallerWithReadReplicas#call() should catch some RegionServer Exceptions
Repository: hbase
Updated Branches:
refs/heads/branch-1 ae151334b -> a97aef516
HBASE-16345 RpcRetryingCallerWithReadReplicas#call() should catch some
RegionServer Exceptions
Fix logic for
1). how to handle exception while waiting for reply from the primary replica.
2). handle exception from replicas while waiting for a correct response.
Signed-off-by: Esteban Gutierrez <este...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/a97aef51
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/a97aef51
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/a97aef51
Branch: refs/heads/branch-1
Commit: a97aef51635539ea382699495613ebe1bf89e475
Parents: ae15133
Author: Huaxiang Sun <h...@cloudera.com>
Authored: Wed Oct 19 14:15:31 2016 -0700
Committer: Esteban Gutierrez <este...@apache.org>
Committed: Wed Oct 19 14:22:42 2016 -0700
--
.../client/ResultBoundedCompletionService.java | 118 +--
.../RpcRetryingCallerWithReadReplicas.java | 29 ++--
.../client/ScannerCallableWithReplicas.java | 60 +---
.../hbase/client/TestReplicaWithCluster.java| 143 ++-
4 files changed, 304 insertions(+), 46 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/a97aef51/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ResultBoundedCompletionService.java
--
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ResultBoundedCompletionService.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ResultBoundedCompletionService.java
index 9b32e93..2848c9d 100644
---
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ResultBoundedCompletionService.java
+++
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ResultBoundedCompletionService.java
@@ -18,13 +18,18 @@
*/
package org.apache.hadoop.hbase.client;
+import java.util.ArrayList;
+import java.util.concurrent.CancellationException;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Executor;
import java.util.concurrent.RunnableFuture;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
import org.apache.htrace.Trace;
/**
@@ -32,13 +37,21 @@ import org.apache.htrace.Trace;
* Keeps the list of the futures, and allows to cancel them all.
* This means as well that it can be used for a small set of tasks only.
* Implementation is not Thread safe.
+ *
+ * CompletedTasks is implemented as a queue, the entry is added based on the
time order. I.e,
+ * when the first task completes (whether it is a success or failure), it is
added as a first
+ * entry in the queue, the next completed task is added as a second entry in
the queue, ...
+ * When iterating through the queue, we know it is based on time order. If the
first
+ * completed task succeeds, it is returned. If it is failure, the iteration
goes on until it
+ * finds a success.
*/
@InterfaceAudience.Private
public class ResultBoundedCompletionService {
+ private static final Log LOG =
LogFactory.getLog(ResultBoundedCompletionService.class);
private final RpcRetryingCallerFactory retryingCallerFactory;
private final Executor executor;
private final QueueingFuture[] tasks; // all the tasks
- private volatile QueueingFuture completed = null;
+ private final ArrayList completedTasks; // completed tasks
private volatile boolean cancelled = false;
class QueueingFuture implements RunnableFuture {
@@ -49,12 +62,14 @@ public class ResultBoundedCompletionService {
private final int callTimeout;
private final RpcRetryingCaller retryingCaller;
private boolean resultObtained = false;
+private final int replicaId; // replica id
-public QueueingFuture(RetryingCallable future, int callTimeout) {
+public QueueingFuture(RetryingCallable future, int callTimeout, int id)
{
this.future = future;
this.callTimeout = callTimeout;
this.retryingCaller = retryingCallerFactory.newCaller();
+ this.replicaId = id;
}
@SuppressWarnings("unchecked")
@@ -70,8 +85,8 @@ public class ResultBoundedCompletionService {
} finally {
synchronized (tasks) {
// If this wasn't canceled then store the result.
- if (!cancelled && completed == null) {
-completed = (QueueingFuture) QueueingFuture.this;
+ if (!cancelled) {
+completedTasks.add(QueueingFuture.this);
hbase git commit: HBASE-16345 RpcRetryingCallerWithReadReplicas#call() should catch some RegionServer Exceptions
Repository: hbase
Updated Branches:
refs/heads/master d9ee25e82 -> 72db95388
HBASE-16345 RpcRetryingCallerWithReadReplicas#call() should catch some
RegionServer Exceptions
Fix logic for
1). how to handle exception while waiting for reply from the primary replica.
2). handle exception from replicas while waiting for a correct response.
Signed-off-by: Esteban Gutierrez <este...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/72db9538
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/72db9538
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/72db9538
Branch: refs/heads/master
Commit: 72db95388664fb23314b2e6bb437b961c797f579
Parents: d9ee25e
Author: Huaxiang Sun <h...@cloudera.com>
Authored: Tue Oct 18 14:10:09 2016 -0700
Committer: Esteban Gutierrez <este...@apache.org>
Committed: Wed Oct 19 13:55:43 2016 -0700
--
.../client/ResultBoundedCompletionService.java | 118 --
.../RpcRetryingCallerWithReadReplicas.java | 32 ++--
.../client/ScannerCallableWithReplicas.java | 60 ---
.../hbase/client/TestReplicaWithCluster.java| 155 +--
4 files changed, 310 insertions(+), 55 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/72db9538/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ResultBoundedCompletionService.java
--
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ResultBoundedCompletionService.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ResultBoundedCompletionService.java
index 9b32e93..2848c9d 100644
---
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ResultBoundedCompletionService.java
+++
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ResultBoundedCompletionService.java
@@ -18,13 +18,18 @@
*/
package org.apache.hadoop.hbase.client;
+import java.util.ArrayList;
+import java.util.concurrent.CancellationException;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Executor;
import java.util.concurrent.RunnableFuture;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
import org.apache.htrace.Trace;
/**
@@ -32,13 +37,21 @@ import org.apache.htrace.Trace;
* Keeps the list of the futures, and allows to cancel them all.
* This means as well that it can be used for a small set of tasks only.
* Implementation is not Thread safe.
+ *
+ * CompletedTasks is implemented as a queue, the entry is added based on the
time order. I.e,
+ * when the first task completes (whether it is a success or failure), it is
added as a first
+ * entry in the queue, the next completed task is added as a second entry in
the queue, ...
+ * When iterating through the queue, we know it is based on time order. If the
first
+ * completed task succeeds, it is returned. If it is failure, the iteration
goes on until it
+ * finds a success.
*/
@InterfaceAudience.Private
public class ResultBoundedCompletionService {
+ private static final Log LOG =
LogFactory.getLog(ResultBoundedCompletionService.class);
private final RpcRetryingCallerFactory retryingCallerFactory;
private final Executor executor;
private final QueueingFuture[] tasks; // all the tasks
- private volatile QueueingFuture completed = null;
+ private final ArrayList completedTasks; // completed tasks
private volatile boolean cancelled = false;
class QueueingFuture implements RunnableFuture {
@@ -49,12 +62,14 @@ public class ResultBoundedCompletionService {
private final int callTimeout;
private final RpcRetryingCaller retryingCaller;
private boolean resultObtained = false;
+private final int replicaId; // replica id
-public QueueingFuture(RetryingCallable future, int callTimeout) {
+public QueueingFuture(RetryingCallable future, int callTimeout, int id)
{
this.future = future;
this.callTimeout = callTimeout;
this.retryingCaller = retryingCallerFactory.newCaller();
+ this.replicaId = id;
}
@SuppressWarnings("unchecked")
@@ -70,8 +85,8 @@ public class ResultBoundedCompletionService {
} finally {
synchronized (tasks) {
// If this wasn't canceled then store the result.
- if (!cancelled && completed == null) {
-completed = (QueueingFuture) QueueingFuture.this;
+ if (!cancelled) {
+completedTasks.add(QueueingFuture.this);
}
/
hbase git commit: HBASE-16774 [shell] Add coverage to TestShell when ZooKeeper is not reachable
Repository: hbase
Updated Branches:
refs/heads/master 0d40a52ee -> 5bc518b38
HBASE-16774 [shell] Add coverage to TestShell when ZooKeeper is not reachable
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/5bc518b3
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/5bc518b3
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/5bc518b3
Branch: refs/heads/master
Commit: 5bc518b38717413c844968a29f817b5c99b9a136
Parents: 0d40a52
Author: Esteban Gutierrez <este...@apache.org>
Authored: Wed Oct 5 15:25:54 2016 -0700
Committer: Esteban Gutierrez <este...@apache.org>
Committed: Tue Oct 18 09:08:33 2016 -0700
--
.../hbase/client/ConnectionImplementation.java | 2 +-
.../apache/hadoop/hbase/client/HBaseAdmin.java | 40 ++---
.../apache/hadoop/hbase/client/Registry.java| 4 +-
.../hadoop/hbase/client/ZooKeeperRegistry.java | 7 +-
.../hadoop/hbase/mapred/TableOutputFormat.java | 13 +++
.../hadoop/hbase/client/TestClientTimeouts.java | 8 +-
.../hadoop/hbase/client/TestShellNoCluster.java | 60 +
.../ruby/hbase/test_connection_no_cluster.rb| 46 ++
.../src/test/ruby/no_cluster_tests_runner.rb| 92
9 files changed, 247 insertions(+), 25 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/5bc518b3/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionImplementation.java
--
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionImplementation.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionImplementation.java
index 922168d..53eb522 100644
---
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionImplementation.java
+++
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionImplementation.java
@@ -439,7 +439,7 @@ class ConnectionImplementation implements
ClusterConnection, Closeable {
protected String clusterId = null;
- protected void retrieveClusterId() throws IOException {
+ protected void retrieveClusterId() {
if (clusterId != null) {
return;
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/5bc518b3/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
--
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
index e7f6929..51d07e3 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
@@ -2112,10 +2112,12 @@ public class HBaseAdmin implements Admin {
/**
* Is HBase available? Throw an exception if not.
* @param conf system configuration
- * @throws ZooKeeperConnectionException if unable to connect to zookeeper]
+ * @throws MasterNotRunningException if the master is not running.
+ * @throws ZooKeeperConnectionException if unable to connect to zookeeper.
+ * // TODO do not expose ZKConnectionException.
*/
public static void available(final Configuration conf)
- throws ZooKeeperConnectionException, InterruptedIOException {
+ throws MasterNotRunningException, ZooKeeperConnectionException, IOException {
Configuration copyOfConf = HBaseConfiguration.create(conf);
// We set it to make it fail as soon as possible if HBase is not available
copyOfConf.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 1);
@@ -2124,19 +2126,29 @@ public class HBaseAdmin implements Admin {
// Check ZK first.
// If the connection exists, we may have a connection to ZK that does not
work anymore
try (ClusterConnection connection =
- (ClusterConnection)
ConnectionFactory.createConnection(copyOfConf);
- ZooKeeperKeepAliveConnection zkw = ((ConnectionImplementation)
connection).
- getKeepAliveZooKeeperWatcher();) {
- // This is NASTY. FIX Dependent on internal implementation! TODO
-
zkw.getRecoverableZooKeeper().getZooKeeper().exists(zkw.znodePaths.baseZNode,
false);
+(ClusterConnection) ConnectionFactory.createConnection(copyOfConf)) {
+ // Check ZK first.
+ // If the connection exists, we may have a connection to ZK that does
not work anymore
+ ZooKeeperKeepAliveConnection zkw = null;
+ try {
+// This is NASTY. FIX Dependent on internal implementation! TODO
+zkw = ((ConnectionImplementation) connection)
+.getKeepAliveZooKeeperWatcher();
+
zkw.getRecoverableZooKeeper().getZooKeeper().exists(zkw.znodePaths.baseZNode,
false);
+ } catch (IOException e) {
+
hbase git commit: HBASE-16450 Shell tool to dump replication queues
Repository: hbase
Updated Branches:
refs/heads/branch-1.3 b30ad10a9 -> c3ccbbc8d
HBASE-16450 Shell tool to dump replication queues
New tool to dump existing replication peers, configurations and
queues when using HBase Replication. The tool provides two flags:
--distributed This flag will poll each RS for information about
the replication queues being processed on this RS.
By default this is not enabled and the information
about the replication queues and configuration will
be obtained from ZooKeeper.
--hdfs When --distributed is used, this flag will attempt
to calculate the total size of the WAL files used
by the replication queues. Since its possible that
multiple peers can be configured this value can be
overestimated.
Signed-off-by: Matteo Bertozzi <matteo.berto...@cloudera.com>
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/c3ccbbc8
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/c3ccbbc8
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/c3ccbbc8
Branch: refs/heads/branch-1.3
Commit: c3ccbbc8dc2a8fbbdf54dc82458aa7bb477a8a6a
Parents: b30ad10
Author: Esteban Gutierrez <este...@apache.org>
Authored: Mon Aug 22 19:53:29 2016 -0700
Committer: Esteban Gutierrez <este...@apache.org>
Committed: Mon Aug 22 19:53:44 2016 -0700
--
.../replication/ReplicationQueuesZKImpl.java| 2 +-
.../apache/hadoop/hbase/zookeeper/ZKUtil.java | 11 +
.../regionserver/DumpReplicationQueues.java | 355 +++
3 files changed, 367 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/c3ccbbc8/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueuesZKImpl.java
--
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueuesZKImpl.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueuesZKImpl.java
index 0833bca..beeaea0 100644
---
a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueuesZKImpl.java
+++
b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueuesZKImpl.java
@@ -161,7 +161,7 @@ public class ReplicationQueuesZKImpl extends
ReplicationStateZKBase implements R
return ZKUtil.parseWALPositionFrom(bytes);
} catch (DeserializationException de) {
LOG.warn("Failed to parse WALPosition for queueId=" + queueId + " and
wal=" + filename
- + "znode content, continuing.");
+ + " znode content, continuing.");
}
// if we can not parse the position, start at the beginning of the wal file
// again
http://git-wip-us.apache.org/repos/asf/hbase/blob/c3ccbbc8/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java
--
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java
index ff837cb..66ddb62 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java
@@ -1825,6 +1825,17 @@ public class ZKUtil {
}
}
+ /**
+ * Returns a string with replication znodes and position of the replication
log
+ * @param zkw
+ * @return aq string of replication znodes and log positions
+ */
+ public static String getReplicationZnodesDump(ZooKeeperWatcher zkw) throws
KeeperException {
+StringBuilder sb = new StringBuilder();
+getReplicationZnodesDump(zkw, sb);
+return sb.toString();
+ }
+
private static void appendRSZnodes(ZooKeeperWatcher zkw, String znode,
StringBuilder sb)
throws KeeperException {
List stack = new LinkedList();
http://git-wip-us.apache.org/repos/asf/hbase/blob/c3ccbbc8/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/DumpReplicationQueues.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/DumpReplicationQueues.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/DumpReplicationQueues.java
new file mode 100644
index 000..bf38d6f
--- /dev/null
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/DumpReplicationQueues.java
@@ -0,0 +1,355 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See
hbase git commit: HBASE-16450 Shell tool to dump replication queues
Repository: hbase
Updated Branches:
refs/heads/master ae42a934d -> abc64fa69
HBASE-16450 Shell tool to dump replication queues
New tool to dump existing replication peers, configurations and
queues when using HBase Replication. The tool provides two flags:
--distributed This flag will poll each RS for information about
the replication queues being processed on this RS.
By default this is not enabled and the information
about the replication queues and configuration will
be obtained from ZooKeeper.
--hdfs When --distributed is used, this flag will attempt
to calculate the total size of the WAL files used
by the replication queues. Since its possible that
multiple peers can be configured this value can be
overestimated.
Signed-off-by: Matteo Bertozzi <matteo.berto...@cloudera.com>
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/abc64fa6
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/abc64fa6
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/abc64fa6
Branch: refs/heads/master
Commit: abc64fa69fcdff41611e1b4bd64e67f54617a28a
Parents: ae42a93
Author: Esteban Gutierrez <este...@apache.org>
Authored: Thu Aug 18 15:28:18 2016 -0700
Committer: Esteban Gutierrez <este...@apache.org>
Committed: Mon Aug 22 19:50:10 2016 -0700
--
.../replication/ReplicationQueuesZKImpl.java| 2 +-
.../apache/hadoop/hbase/zookeeper/ZKUtil.java | 11 +
.../regionserver/DumpReplicationQueues.java | 356 +++
3 files changed, 368 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/abc64fa6/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueuesZKImpl.java
--
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueuesZKImpl.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueuesZKImpl.java
index c1e85cb..1c579ab 100644
---
a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueuesZKImpl.java
+++
b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueuesZKImpl.java
@@ -165,7 +165,7 @@ public class ReplicationQueuesZKImpl extends
ReplicationStateZKBase implements R
return ZKUtil.parseWALPositionFrom(bytes);
} catch (DeserializationException de) {
LOG.warn("Failed to parse WALPosition for queueId=" + queueId + " and
wal=" + filename
- + "znode content, continuing.");
+ + " znode content, continuing.");
}
// if we can not parse the position, start at the beginning of the wal file
// again
http://git-wip-us.apache.org/repos/asf/hbase/blob/abc64fa6/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java
--
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java
index f5b720e..f6914ae 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java
@@ -1823,6 +1823,17 @@ public class ZKUtil {
}
}
+ /**
+ * Returns a string with replication znodes and position of the replication
log
+ * @param zkw
+ * @return aq string of replication znodes and log positions
+ */
+ public static String getReplicationZnodesDump(ZooKeeperWatcher zkw) throws
KeeperException {
+StringBuilder sb = new StringBuilder();
+getReplicationZnodesDump(zkw, sb);
+return sb.toString();
+ }
+
private static void appendRSZnodes(ZooKeeperWatcher zkw, String znode,
StringBuilder sb)
throws KeeperException {
List stack = new LinkedList();
http://git-wip-us.apache.org/repos/asf/hbase/blob/abc64fa6/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/DumpReplicationQueues.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/DumpReplicationQueues.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/DumpReplicationQueues.java
new file mode 100644
index 000..5305149
--- /dev/null
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/DumpReplicationQueues.java
@@ -0,0 +1,356 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE f
hbase git commit: HBASE-16379 [replication] Minor improvement to replication/copy_tables_desc.rb
Repository: hbase
Updated Branches:
refs/heads/branch-1 89e337309 -> 429f398c0
HBASE-16379 [replication] Minor improvement to replication/copy_tables_desc.rb
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/429f398c
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/429f398c
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/429f398c
Branch: refs/heads/branch-1
Commit: 429f398c0769bfc76eddefcaa0230f6c3de40933
Parents: 89e3373
Author: Esteban Gutierrez <este...@apache.org>
Authored: Mon Aug 8 16:10:29 2016 -0700
Committer: Esteban Gutierrez <este...@apache.org>
Committed: Wed Aug 10 11:16:53 2016 -0700
--
bin/replication/copy_tables_desc.rb | 41 +---
1 file changed, 33 insertions(+), 8 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/429f398c/bin/replication/copy_tables_desc.rb
--
diff --git a/bin/replication/copy_tables_desc.rb
b/bin/replication/copy_tables_desc.rb
index bc70031..07b17a8 100644
--- a/bin/replication/copy_tables_desc.rb
+++ b/bin/replication/copy_tables_desc.rb
@@ -27,7 +27,6 @@ include Java
import org.apache.commons.logging.LogFactory
import org.apache.hadoop.hbase.HBaseConfiguration
import org.apache.hadoop.hbase.HConstants
-import org.apache.hadoop.hbase.EmptyWatcher
import org.apache.hadoop.hbase.client.HBaseAdmin
import org.apache.hadoop.hbase.HTableDescriptor
import org.apache.hadoop.conf.Configuration
@@ -37,11 +36,32 @@ NAME = "copy_tables_desc"
# Print usage for this script
def usage
- puts 'Usage: %s.rb master_zookeeper.quorum.peers:clientport:znode_parent
slave_zookeeper.quorum.peers:clientport:znode_parent' % NAME
+ puts 'Usage: %s.rb master_zookeeper.quorum.peers:clientport:znode_parent
slave_zookeeper.quorum.peers:clientport:znode_parent
[table1,table2,table3,...]' % NAME
exit!
end
-if ARGV.size != 2
+def copy (src, dst, table)
+ # verify if table exists in source cluster
+ begin
+t = src.getTableDescriptor(table.to_java_bytes)
+ rescue org.apache.hadoop.hbase.TableNotFoundException
+puts "Source table \"%s\" doesn't exist, skipping." % table
+return
+ end
+
+ # verify if table *doesn't* exists in the target cluster
+ begin
+dst.createTable(t)
+ rescue org.apache.hadoop.hbase.TableExistsException
+puts "Destination table \"%s\" exists in remote cluster, skipping." % table
+return
+ end
+
+ puts "Schema for table \"%s\" was succesfully copied to remote cluster." %
table
+end
+
+
+if ARGV.size < 2 || ARGV.size > 3
usage
end
@@ -51,6 +71,8 @@ parts1 = ARGV[0].split(":")
parts2 = ARGV[1].split(":")
+parts3 = ARGV[2].split(",") unless ARGV[2].nil?
+
c1 = HBaseConfiguration.create()
c1.set(HConstants::ZOOKEEPER_QUORUM, parts1[0])
c1.set("hbase.zookeeper.property.clientPort", parts1[1])
@@ -65,9 +87,12 @@ c2.set(HConstants::ZOOKEEPER_ZNODE_PARENT, parts2[2])
admin2 = HBaseAdmin.new(c2)
-for t in admin1.listTables()
- admin2.createTable(t)
+if parts3.nil?
+ admin1.listTableNames().each do |t|
+copy(admin1, admin2, t.nameAsString())
+ end
+else
+ parts3.each do |t|
+copy(admin1, admin2, t)
+ end
end
-
-
-puts "All descriptions were copied"
hbase git commit: HBASE-16379 [replication] Minor improvement to replication/copy_tables_desc.rb
Repository: hbase
Updated Branches:
refs/heads/branch-1.1 e5697bdb1 -> 3259f03a5
HBASE-16379 [replication] Minor improvement to replication/copy_tables_desc.rb
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/3259f03a
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/3259f03a
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/3259f03a
Branch: refs/heads/branch-1.1
Commit: 3259f03a592b42abe05ae1b89540b15ee4245cbe
Parents: e5697bd
Author: Esteban Gutierrez <este...@apache.org>
Authored: Mon Aug 8 16:10:29 2016 -0700
Committer: Esteban Gutierrez <este...@apache.org>
Committed: Wed Aug 10 11:14:54 2016 -0700
--
bin/replication/copy_tables_desc.rb | 41 +---
1 file changed, 33 insertions(+), 8 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/3259f03a/bin/replication/copy_tables_desc.rb
--
diff --git a/bin/replication/copy_tables_desc.rb
b/bin/replication/copy_tables_desc.rb
index bc70031..07b17a8 100644
--- a/bin/replication/copy_tables_desc.rb
+++ b/bin/replication/copy_tables_desc.rb
@@ -27,7 +27,6 @@ include Java
import org.apache.commons.logging.LogFactory
import org.apache.hadoop.hbase.HBaseConfiguration
import org.apache.hadoop.hbase.HConstants
-import org.apache.hadoop.hbase.EmptyWatcher
import org.apache.hadoop.hbase.client.HBaseAdmin
import org.apache.hadoop.hbase.HTableDescriptor
import org.apache.hadoop.conf.Configuration
@@ -37,11 +36,32 @@ NAME = "copy_tables_desc"
# Print usage for this script
def usage
- puts 'Usage: %s.rb master_zookeeper.quorum.peers:clientport:znode_parent
slave_zookeeper.quorum.peers:clientport:znode_parent' % NAME
+ puts 'Usage: %s.rb master_zookeeper.quorum.peers:clientport:znode_parent
slave_zookeeper.quorum.peers:clientport:znode_parent
[table1,table2,table3,...]' % NAME
exit!
end
-if ARGV.size != 2
+def copy (src, dst, table)
+ # verify if table exists in source cluster
+ begin
+t = src.getTableDescriptor(table.to_java_bytes)
+ rescue org.apache.hadoop.hbase.TableNotFoundException
+puts "Source table \"%s\" doesn't exist, skipping." % table
+return
+ end
+
+ # verify if table *doesn't* exists in the target cluster
+ begin
+dst.createTable(t)
+ rescue org.apache.hadoop.hbase.TableExistsException
+puts "Destination table \"%s\" exists in remote cluster, skipping." % table
+return
+ end
+
+ puts "Schema for table \"%s\" was succesfully copied to remote cluster." %
table
+end
+
+
+if ARGV.size < 2 || ARGV.size > 3
usage
end
@@ -51,6 +71,8 @@ parts1 = ARGV[0].split(":")
parts2 = ARGV[1].split(":")
+parts3 = ARGV[2].split(",") unless ARGV[2].nil?
+
c1 = HBaseConfiguration.create()
c1.set(HConstants::ZOOKEEPER_QUORUM, parts1[0])
c1.set("hbase.zookeeper.property.clientPort", parts1[1])
@@ -65,9 +87,12 @@ c2.set(HConstants::ZOOKEEPER_ZNODE_PARENT, parts2[2])
admin2 = HBaseAdmin.new(c2)
-for t in admin1.listTables()
- admin2.createTable(t)
+if parts3.nil?
+ admin1.listTableNames().each do |t|
+copy(admin1, admin2, t.nameAsString())
+ end
+else
+ parts3.each do |t|
+copy(admin1, admin2, t)
+ end
end
-
-
-puts "All descriptions were copied"
hbase git commit: HBASE-16379 [replication] Minor improvement to replication/copy_tables_desc.rb
Repository: hbase
Updated Branches:
refs/heads/branch-1.2 188963cb8 -> eef304b88
HBASE-16379 [replication] Minor improvement to replication/copy_tables_desc.rb
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/eef304b8
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/eef304b8
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/eef304b8
Branch: refs/heads/branch-1.2
Commit: eef304b8874c5647ee6b52ba3690581f57eb9b8e
Parents: 188963c
Author: Esteban Gutierrez <este...@apache.org>
Authored: Mon Aug 8 16:10:29 2016 -0700
Committer: Esteban Gutierrez <este...@apache.org>
Committed: Wed Aug 10 11:14:23 2016 -0700
--
bin/replication/copy_tables_desc.rb | 41 +---
1 file changed, 33 insertions(+), 8 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/eef304b8/bin/replication/copy_tables_desc.rb
--
diff --git a/bin/replication/copy_tables_desc.rb
b/bin/replication/copy_tables_desc.rb
index bc70031..07b17a8 100644
--- a/bin/replication/copy_tables_desc.rb
+++ b/bin/replication/copy_tables_desc.rb
@@ -27,7 +27,6 @@ include Java
import org.apache.commons.logging.LogFactory
import org.apache.hadoop.hbase.HBaseConfiguration
import org.apache.hadoop.hbase.HConstants
-import org.apache.hadoop.hbase.EmptyWatcher
import org.apache.hadoop.hbase.client.HBaseAdmin
import org.apache.hadoop.hbase.HTableDescriptor
import org.apache.hadoop.conf.Configuration
@@ -37,11 +36,32 @@ NAME = "copy_tables_desc"
# Print usage for this script
def usage
- puts 'Usage: %s.rb master_zookeeper.quorum.peers:clientport:znode_parent
slave_zookeeper.quorum.peers:clientport:znode_parent' % NAME
+ puts 'Usage: %s.rb master_zookeeper.quorum.peers:clientport:znode_parent
slave_zookeeper.quorum.peers:clientport:znode_parent
[table1,table2,table3,...]' % NAME
exit!
end
-if ARGV.size != 2
+def copy (src, dst, table)
+ # verify if table exists in source cluster
+ begin
+t = src.getTableDescriptor(table.to_java_bytes)
+ rescue org.apache.hadoop.hbase.TableNotFoundException
+puts "Source table \"%s\" doesn't exist, skipping." % table
+return
+ end
+
+ # verify if table *doesn't* exists in the target cluster
+ begin
+dst.createTable(t)
+ rescue org.apache.hadoop.hbase.TableExistsException
+puts "Destination table \"%s\" exists in remote cluster, skipping." % table
+return
+ end
+
+ puts "Schema for table \"%s\" was succesfully copied to remote cluster." %
table
+end
+
+
+if ARGV.size < 2 || ARGV.size > 3
usage
end
@@ -51,6 +71,8 @@ parts1 = ARGV[0].split(":")
parts2 = ARGV[1].split(":")
+parts3 = ARGV[2].split(",") unless ARGV[2].nil?
+
c1 = HBaseConfiguration.create()
c1.set(HConstants::ZOOKEEPER_QUORUM, parts1[0])
c1.set("hbase.zookeeper.property.clientPort", parts1[1])
@@ -65,9 +87,12 @@ c2.set(HConstants::ZOOKEEPER_ZNODE_PARENT, parts2[2])
admin2 = HBaseAdmin.new(c2)
-for t in admin1.listTables()
- admin2.createTable(t)
+if parts3.nil?
+ admin1.listTableNames().each do |t|
+copy(admin1, admin2, t.nameAsString())
+ end
+else
+ parts3.each do |t|
+copy(admin1, admin2, t)
+ end
end
-
-
-puts "All descriptions were copied"
hbase git commit: HBASE-16379 [replication] Minor improvement to replication/copy_tables_desc.rb
Repository: hbase
Updated Branches:
refs/heads/branch-1.3 2759b4a4a -> 66c23e370
HBASE-16379 [replication] Minor improvement to replication/copy_tables_desc.rb
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/66c23e37
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/66c23e37
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/66c23e37
Branch: refs/heads/branch-1.3
Commit: 66c23e37003a97778840f12cb25f65f51ce677fe
Parents: 2759b4a
Author: Esteban Gutierrez <este...@apache.org>
Authored: Mon Aug 8 16:10:29 2016 -0700
Committer: Esteban Gutierrez <este...@apache.org>
Committed: Wed Aug 10 11:13:42 2016 -0700
--
bin/replication/copy_tables_desc.rb | 41 +---
1 file changed, 33 insertions(+), 8 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/66c23e37/bin/replication/copy_tables_desc.rb
--
diff --git a/bin/replication/copy_tables_desc.rb
b/bin/replication/copy_tables_desc.rb
index bc70031..07b17a8 100644
--- a/bin/replication/copy_tables_desc.rb
+++ b/bin/replication/copy_tables_desc.rb
@@ -27,7 +27,6 @@ include Java
import org.apache.commons.logging.LogFactory
import org.apache.hadoop.hbase.HBaseConfiguration
import org.apache.hadoop.hbase.HConstants
-import org.apache.hadoop.hbase.EmptyWatcher
import org.apache.hadoop.hbase.client.HBaseAdmin
import org.apache.hadoop.hbase.HTableDescriptor
import org.apache.hadoop.conf.Configuration
@@ -37,11 +36,32 @@ NAME = "copy_tables_desc"
# Print usage for this script
def usage
- puts 'Usage: %s.rb master_zookeeper.quorum.peers:clientport:znode_parent
slave_zookeeper.quorum.peers:clientport:znode_parent' % NAME
+ puts 'Usage: %s.rb master_zookeeper.quorum.peers:clientport:znode_parent
slave_zookeeper.quorum.peers:clientport:znode_parent
[table1,table2,table3,...]' % NAME
exit!
end
-if ARGV.size != 2
+def copy (src, dst, table)
+ # verify if table exists in source cluster
+ begin
+t = src.getTableDescriptor(table.to_java_bytes)
+ rescue org.apache.hadoop.hbase.TableNotFoundException
+puts "Source table \"%s\" doesn't exist, skipping." % table
+return
+ end
+
+ # verify if table *doesn't* exists in the target cluster
+ begin
+dst.createTable(t)
+ rescue org.apache.hadoop.hbase.TableExistsException
+puts "Destination table \"%s\" exists in remote cluster, skipping." % table
+return
+ end
+
+ puts "Schema for table \"%s\" was succesfully copied to remote cluster." %
table
+end
+
+
+if ARGV.size < 2 || ARGV.size > 3
usage
end
@@ -51,6 +71,8 @@ parts1 = ARGV[0].split(":")
parts2 = ARGV[1].split(":")
+parts3 = ARGV[2].split(",") unless ARGV[2].nil?
+
c1 = HBaseConfiguration.create()
c1.set(HConstants::ZOOKEEPER_QUORUM, parts1[0])
c1.set("hbase.zookeeper.property.clientPort", parts1[1])
@@ -65,9 +87,12 @@ c2.set(HConstants::ZOOKEEPER_ZNODE_PARENT, parts2[2])
admin2 = HBaseAdmin.new(c2)
-for t in admin1.listTables()
- admin2.createTable(t)
+if parts3.nil?
+ admin1.listTableNames().each do |t|
+copy(admin1, admin2, t.nameAsString())
+ end
+else
+ parts3.each do |t|
+copy(admin1, admin2, t)
+ end
end
-
-
-puts "All descriptions were copied"
hbase git commit: HBASE-16379 [replication] Minor improvement to replication/copy_tables_desc.rb
Repository: hbase
Updated Branches:
refs/heads/master 45bb6180a -> b3888eadf
HBASE-16379 [replication] Minor improvement to replication/copy_tables_desc.rb
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/b3888ead
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/b3888ead
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/b3888ead
Branch: refs/heads/master
Commit: b3888eadf8ac081729385b434f8da626d02d031a
Parents: 45bb618
Author: Esteban Gutierrez <este...@apache.org>
Authored: Mon Aug 8 16:10:29 2016 -0700
Committer: Esteban Gutierrez <este...@apache.org>
Committed: Wed Aug 10 11:00:23 2016 -0700
--
bin/replication/copy_tables_desc.rb | 39 +++-
1 file changed, 33 insertions(+), 6 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/b3888ead/bin/replication/copy_tables_desc.rb
--
diff --git a/bin/replication/copy_tables_desc.rb
b/bin/replication/copy_tables_desc.rb
index 8a6c670..a7cc3a3 100644
--- a/bin/replication/copy_tables_desc.rb
+++ b/bin/replication/copy_tables_desc.rb
@@ -27,7 +27,6 @@ include Java
import org.apache.commons.logging.LogFactory
import org.apache.hadoop.hbase.HBaseConfiguration
import org.apache.hadoop.hbase.HConstants
-import org.apache.hadoop.hbase.EmptyWatcher
import org.apache.hadoop.hbase.client.HBaseAdmin
import org.apache.hadoop.hbase.HTableDescriptor
import org.apache.hadoop.conf.Configuration
@@ -38,11 +37,32 @@ NAME = "copy_tables_desc"
# Print usage for this script
def usage
- puts 'Usage: %s.rb master_zookeeper.quorum.peers:clientport:znode_parent
slave_zookeeper.quorum.peers:clientport:znode_parent' % NAME
+ puts 'Usage: %s.rb master_zookeeper.quorum.peers:clientport:znode_parent
slave_zookeeper.quorum.peers:clientport:znode_parent
[table1,table2,table3,...]' % NAME
exit!
end
-if ARGV.size != 2
+def copy (src, dst, table)
+ # verify if table exists in source cluster
+ begin
+t = src.getTableDescriptor(table.to_java_bytes)
+ rescue org.apache.hadoop.hbase.TableNotFoundException
+puts "Source table \"%s\" doesn't exist, skipping." % table
+return
+ end
+
+ # verify if table *doesn't* exists in the target cluster
+ begin
+dst.createTable(t)
+ rescue org.apache.hadoop.hbase.TableExistsException
+puts "Destination table \"%s\" exists in remote cluster, skipping." % table
+return
+ end
+
+ puts "Schema for table \"%s\" was succesfully copied to remote cluster." %
table
+end
+
+
+if ARGV.size < 2 || ARGV.size > 3
usage
end
@@ -52,6 +72,8 @@ parts1 = ARGV[0].split(":")
parts2 = ARGV[1].split(":")
+parts3 = ARGV[2].split(",") unless ARGV[2].nil?
+
c1 = HBaseConfiguration.create()
c1.set(HConstants::ZOOKEEPER_QUORUM, parts1[0])
c1.set("hbase.zookeeper.property.clientPort", parts1[1])
@@ -68,11 +90,16 @@ c2.set(HConstants::ZOOKEEPER_ZNODE_PARENT, parts2[2])
connection2 = ConnectionFactory.createConnection(c2)
admin2 = connection2.getAdmin()
-for t in admin1.listTables()
- admin2.createTable(t)
+if parts3.nil?
+ admin1.listTableNames().each do |t|
+copy(admin1, admin2, t.nameAsString())
+ end
+else
+ parts3.each do |t|
+copy(admin1, admin2, t)
+ end
end
-puts "All descriptions were copied"
admin1.close()
admin2.close()
connection1.close()
hbase git commit: HBASE-14500 Remove load of deprecated MOB ruby scripts after HBASE-14227
Repository: hbase
Updated Branches:
refs/heads/master cfce91e51 -> 54b86b339
HBASE-14500 Remove load of deprecated MOB ruby scripts after HBASE-14227
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/54b86b33
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/54b86b33
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/54b86b33
Branch: refs/heads/master
Commit: 54b86b33940b57140d665a02156cfcf14ba792a2
Parents: cfce91e
Author: Esteban Gutierrez <este...@apache.org>
Authored: Mon Sep 28 10:49:08 2015 -0700
Committer: Esteban Gutierrez <este...@apache.org>
Committed: Mon Sep 28 10:49:08 2015 -0700
--
hbase-shell/src/main/ruby/shell.rb | 2 --
1 file changed, 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/54b86b33/hbase-shell/src/main/ruby/shell.rb
--
diff --git a/hbase-shell/src/main/ruby/shell.rb
b/hbase-shell/src/main/ruby/shell.rb
index fa19c26..a8f6ae6 100644
--- a/hbase-shell/src/main/ruby/shell.rb
+++ b/hbase-shell/src/main/ruby/shell.rb
@@ -329,8 +329,6 @@ Shell.load_command_group(
catalogjanitor_enabled
compact_rs
trace
-compact_mob
-major_compact_mob
],
# TODO remove older hlog_roll command
:aliases => {
hbase git commit: HBASE-14354 Minor improvements for usage of the mlock agent
Repository: hbase
Updated Branches:
refs/heads/branch-1.2 af7e34f99 -> 7c287597a
HBASE-14354 Minor improvements for usage of the mlock agent
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/7c287597
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/7c287597
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/7c287597
Branch: refs/heads/branch-1.2
Commit: 7c287597accefc1dea13af9760735b87fa3aeb3e
Parents: af7e34f
Author: Esteban Gutierrez <este...@apache.org>
Authored: Tue Sep 1 18:36:39 2015 -0700
Committer: Esteban Gutierrez <este...@apache.org>
Committed: Tue Sep 1 18:45:18 2015 -0700
--
bin/hbase-config.sh | 14 ++
1 file changed, 6 insertions(+), 8 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/7c287597/bin/hbase-config.sh
--
diff --git a/bin/hbase-config.sh b/bin/hbase-config.sh
index 5c518b1..c113adb 100644
--- a/bin/hbase-config.sh
+++ b/bin/hbase-config.sh
@@ -98,22 +98,20 @@ if [ -z "$HBASE_ENV_INIT" ] && [ -f
"${HBASE_CONF_DIR}/hbase-env.sh" ]; then
export HBASE_ENV_INIT="true"
fi
-# Set default value for regionserver uid if not present
-if [ -z "$HBASE_REGIONSERVER_UID" ]; then
- HBASE_REGIONSERVER_UID="hbase"
-fi
-
# Verify if hbase has the mlock agent
if [ "$HBASE_REGIONSERVER_MLOCK" = "true" ]; then
- MLOCK_AGENT="$HBASE_HOME/native/libmlockall_agent.so"
+ MLOCK_AGENT="$HBASE_HOME/lib/native/libmlockall_agent.so"
if [ ! -f "$MLOCK_AGENT" ]; then
cat 1>&2 <
hbase git commit: HBASE-14354 Minor improvements for usage of the mlock agent
Repository: hbase
Updated Branches:
refs/heads/branch-1.1 026330588 -> f0891a6a0
HBASE-14354 Minor improvements for usage of the mlock agent
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/f0891a6a
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/f0891a6a
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/f0891a6a
Branch: refs/heads/branch-1.1
Commit: f0891a6a0c49ea0b3c3c394753dadef5617fedaa
Parents: 0263305
Author: Esteban Gutierrez <este...@apache.org>
Authored: Tue Sep 1 18:36:39 2015 -0700
Committer: Esteban Gutierrez <este...@apache.org>
Committed: Tue Sep 1 18:44:45 2015 -0700
--
bin/hbase-config.sh | 14 ++
1 file changed, 6 insertions(+), 8 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/f0891a6a/bin/hbase-config.sh
--
diff --git a/bin/hbase-config.sh b/bin/hbase-config.sh
index 5c518b1..c113adb 100644
--- a/bin/hbase-config.sh
+++ b/bin/hbase-config.sh
@@ -98,22 +98,20 @@ if [ -z "$HBASE_ENV_INIT" ] && [ -f
"${HBASE_CONF_DIR}/hbase-env.sh" ]; then
export HBASE_ENV_INIT="true"
fi
-# Set default value for regionserver uid if not present
-if [ -z "$HBASE_REGIONSERVER_UID" ]; then
- HBASE_REGIONSERVER_UID="hbase"
-fi
-
# Verify if hbase has the mlock agent
if [ "$HBASE_REGIONSERVER_MLOCK" = "true" ]; then
- MLOCK_AGENT="$HBASE_HOME/native/libmlockall_agent.so"
+ MLOCK_AGENT="$HBASE_HOME/lib/native/libmlockall_agent.so"
if [ ! -f "$MLOCK_AGENT" ]; then
cat 1>&2 <
hbase git commit: HBASE-14354 Minor improvements for usage of the mlock agent
Repository: hbase
Updated Branches:
refs/heads/branch-1 a31adde0b -> 1b5fb8193
HBASE-14354 Minor improvements for usage of the mlock agent
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/1b5fb819
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/1b5fb819
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/1b5fb819
Branch: refs/heads/branch-1
Commit: 1b5fb8193e0b0ffb817658fff182b84ff6e84be8
Parents: a31adde
Author: Esteban Gutierrez <este...@apache.org>
Authored: Tue Sep 1 18:36:39 2015 -0700
Committer: Esteban Gutierrez <este...@apache.org>
Committed: Tue Sep 1 18:38:32 2015 -0700
--
bin/hbase-config.sh | 14 ++
1 file changed, 6 insertions(+), 8 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/1b5fb819/bin/hbase-config.sh
--
diff --git a/bin/hbase-config.sh b/bin/hbase-config.sh
index 5c518b1..c113adb 100644
--- a/bin/hbase-config.sh
+++ b/bin/hbase-config.sh
@@ -98,22 +98,20 @@ if [ -z "$HBASE_ENV_INIT" ] && [ -f
"${HBASE_CONF_DIR}/hbase-env.sh" ]; then
export HBASE_ENV_INIT="true"
fi
-# Set default value for regionserver uid if not present
-if [ -z "$HBASE_REGIONSERVER_UID" ]; then
- HBASE_REGIONSERVER_UID="hbase"
-fi
-
# Verify if hbase has the mlock agent
if [ "$HBASE_REGIONSERVER_MLOCK" = "true" ]; then
- MLOCK_AGENT="$HBASE_HOME/native/libmlockall_agent.so"
+ MLOCK_AGENT="$HBASE_HOME/lib/native/libmlockall_agent.so"
if [ ! -f "$MLOCK_AGENT" ]; then
cat 1>&2 <
hbase git commit: HBASE-14354 Minor improvements for usage of the mlock agent
Repository: hbase
Updated Branches:
refs/heads/master 5bb36f159 -> f8dd99d73
HBASE-14354 Minor improvements for usage of the mlock agent
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/f8dd99d7
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/f8dd99d7
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/f8dd99d7
Branch: refs/heads/master
Commit: f8dd99d7380e5eafae62a9f0c526ba24f98eb2e5
Parents: 5bb36f1
Author: Esteban Gutierrez <este...@apache.org>
Authored: Tue Sep 1 18:36:39 2015 -0700
Committer: Esteban Gutierrez <este...@apache.org>
Committed: Tue Sep 1 18:37:34 2015 -0700
--
bin/hbase-config.sh | 14 ++
1 file changed, 6 insertions(+), 8 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/f8dd99d7/bin/hbase-config.sh
--
diff --git a/bin/hbase-config.sh b/bin/hbase-config.sh
index 5c518b1..c113adb 100644
--- a/bin/hbase-config.sh
+++ b/bin/hbase-config.sh
@@ -98,22 +98,20 @@ if [ -z "$HBASE_ENV_INIT" ] && [ -f
"${HBASE_CONF_DIR}/hbase-env.sh" ]; then
export HBASE_ENV_INIT="true"
fi
-# Set default value for regionserver uid if not present
-if [ -z "$HBASE_REGIONSERVER_UID" ]; then
- HBASE_REGIONSERVER_UID="hbase"
-fi
-
# Verify if hbase has the mlock agent
if [ "$HBASE_REGIONSERVER_MLOCK" = "true" ]; then
- MLOCK_AGENT="$HBASE_HOME/native/libmlockall_agent.so"
+ MLOCK_AGENT="$HBASE_HOME/lib/native/libmlockall_agent.so"
if [ ! -f "$MLOCK_AGENT" ]; then
cat 1>&2 <
hbase git commit: HBASE-13089 Fix test compilation error on building against htrace-3.2.0-incubating
Repository: hbase
Updated Branches:
refs/heads/branch-1.2 a692295e2 - 1e348def0
HBASE-13089 Fix test compilation error on building against
htrace-3.2.0-incubating
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/1e348def
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/1e348def
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/1e348def
Branch: refs/heads/branch-1.2
Commit: 1e348def023673423e2990894716fb10df377ba3
Parents: a692295
Author: Esteban Gutierrez este...@apache.org
Authored: Fri Jul 24 15:34:40 2015 -0700
Committer: Esteban Gutierrez este...@apache.org
Committed: Mon Jul 27 13:06:37 2015 -0700
--
.../apache/hadoop/hbase/trace/TestHTraceHooks.java | 15 +--
1 file changed, 13 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/1e348def/hbase-server/src/test/java/org/apache/hadoop/hbase/trace/TestHTraceHooks.java
--
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/trace/TestHTraceHooks.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/trace/TestHTraceHooks.java
index eba30a8..eeda043 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/trace/TestHTraceHooks.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/trace/TestHTraceHooks.java
@@ -21,6 +21,7 @@ import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
+import java.lang.reflect.Method;
import java.util.Collection;
import org.apache.hadoop.hbase.HBaseTestingUtility;
@@ -48,9 +49,19 @@ public class TestHTraceHooks {
private static final byte[] FAMILY_BYTES = family.getBytes();
private static final HBaseTestingUtility TEST_UTIL = new
HBaseTestingUtility();
private static POJOSpanReceiver rcvr;
+ private static long ROOT_SPAN_ID = 0;
@BeforeClass
public static void before() throws Exception {
+
+// Find out what the right value to use fo SPAN_ROOT_ID after HTRACE-111.
We use HTRACE-32
+// to find out to detect if we are using HTrace 3.2 or not.
+try {
+Method m = Span.class.getMethod(addKVAnnotation, String.class,
String.class);
+} catch (NoSuchMethodException e) {
+ ROOT_SPAN_ID = 0x74aceL; // Span.SPAN_ROOT_ID pre HTrace-3.2
+}
+
TEST_UTIL.startMiniCluster(2, 3);
rcvr = new POJOSpanReceiver(new
HBaseHTraceConfiguration(TEST_UTIL.getConfiguration()));
Trace.addReceiver(rcvr);
@@ -86,7 +97,7 @@ public class TestHTraceHooks {
CollectionSpan spans = rcvr.getSpans();
TraceTree traceTree = new TraceTree(spans);
-CollectionSpan roots =
traceTree.getSpansByParent().find(Span.ROOT_SPAN_ID);
+CollectionSpan roots = traceTree.getSpansByParent().find(ROOT_SPAN_ID);
assertEquals(1, roots.size());
Span createTableRoot = roots.iterator().next();
@@ -117,7 +128,7 @@ public class TestHTraceHooks {
spans = rcvr.getSpans();
traceTree = new TraceTree(spans);
-roots = traceTree.getSpansByParent().find(Span.ROOT_SPAN_ID);
+roots = traceTree.getSpansByParent().find(ROOT_SPAN_ID);
assertEquals(2, roots.size());
Span putRoot = null;
hbase git commit: HBASE-13089 Fix test compilation error on building against htrace-3.2.0-incubating
Repository: hbase
Updated Branches:
refs/heads/master 3f80e0ea4 - 1566ec5fd
HBASE-13089 Fix test compilation error on building against
htrace-3.2.0-incubating
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/1566ec5f
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/1566ec5f
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/1566ec5f
Branch: refs/heads/master
Commit: 1566ec5fdc751897b2e931b2b0920c6d503c85ce
Parents: 3f80e0e
Author: Esteban Gutierrez este...@apache.org
Authored: Fri Jul 24 15:34:40 2015 -0700
Committer: Esteban Gutierrez este...@apache.org
Committed: Mon Jul 27 12:57:03 2015 -0700
--
.../apache/hadoop/hbase/trace/TestHTraceHooks.java | 15 +--
1 file changed, 13 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/1566ec5f/hbase-server/src/test/java/org/apache/hadoop/hbase/trace/TestHTraceHooks.java
--
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/trace/TestHTraceHooks.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/trace/TestHTraceHooks.java
index 7a47f52..65dff1b 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/trace/TestHTraceHooks.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/trace/TestHTraceHooks.java
@@ -21,6 +21,7 @@ import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
+import java.lang.reflect.Method;
import java.util.Collection;
import org.apache.hadoop.hbase.HBaseTestingUtility;
@@ -49,9 +50,19 @@ public class TestHTraceHooks {
private static final byte[] FAMILY_BYTES = family.getBytes();
private static final HBaseTestingUtility TEST_UTIL = new
HBaseTestingUtility();
private static POJOSpanReceiver rcvr;
+ private static long ROOT_SPAN_ID = 0;
@BeforeClass
public static void before() throws Exception {
+
+// Find out what the right value to use fo SPAN_ROOT_ID after HTRACE-111.
We use HTRACE-32
+// to find out to detect if we are using HTrace 3.2 or not.
+try {
+Method m = Span.class.getMethod(addKVAnnotation, String.class,
String.class);
+} catch (NoSuchMethodException e) {
+ ROOT_SPAN_ID = 0x74aceL; // Span.SPAN_ROOT_ID pre HTrace-3.2
+}
+
TEST_UTIL.startMiniCluster(2, 3);
rcvr = new POJOSpanReceiver(new
HBaseHTraceConfiguration(TEST_UTIL.getConfiguration()));
Trace.addReceiver(rcvr);
@@ -87,7 +98,7 @@ public class TestHTraceHooks {
CollectionSpan spans = rcvr.getSpans();
TraceTree traceTree = new TraceTree(spans);
-CollectionSpan roots =
traceTree.getSpansByParent().find(Span.ROOT_SPAN_ID);
+CollectionSpan roots = traceTree.getSpansByParent().find(ROOT_SPAN_ID);
assertEquals(1, roots.size());
Span createTableRoot = roots.iterator().next();
@@ -118,7 +129,7 @@ public class TestHTraceHooks {
spans = rcvr.getSpans();
traceTree = new TraceTree(spans);
-roots = traceTree.getSpansByParent().find(Span.ROOT_SPAN_ID);
+roots = traceTree.getSpansByParent().find(ROOT_SPAN_ID);
assertEquals(2, roots.size());
Span putRoot = null;
hbase git commit: Adding myself as a committer.
Repository: hbase Updated Branches: refs/heads/master 460d76f62 - 014829212 Adding myself as a committer. Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/01482921 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/01482921 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/01482921 Branch: refs/heads/master Commit: 01482921271146ddd1e76aec957b2d9e3f64487d Parents: 460d76f Author: Esteban Gutierrez este...@apache.org Authored: Wed Jul 1 17:52:20 2015 -0700 Committer: Esteban Gutierrez este...@apache.org Committed: Wed Jul 1 17:52:20 2015 -0700 -- pom.xml | 8 1 file changed, 8 insertions(+) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/01482921/pom.xml -- diff --git a/pom.xml b/pom.xml index 5551905..5da039f 100644 --- a/pom.xml +++ b/pom.xml @@ -260,6 +260,14 @@ organizationUrlhttp://www.continuuity.com/organizationUrl /developer developer + idesteban/id + nameEsteban Gutierrez/name + emaileste...@apache.org/email + timezone-8/timezone + organizationCloudera/organization + organizationUrlhttp://www.cloudera.com/organizationUrl +/developer +developer idjmhsieh/id nameJonathan Hsieh/name emailjmhs...@apache.org/email
