date:20180618

hbase git commit: HBASE-20727 Persist FlushedSequenceId to speed up WAL split after cluster restart

2018-06-18 Thread allan163

Repository: hbase
Updated Branches:
  refs/heads/master 78da0e366 -> b336da925


HBASE-20727 Persist FlushedSequenceId to speed up WAL split after cluster 
restart


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/b336da92
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/b336da92
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/b336da92

Branch: refs/heads/master
Commit: b336da925ac5c5ee3565112de4b808fe2eed08a2
Parents: 78da0e3
Author: Allan Yang 
Authored: Tue Jun 19 09:45:47 2018 +0800
Committer: Allan Yang 
Committed: Tue Jun 19 09:45:47 2018 +0800

--
 .../src/main/protobuf/HBase.proto   |  15 ++
 .../org/apache/hadoop/hbase/master/HMaster.java |   8 +
 .../hadoop/hbase/master/ServerManager.java  | 213 ++-
 .../apache/hadoop/hbase/master/TestMaster.java  |  33 +++
 4 files changed, 268 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/b336da92/hbase-protocol-shaded/src/main/protobuf/HBase.proto
--
diff --git a/hbase-protocol-shaded/src/main/protobuf/HBase.proto 
b/hbase-protocol-shaded/src/main/protobuf/HBase.proto
index 29067f1..0af2ffd 100644
--- a/hbase-protocol-shaded/src/main/protobuf/HBase.proto
+++ b/hbase-protocol-shaded/src/main/protobuf/HBase.proto
@@ -252,4 +252,19 @@ message CacheEvictionStats {
   optional int64 bytes_evicted = 2;
   optional int64 max_cache_size = 3;
   repeated RegionExceptionMessage exception = 4;
+}
+
+message FlushedStoreSequenceId {
+  required bytes family = 1;
+  required uint64 seqId = 2;
+}
+
+message FlushedRegionSequenceId {
+  required bytes regionEncodedName = 1;
+  required uint64 seqId = 2;
+  repeated FlushedStoreSequenceId stores = 3;
+}
+
+message FlushedSequenceId {
+  repeated FlushedRegionSequenceId regionSequenceId = 1;
 }
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hbase/blob/b336da92/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index 883bb4f..38aac50 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -862,6 +862,13 @@ public class HMaster extends HRegionServer implements 
MasterServices {
 
 status.setStatus("Initializing ZK system trackers");
 initializeZKBasedSystemTrackers();
+status.setStatus("Loading last flushed sequence id of regions");
+try {
+  this.serverManager.loadLastFlushedSequenceIds();
+} catch (IOException e) {
+  LOG.debug("Failed to load last flushed sequence id of regions"
+  + " from file system", e);
+}
 // Set ourselves as active Master now our claim has succeeded up in zk.
 this.activeMaster = true;
 
@@ -946,6 +953,7 @@ public class HMaster extends HRegionServer implements 
MasterServices {
 getChoreService().scheduleChore(normalizerChore);
 this.catalogJanitorChore = new CatalogJanitor(this);
 getChoreService().scheduleChore(catalogJanitorChore);
+this.serverManager.startChore();
 
 status.setStatus("Starting cluster schema service");
 initClusterSchemaService();

http://git-wip-us.apache.org/repos/asf/hbase/blob/b336da92/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
index c746502..cfbd52f 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
@@ -38,10 +38,15 @@ import java.util.concurrent.CopyOnWriteArrayList;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.function.Predicate;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.ClockOutOfSyncException;
 import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.NotServingRegionException;
 import org.apache.hadoop.hbase.RegionMetrics;
+import org.apache.hadoop.hbase.ScheduledChore;
 import org.apache.hadoop.hbase.ServerMetrics;
 import org.apache.hadoop.hbase.ServerMetricsBuilder;
 import org.apache.hadoop.hbase.ServerName;
@@ -51,9 +56,11 @@ import

hbase-site git commit: INFRA-10751 Empty commit

2018-06-18 Thread git-site-role

Repository: hbase-site
Updated Branches:
  refs/heads/asf-site fd9704620 -> 630414573


INFRA-10751 Empty commit


Project: http://git-wip-us.apache.org/repos/asf/hbase-site/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase-site/commit/63041457
Tree: http://git-wip-us.apache.org/repos/asf/hbase-site/tree/63041457
Diff: http://git-wip-us.apache.org/repos/asf/hbase-site/diff/63041457

Branch: refs/heads/asf-site
Commit: 6304145739cc4b670d2a967b74edd4ede998b803
Parents: fd97046
Author: jenkins 
Authored: Tue Jun 19 00:36:44 2018 +
Committer: jenkins 
Committed: Tue Jun 19 00:36:44 2018 +

--

--

[1/3] hbase-site git commit: Published site at 78da0e366970cf9bab6add9ba9fd74ca37c75be3.

2018-06-18 Thread git-site-role

Repository: hbase-site
Updated Branches:
  refs/heads/asf-site 43d113c9f -> fd9704620


http://git-wip-us.apache.org/repos/asf/hbase-site/blob/fd970462/dependency-info.html
--
diff --git a/dependency-info.html b/dependency-info.html
index ea22763..1f06be0 100644
--- a/dependency-info.html
+++ b/dependency-info.html
@@ -7,7 +7,7 @@
   
 
 
-
+
 
 Apache HBase  Dependency Information
 
@@ -313,7 +313,7 @@
 https://www.apache.org/;>The Apache Software 
Foundation.
 All rights reserved.  
 
-  Last Published: 
2018-06-16
+  Last Published: 
2018-06-19
 
 
 

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/fd970462/dependency-management.html
--
diff --git a/dependency-management.html b/dependency-management.html
index 56a6a2d..1f39c73 100644
--- a/dependency-management.html
+++ b/dependency-management.html
@@ -7,7 +7,7 @@
   
 
 
-
+
 
 Apache HBase  Project Dependency Management
 
@@ -283,6 +283,18 @@
 License
 
 com.fasterxml.jackson.core
+http://github.com/FasterXML/jackson;>jackson-annotations
+2.9.2
+jar
+http://www.apache.org/licenses/LICENSE-2.0.txt;>The Apache Software 
License, Version 2.0
+
+com.fasterxml.jackson.core
+https://github.com/FasterXML/jackson-core;>jackson-core
+2.9.2
+jar
+http://www.apache.org/licenses/LICENSE-2.0.txt;>The Apache Software 
License, Version 2.0
+
+com.fasterxml.jackson.core
 http://github.com/FasterXML/jackson;>jackson-databind
 2.9.2
 jar
@@ -324,17 +336,29 @@
 jar
 http://www.apache.org/licenses/LICENSE-2.0.txt;>Apache License, Version 
2.0
 
+commons-logging
+http://commons.apache.org/proper/commons-logging/;>commons-logging
+1.2
+jar
+http://www.apache.org/licenses/LICENSE-2.0.txt;>The Apache Software 
License, Version 2.0
+
 io.dropwizard.metrics
 http://metrics.codahale.com/metrics-core/;>metrics-core
 3.2.1
 jar
 http://www.apache.org/licenses/LICENSE-2.0.html;>Apache License 
2.0
-
+
 javax.servlet
 http://servlet-spec.java.net;>javax.servlet-api
 3.1.0
 jar
 https://glassfish.dev.java.net/nonav/public/CDDL+GPL.html;>CDDL + GPLv2 
with classpath exception
+
+javax.servlet.jsp
+http://jsp.java.net;>javax.servlet.jsp-api
+2.3.1
+jar
+http://glassfish.dev.java.net/nonav/public/CDDL+GPL.html;>CDDL + GPLv2 
with classpath exception
 
 javax.ws.rs
 http://jax-rs-spec.java.net;>javax.ws.rs-api
@@ -595,227 +619,245 @@
 https://www.apache.org/licenses/LICENSE-2.0.txt;>Apache License, Version 
2.0
 
 org.apache.hbase
-http://hbase.apache.org/hbase-build-configuration/hbase-shell;>hbase-shell
+http://hbase.apache.org/hbase-build-configuration/hbase-shaded/hbase-shaded-client;>hbase-shaded-client
 3.0.0-SNAPSHOT
 jar
 https://www.apache.org/licenses/LICENSE-2.0.txt;>Apache License, Version 
2.0
 
 org.apache.hbase
-http://hbase.apache.org/hbase-build-configuration/hbase-spark;>hbase-spark
+http://hbase.apache.org/hbase-build-configuration/hbase-shaded/hbase-shaded-client-byo-hadoop;>hbase-shaded-client-byo-hadoop
 3.0.0-SNAPSHOT
 jar
 https://www.apache.org/licenses/LICENSE-2.0.txt;>Apache License, Version 
2.0
 
 org.apache.hbase
-http://hbase.apache.org/hbase-build-configuration/hbase-thrift;>hbase-thrift
+http://hbase.apache.org/hbase-build-configuration/hbase-shaded/hbase-shaded-mapreduce;>hbase-shaded-mapreduce
 3.0.0-SNAPSHOT
 jar
 https://www.apache.org/licenses/LICENSE-2.0.txt;>Apache License, Version 
2.0
 
 org.apache.hbase
-http://hbase.apache.org/hbase-build-configuration/hbase-zookeeper;>hbase-zookeeper
+http://hbase.apache.org/hbase-build-configuration/hbase-shell;>hbase-shell
+3.0.0-SNAPSHOT
+jar
+https://www.apache.org/licenses/LICENSE-2.0.txt;>Apache License, Version 
2.0
+
+org.apache.hbase
+http://hbase.apache.org/hbase-build-configuration/hbase-spark;>hbase-spark
+3.0.0-SNAPSHOT
+jar
+https://www.apache.org/licenses/LICENSE-2.0.txt;>Apache License, Version 
2.0
+
+org.apache.hbase
+http://hbase.apache.org/hbase-build-configuration/hbase-thrift;>hbase-thrift
 3.0.0-SNAPSHOT
 jar
 https://www.apache.org/licenses/LICENSE-2.0.txt;>Apache License, Version 
2.0
 
+org.apache.hbase
+http://hbase.apache.org/hbase-build-configuration/hbase-zookeeper;>hbase-zookeeper
+3.0.0-SNAPSHOT
+jar
+https://www.apache.org/licenses/LICENSE-2.0.txt;>Apache License, Version 
2.0
+
 org.apache.hbase.thirdparty
 http://hbase.apache.org/hbase-shaded-miscellaneous;>hbase-shaded-miscellaneous
 2.1.0
 jar
 https://www.apache.org/licenses/LICENSE-2.0.txt;>Apache License, Version 
2.0
-
+
 org.apache.hbase.thirdparty
 http://hbase.apache.org/hbase-shaded-netty;>hbase-shaded-netty
 2.1.0
 jar
 https://www.apache.org/licenses/LICENSE-2.0.txt;>Apache License, Version 
2.0
-
+
 org.apache.hbase.thirdparty
 http://hbase.apache.org/hbase-shaded-protobuf;>hbase-shaded-protobuf

[2/3] hbase-site git commit: Published site at 78da0e366970cf9bab6add9ba9fd74ca37c75be3.

2018-06-18 Thread git-site-role

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/fd970462/dependency-convergence.html
--
diff --git a/dependency-convergence.html b/dependency-convergence.html
index 8400033..65ec4f1 100644
--- a/dependency-convergence.html
+++ b/dependency-convergence.html
@@ -7,7 +7,7 @@
   
 
 
-
+
 
 Apache HBase  Reactor Dependency Convergence
 
@@ -282,52 +282,28 @@
 
 
 Number of modules:
-42
+44
 
 Number of dependencies (NOD):
-314
+315
 
 Number of unique artifacts (NOA):
-350
+344
 
 Number of version-conflicting artifacts (NOC):
-23
+20
 
 Number of SNAPSHOT artifacts (NOS):
 0
 
 Convergence (NOD/NOA):
-89 
%
+91 
%
 
 Ready for release (100% convergence and no SNAPSHOTS):
 ErrorYou do not have 100% convergence.
 
 Dependencies used in 
modules
 
-com.fasterxml.jackson.core:jackson-annotations
-
-
-
-
-
-
-2.6.5
-
-
-org.apache.hbase:hbase-spark-it:jar:3.0.0-SNAPSHOT\-org.apache.spark:spark-core_2.11:jar:2.1.1:provided\-org.apache.spark:spark-network-common_2.11:jar:2.1.1:provided\-(com.fasterxml.jackson.core:jackson-annotations:jar:2.6.5:provided
 - omitted for conflict with 2.9.0)
-
-2.9.0
-
-
-org.apache.hbase:hbase-assembly:pom:3.0.0-SNAPSHOT+-org.apache.hbase:hbase-mapreduce:jar:3.0.0-SNAPSHOT:compile|\-com.fasterxml.jackson.core:jackson-databind:jar:2.9.2:compile|\-com.fasterxml.jackson.core:jackson-annotations:jar:2.9.0:compile\-org.apache.hbase:hbase-rest:jar:3.0.0-SNAPSHOT:compile\-com.fasterxml.jackson.jaxrs:jackson-jaxrs-json-provider:jar:2.9.2:compile\-com.fasterxml.jackson.module:jackson-module-jaxb-annotations:jar:2.9.2:compile\-(com.fasterxml.jackson.core:jackson-annotations:jar:2.9.0:compile
 - omitted for duplicate)
-org.apache.hbase:hbase-spark-it:jar:3.0.0-SNAPSHOT\-org.apache.hbase:hbase-common:jar:3.0.0-SNAPSHOT:compile\-com.fasterxml.jackson.core:jackson-databind:jar:2.9.2:compile\-com.fasterxml.jackson.core:jackson-annotations:jar:2.9.0:compile
-
-2.9.2
-
-
-org.apache.hbase:hbase-assembly:pom:3.0.0-SNAPSHOT\-org.apache.hbase:hbase-spark:jar:3.0.0-SNAPSHOT:compile\-com.fasterxml.jackson.module:jackson-module-scala_2.11:jar:2.9.2:compile\-(com.fasterxml.jackson.core:jackson-annotations:jar:2.9.2:compile
 - omitted for conflict with 2.9.0)
-org.apache.hbase:hbase-spark-it:jar:3.0.0-SNAPSHOT\-org.apache.hbase:hbase-spark:jar:3.0.0-SNAPSHOT:compile\-com.fasterxml.jackson.module:jackson-module-scala_2.11:jar:2.9.2:compile\-(com.fasterxml.jackson.core:jackson-annotations:jar:2.9.2:compile
 - omitted for conflict with 2.9.0)
-
 com.fasterxml.jackson.module:jackson-module-scala_2.11
 
 
@@ -454,10 +430,12 @@
 org.apache.hbase:hbase-rest:jar:3.0.0-SNAPSHOT\-org.apache.hadoop:hadoop-common:jar:2.7.4:compile\-commons-configuration:commons-configuration:jar:1.6:compile\-(commons-collections:commons-collections:jar:3.2.1:compile
 - omitted for conflict with 3.2.2)
 org.apache.hbase:hbase-rsgroup:jar:3.0.0-SNAPSHOT\-org.apache.hadoop:hadoop-common:jar:2.7.4:compile\-commons-configuration:commons-configuration:jar:1.6:compile\-(commons-collections:commons-collections:jar:3.2.1:compile
 - omitted for conflict with 3.2.2)
 org.apache.hbase:hbase-server:jar:3.0.0-SNAPSHOT+-org.apache.hadoop:hadoop-minikdc:jar:2.7.4:test|+-org.apache.directory.server:apacheds-core-api:jar:2.0.0-M15:test||+-org.apache.directory.api:api-ldap-codec-core:jar:1.0.0-M20:test|||\-(commons-collections:commons-collections:jar:3.2.1:test
 - omitted for conflict with 3.2.2)||\-org.apache.directory.api:api-ldap-model:jar:1.0.0-M20:test||\-(commons-collections:commons-collections:jar:3.2.1:test
 - omitted for conflict with 3.2.2)|+-org.apache.directory.server:apacheds-interceptor-kerberos:jar:2.0.0-M15:test||\-org.apache.directory.server:apacheds-core:jar:2.0.0-M15:test||+-org.apache.directory.server:apacheds-int
 erceptors-authn:jar:2.0.0-M15:test|||\-(commons-collections:commons-collections:jar:3.2.1:test
 - omitted for conflict with 3.2.2)||\-org.apache.directory.server:apacheds-interceptors-exception:jar:2.0.0-M15:test||\-(commons-collections:commons-collections:jar:3.2.1:test
 - omitted for conflict with 3.2.2)|+-org.apache.directory.server:apacheds-ldif-partition:jar:2.0.0-M15:test||\-org.apache.directory.server:apacheds-xdbm-partition:jar:2.0.0-M15:test||\-(commons-collections:commons-collections:jar:3.2.1:test
 - omitted for conflict with 3.2.2)|+-org.apache.directory.server:apacheds-mavibot-partition:jar:2.0.0-M15:test||\-org.a
 pache.directory.mavibot:mavibot:jar:1.0.0-M1:test||\-(commons-collections:commons-collections:jar:3.2.1:test
 - omitted for conflict with 3.2.2)|\-org.apache.directory.api:api-all:jar:1.0.0-M20:test|\-(commons-collections:commons-collections:jar:3.2.1:test
 - omitted for conflict with 3.2.2)\-org.apache.hadoop:hadoop-common:jar:2.7.4:compile\-commons-configuration:commons-configuration:jar:1.6:compile\-(commons-collections:commons-collections:jar:3.2.1:compile
 - omitted for conflict with 3.2.2)

[3/3] hbase-site git commit: Published site at 78da0e366970cf9bab6add9ba9fd74ca37c75be3.

2018-06-18 Thread git-site-role

Published site at 78da0e366970cf9bab6add9ba9fd74ca37c75be3.


Project: http://git-wip-us.apache.org/repos/asf/hbase-site/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase-site/commit/fd970462
Tree: http://git-wip-us.apache.org/repos/asf/hbase-site/tree/fd970462
Diff: http://git-wip-us.apache.org/repos/asf/hbase-site/diff/fd970462

Branch: refs/heads/asf-site
Commit: fd97046205a43da2afd721c0199170fcf457ff2c
Parents: 43d113c
Author: jenkins 
Authored: Tue Jun 19 00:36:34 2018 +
Committer: jenkins 
Committed: Tue Jun 19 00:36:34 2018 +

--
 acid-semantics.html |   4 +-
 apache_hbase_reference_guide.pdf|   4 +-
 book.html   |   2 +-
 bulk-loads.html |   4 +-
 checkstyle-aggregate.html   |   4 +-
 coc.html|   4 +-
 dependencies.html   |   4 +-
 dependency-convergence.html | 340 ---
 dependency-info.html|   4 +-
 dependency-management.html  | 122 ---
 devapidocs/constant-values.html |   4 +-
 .../org/apache/hadoop/hbase/Version.html|   4 +-
 downloads.html  |   4 +-
 export_control.html |   4 +-
 index.html  |   4 +-
 integration.html|   4 +-
 issue-tracking.html |   4 +-
 license.html|   4 +-
 mail-lists.html |   4 +-
 metrics.html|   4 +-
 old_news.html   |   4 +-
 plugin-management.html  |   4 +-
 plugins.html|   4 +-
 poweredbyhbase.html |   4 +-
 project-info.html   |   4 +-
 project-reports.html|   4 +-
 project-summary.html|   4 +-
 pseudo-distributed.html |   4 +-
 replication.html|   4 +-
 resources.html  |   4 +-
 source-repository.html  |   4 +-
 sponsors.html   |   4 +-
 supportingprojects.html |   4 +-
 team-list.html  |   4 +-
 34 files changed, 215 insertions(+), 373 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase-site/blob/fd970462/acid-semantics.html
--
diff --git a/acid-semantics.html b/acid-semantics.html
index de49b0c..034fc0e 100644
--- a/acid-semantics.html
+++ b/acid-semantics.html
@@ -7,7 +7,7 @@
   
 
 
-
+
 
 Apache HBase   
   Apache HBase (TM) ACID Properties
@@ -601,7 +601,7 @@ under the License. -->
 https://www.apache.org/;>The Apache Software 
Foundation.
 All rights reserved.  
 
-  Last Published: 
2018-06-16
+  Last Published: 
2018-06-19
 
 
 

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/fd970462/apache_hbase_reference_guide.pdf
--
diff --git a/apache_hbase_reference_guide.pdf b/apache_hbase_reference_guide.pdf
index 4bd2008..90255f0 100644
--- a/apache_hbase_reference_guide.pdf
+++ b/apache_hbase_reference_guide.pdf
@@ -5,8 +5,8 @@
 /Author (Apache HBase Team)
 /Creator (Asciidoctor PDF 1.5.0.alpha.15, based on Prawn 2.2.2)
 /Producer (Apache HBase Team)
-/ModDate (D:20180616212517+00'00')
-/CreationDate (D:20180616214046+00'00')
+/ModDate (D:20180619001822+00'00')
+/CreationDate (D:20180619003342+00'00')
 >>
 endobj
 2 0 obj

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/fd970462/book.html
--
diff --git a/book.html b/book.html
index 151f29b..0566e2e 100644
--- a/book.html
+++ b/book.html
@@ -38612,7 +38612,7 @@ The server will return cellblocks compressed using this 
same compressor as long
 
 
 Version 3.0.0-SNAPSHOT
-Last updated 2018-06-16 21:25:17 UTC
+Last updated 2018-06-19 00:18:22 UTC
 
 
 

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/fd970462/bulk-loads.html
--
diff --git a/bulk-loads.html b/bulk-loads.html
index da05d81..97aea59 100644
--- a/bulk-loads.html
+++ b/bulk-loads.html
@@ -7,7 +7,7 @@
   
 
 
-
+
 
 Apache HBase   
   Bulk Loads in Apache HBase (TM)
@@ -306,7

[hbase] Git Push Summary

2018-06-18 Thread busbey

Repository: hbase
Updated Branches:
  refs/heads/HBASE-20331 [deleted] 1d14768d6

[3/6] hbase git commit: HBASE-20333 Provide a shaded client that allows downstream to provide Hadoop needs.

2018-06-18 Thread busbey

HBASE-20333 Provide a shaded client that allows downstream to provide Hadoop 
needs.

Signed-off-by: Mike Drob 


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/e19fdd39
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/e19fdd39
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/e19fdd39

Branch: refs/heads/branch-2
Commit: e19fdd39036e7f31a864f709f429ed3bfcb641c3
Parents: ee84a8f
Author: Sean Busbey 
Authored: Tue Apr 24 14:51:12 2018 -0500
Committer: Sean Busbey 
Committed: Mon Jun 18 14:03:13 2018 -0700

--
 .../hbase-shaded-check-invariants/pom.xml   |  5 ++
 .../hbase-shaded-client-byo-hadoop/pom.xml  | 70 
 hbase-shaded/hbase-shaded-client/pom.xml| 35 --
 hbase-shaded/hbase-shaded-mapreduce/pom.xml | 30 ++---
 .../pom.xml |  2 +-
 hbase-shaded/pom.xml|  6 ++
 6 files changed, 116 insertions(+), 32 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/e19fdd39/hbase-shaded/hbase-shaded-check-invariants/pom.xml
--
diff --git a/hbase-shaded/hbase-shaded-check-invariants/pom.xml 
b/hbase-shaded/hbase-shaded-check-invariants/pom.xml
index 097de41..91956b1 100644
--- a/hbase-shaded/hbase-shaded-check-invariants/pom.xml
+++ b/hbase-shaded/hbase-shaded-check-invariants/pom.xml
@@ -48,6 +48,11 @@
   hbase-shaded-mapreduce
   ${project.version}
 
+
+  org.apache.hbase
+  hbase-shaded-client-byo-hadoop
+  ${project.version}
+
 
 
   com.github.stephenc.findbugs

http://git-wip-us.apache.org/repos/asf/hbase/blob/e19fdd39/hbase-shaded/hbase-shaded-client-byo-hadoop/pom.xml
--
diff --git a/hbase-shaded/hbase-shaded-client-byo-hadoop/pom.xml 
b/hbase-shaded/hbase-shaded-client-byo-hadoop/pom.xml
new file mode 100644
index 000..8b425f5
--- /dev/null
+++ b/hbase-shaded/hbase-shaded-client-byo-hadoop/pom.xml
@@ -0,0 +1,70 @@
+http://maven.apache.org/POM/4.0.0;
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance;
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 
http://maven.apache.org/xsd/maven-4.0.0.xsd;>
+
+4.0.0
+
+hbase-shaded
+org.apache.hbase
+2.1.0-SNAPSHOT
+..
+
+hbase-shaded-client-byo-hadoop
+Apache HBase - Shaded - Client
+
+
+
+org.apache.maven.plugins
+maven-site-plugin
+
+true
+
+
+
+
+maven-assembly-plugin
+
+true
+
+
+
+
+
+
+org.apache.hbase
+hbase-client
+
+
+
+
+
+release
+
+
+
+org.apache.maven.plugins
+maven-shade-plugin
+
+
+
+
+
+

http://git-wip-us.apache.org/repos/asf/hbase/blob/e19fdd39/hbase-shaded/hbase-shaded-client/pom.xml
--
diff --git a/hbase-shaded/hbase-shaded-client/pom.xml 
b/hbase-shaded/hbase-shaded-client/pom.xml
index ce9c111..74c2526 100644
--- a/hbase-shaded/hbase-shaded-client/pom.xml
+++ b/hbase-shaded/hbase-shaded-client/pom.xml
@@ -28,7 +28,7 @@
 ..
 
 hbase-shaded-client
-Apache HBase - Shaded - Client
+Apache HBase - Shaded - Client (with Hadoop bundled)
 
 
 
@@ -51,6 +51,7 @@
 
 org.apache.hbase
 hbase-client
+${project.version}
 
 
 
@@ -59,10 +60,34 @@
 release
 
 
-
-org.apache.maven.plugins
-maven-shade-plugin
-
+
+org.apache.maven.plugins
+maven-shade-plugin
+
+
+
aggregate-into-a-jar-with-relocated-third-parties
+
+
+
+
+
+
org.apache.hbase:hbase-resource-bundle
+org.slf4j:*
+
com.google.code.findbugs:*
+
com.github.stephenc.findbugs:*
+

[2/6] hbase git commit: HBASE-20332 shaded mapreduce module shouldn't include hadoop

2018-06-18 Thread busbey

HBASE-20332 shaded mapreduce module shouldn't include hadoop

* modify the jar checking script to take args; make hadoop stuff optional
* separate out checking the artifacts that have hadoop vs those that don't.
* * Unfortunately means we need two modules for checking things
* * put in a safety check that the support script for checking jar contents is 
maintained in both modules
* * have to carve out an exception for o.a.hadoop.metrics2. :(
* fix duplicated class warning
* clean up dependencies in hbase-server and some modules that depend on it.
* allow Hadoop to have its own htrace where it needs it
* add a precommit check to make sure we're not using old htrace imports

 Conflicts:
hbase-backup/pom.xml
hbase-checkstyle/src/main/resources/hbase/checkstyle-suppressions.xml

Signed-off-by: Mike Drob 


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/ee84a8f2
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/ee84a8f2
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/ee84a8f2

Branch: refs/heads/branch-2
Commit: ee84a8f2430f8da58c5274d4a76b4e1e32259b26
Parents: b04c976
Author: Sean Busbey 
Authored: Mon Apr 9 13:37:44 2018 -0500
Committer: Sean Busbey 
Committed: Mon Jun 18 14:02:48 2018 -0700

--
 .../resources/hbase/checkstyle-suppressions.xml |  11 +
 .../src/main/resources/hbase/checkstyle.xml |   9 +-
 hbase-client/pom.xml|   4 -
 hbase-common/pom.xml|  12 --
 hbase-endpoint/pom.xml  |  14 --
 hbase-examples/pom.xml  |  12 --
 hbase-external-blockcache/pom.xml   |   4 -
 hbase-hadoop2-compat/pom.xml|   6 -
 hbase-it/pom.xml|   6 -
 hbase-mapreduce/pom.xml |  30 +--
 hbase-replication/pom.xml   |   4 -
 hbase-rest/pom.xml  |  19 +-
 hbase-rsgroup/pom.xml   |   4 -
 hbase-server/pom.xml| 103 -
 .../hbase-shaded-check-invariants/pom.xml   |  54 +++--
 .../ensure-jars-have-correct-contents.sh|  92 ++--
 hbase-shaded/hbase-shaded-mapreduce/pom.xml | 190 +++-
 .../pom.xml | 215 +++
 .../ensure-jars-have-correct-contents.sh| 129 +++
 hbase-shaded/pom.xml|  13 ++
 hbase-shell/pom.xml |  14 --
 hbase-testing-util/pom.xml  |  16 --
 hbase-thrift/pom.xml|  16 --
 pom.xml |  65 +++---
 24 files changed, 784 insertions(+), 258 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/ee84a8f2/hbase-checkstyle/src/main/resources/hbase/checkstyle-suppressions.xml
--
diff --git 
a/hbase-checkstyle/src/main/resources/hbase/checkstyle-suppressions.xml 
b/hbase-checkstyle/src/main/resources/hbase/checkstyle-suppressions.xml
index aea7e94..2673b8b 100644
--- a/hbase-checkstyle/src/main/resources/hbase/checkstyle-suppressions.xml
+++ b/hbase-checkstyle/src/main/resources/hbase/checkstyle-suppressions.xml
@@ -2,6 +2,13 @@
 http://www.puppycrawl.com/dtds/suppressions_1_0.dtd;>
+
   
 

http://git-wip-us.apache.org/repos/asf/hbase/blob/ee84a8f2/hbase-checkstyle/src/main/resources/hbase/checkstyle.xml
--
diff --git a/hbase-checkstyle/src/main/resources/hbase/checkstyle.xml 
b/hbase-checkstyle/src/main/resources/hbase/checkstyle.xml
index 7ad797c..22864a3 100644
--- a/hbase-checkstyle/src/main/resources/hbase/checkstyle.xml
+++ b/hbase-checkstyle/src/main/resources/hbase/checkstyle.xml
@@ -77,6 +77,12 @@
   
 
 
+
   
+  org.codehaus.jackson,
+  org.htrace"/>
   

http://git-wip-us.apache.org/repos/asf/hbase/blob/ee84a8f2/hbase-client/pom.xml
--
diff --git a/hbase-client/pom.xml b/hbase-client/pom.xml
index 6798f53..c209a8a 100644
--- a/hbase-client/pom.xml
+++ b/hbase-client/pom.xml
@@ -235,10 +235,6 @@
   hadoop-common
   
 
-  org.apache.htrace
-  htrace-core
-
-
   net.java.dev.jets3t
   jets3t
 

http://git-wip-us.apache.org/repos/asf/hbase/blob/ee84a8f2/hbase-common/pom.xml
--
diff --git a/hbase-common/pom.xml b/hbase-common/pom.xml
index 5a4fa44..a0a4d3b 100644
--- a/hbase-common/pom.xml
+++ b/hbase-common/pom.xml
@@ -314,12 +314,6

[5/6] hbase git commit: HBASE-20615 emphasize shaded artifacts in client tarball.

2018-06-18 Thread busbey

HBASE-20615 emphasize shaded artifacts in client tarball.

 Conflicts:
bin/hbase

Signed-off-by: Josh Elser 


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/133892aa
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/133892aa
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/133892aa

Branch: refs/heads/branch-2
Commit: 133892aae21be48cba6230083ffa4d2272dd63fa
Parents: 1725094
Author: Sean Busbey 
Authored: Fri May 18 11:11:42 2018 -0500
Committer: Sean Busbey 
Committed: Mon Jun 18 14:03:28 2018 -0700

--
 bin/hbase   | 204 ---
 bin/hbase-config.sh |  10 +
 hbase-assembly/pom.xml  |  21 +-
 .../src/main/assembly/client-components.xml |  43 +++-
 hbase-assembly/src/main/assembly/client.xml | 131 ++--
 hbase-assembly/src/main/assembly/components.xml |   3 +-
 .../src/main/assembly/hadoop-two-compat.xml |  80 +++-
 pom.xml |   6 +
 8 files changed, 395 insertions(+), 103 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/133892aa/bin/hbase
--
diff --git a/bin/hbase b/bin/hbase
index 18f58c9..7fe61b4 100755
--- a/bin/hbase
+++ b/bin/hbase
@@ -71,11 +71,18 @@ if [ -d "${HBASE_HOME}/target" ]; then
   in_dev_env=true
 fi
 
+# Detect if we are in the omnibus tarball
+in_omnibus_tarball="false"
+if [ -f "${HBASE_HOME}/bin/hbase-daemons.sh" ]; then
+  in_omnibus_tarball="true"
+fi
+
 read -d '' options_string << EOF
 Options:
-  --config DIR Configuration direction to use. Default: ./conf
-  --hosts HOSTSOverride the list in 'regionservers' file
-  --auth-as-server Authenticate to ZooKeeper using servers configuration
+  --config DIR Configuration direction to use. Default: ./conf
+  --hosts HOSTSOverride the list in 'regionservers' file
+  --auth-as-server Authenticate to ZooKeeper using servers configuration
+  --internal-classpath Skip attempting to use client facing jars (WARNING: 
unstable results between versions)
 EOF
 # if no args specified, show usage
 if [ $# = 0 ]; then
@@ -87,16 +94,18 @@ if [ $# = 0 ]; then
   echo "  shell   Run the HBase shell"
   echo "  hbckRun the hbase 'fsck' tool"
   echo "  snapshotTool for managing snapshots"
-  echo "  wal Write-ahead-log analyzer"
-  echo "  hfile   Store file analyzer"
-  echo "  zkcli   Run the ZooKeeper shell"
-  echo "  master  Run an HBase HMaster node"
-  echo "  regionserverRun an HBase HRegionServer node"
-  echo "  zookeeper   Run a ZooKeeper server"
-  echo "  restRun an HBase REST server"
-  echo "  thrift  Run the HBase Thrift server"
-  echo "  thrift2 Run the HBase Thrift2 server"
-  echo "  clean   Run the HBase clean up script"
+  if [ "${in_omnibus_tarball}" = "true" ]; then
+echo "  wal Write-ahead-log analyzer"
+echo "  hfile   Store file analyzer"
+echo "  zkcli   Run the ZooKeeper shell"
+echo "  master  Run an HBase HMaster node"
+echo "  regionserverRun an HBase HRegionServer node"
+echo "  zookeeper   Run a ZooKeeper server"
+echo "  restRun an HBase REST server"
+echo "  thrift  Run the HBase Thrift server"
+echo "  thrift2 Run the HBase Thrift2 server"
+echo "  clean   Run the HBase clean up script"
+  fi
   echo "  classpath   Dump hbase CLASSPATH"
   echo "  mapredcpDump CLASSPATH entries required by mapreduce"
   echo "  pe  Run PerformanceEvaluation"
@@ -184,9 +193,99 @@ for f in $HBASE_HOME/hbase-jars/hbase*.jar; do
   fi
 done
 
+#If avail, add Hadoop to the CLASSPATH and to the JAVA_LIBRARY_PATH
+# Allow this functionality to be disabled
+if [ "$HBASE_DISABLE_HADOOP_CLASSPATH_LOOKUP" != "true" ] ; then
+  HADOOP_IN_PATH=$(PATH="${HADOOP_HOME:-${HADOOP_PREFIX}}/bin:$PATH" which 
hadoop 2>/dev/null)
+fi
+
 # Add libs to CLASSPATH
-for f in $HBASE_HOME/lib/*.jar; do
-  CLASSPATH=${CLASSPATH}:$f;
+declare shaded_jar
+
+if [ "${INTERNAL_CLASSPATH}" != "true" ]; then
+  # find our shaded jars
+  declare shaded_client
+  declare shaded_client_byo_hadoop
+  declare shaded_mapreduce
+  for f in "${HBASE_HOME}"/lib/shaded-clients/hbase-shaded-client*.jar; do
+if [[ "${f}" =~ byo-hadoop ]]; then
+  shaded_client_byo_hadoop="${f}"
+else
+  shaded_client="${f}"
+fi
+  done
+  for f in "${HBASE_HOME}"/lib/shaded-clients/hbase-shaded-mapreduce*.jar; do
+shaded_mapreduce="${f}"
+  done
+
+  # If command can use our shaded client, use it
+  declare -a

[1/6] hbase git commit: HBASE-20478 Update checkstyle to v8.2

2018-06-18 Thread busbey

Repository: hbase
Updated Branches:
  refs/heads/branch-2 8edd5d948 -> 0a4ae9fa9


HBASE-20478 Update checkstyle to v8.2

Cannot go to latest (8.9) yet due to
  https://github.com/checkstyle/checkstyle/issues/5279

* move hbaseanti import checks to checkstyle
* implment a few missing equals checks, and ignore one
* fix lots of javadoc errors

Signed-off-by: Sean Busbey 


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/b04c976f
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/b04c976f
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/b04c976f

Branch: refs/heads/branch-2
Commit: b04c976fe6b818377c757f4ab52d27ba62cf8ceb
Parents: 8edd5d9
Author: Mike Drob 
Authored: Thu Apr 26 20:12:07 2018 -0500
Committer: Sean Busbey 
Committed: Mon Jun 18 14:02:40 2018 -0700

--
 dev-support/hbase-personality.sh| 18 --
 .../resources/hbase/checkstyle-suppressions.xml |  2 ++
 .../src/main/resources/hbase/checkstyle.xml |  8 -
 .../hadoop/hbase/client/ClusterConnection.java  |  4 +--
 .../hbase/client/replication/TableCFs.java  |  2 +-
 .../hadoop/hbase/protobuf/ProtobufUtil.java |  2 +-
 .../hadoop/hbase/quotas/QuotaTableUtil.java |  2 +-
 .../hbase/shaded/protobuf/RequestConverter.java | 12 +++
 .../org/apache/hadoop/hbase/net/Address.java|  2 +-
 .../hadoop/hbase/util/AbstractByteRange.java| 14 
 .../apache/hadoop/hbase/util/JRubyFormat.java   |  8 +++--
 .../apache/hadoop/hbase/util/OrderedBytes.java  |  2 +-
 .../java/org/apache/hadoop/hbase/Waiter.java|  2 +-
 .../security/access/SecureBulkLoadEndpoint.java |  9 +++--
 .../hadoop/hbase/http/TestHttpServer.java   |  8 ++---
 .../hadoop/hbase/IntegrationTestBase.java   |  6 ++--
 ...IntegrationTestRegionReplicaReplication.java |  2 +-
 .../hbase/mapred/TestTableInputFormat.java  |  6 ++--
 .../hbase/mapreduce/TestImportExport.java   |  6 ++--
 .../hbase/mapreduce/TestTableInputFormat.java   |  4 +--
 .../apache/hadoop/hbase/util/LoadTestTool.java  |  4 +--
 .../favored/StartcodeAgnosticServerName.java|  4 +++
 .../hadoop/hbase/io/hfile/HFileBlockIndex.java  |  2 +-
 .../org/apache/hadoop/hbase/ipc/RpcServer.java  |  2 +-
 .../hbase/regionserver/StoreFlushContext.java   |  2 +-
 .../querymatcher/ColumnTracker.java |  4 +--
 .../security/access/AccessControlFilter.java|  2 +-
 .../hadoop/hbase/HBaseTestingUtility.java   | 34 +-
 .../hadoop/hbase/MultithreadedTestUtil.java | 13 ---
 .../hbase/TestMetaTableAccessorNoCluster.java   |  2 +-
 .../hbase/TestPartialResultsFromClientSide.java | 28 +++
 .../hadoop/hbase/backup/TestHFileArchiving.java |  6 ++--
 .../hbase/client/HConnectionTestingUtility.java |  4 +--
 .../hbase/filter/TestColumnRangeFilter.java | 38 +---
 .../hadoop/hbase/io/hfile/KVGenerator.java  |  2 +-
 .../hbase/master/TestRegionPlacement.java   |  7 ++--
 .../regionserver/TestCompoundBloomFilter.java   |  4 +--
 .../hadoop/hbase/regionserver/TestHRegion.java  |  2 +-
 .../hbase/snapshot/SnapshotTestingUtils.java|  8 ++---
 .../hbase/util/HFileArchiveTestingUtil.java |  2 +-
 .../hadoop/hbase/util/TestRegionSplitter.java   |  2 +-
 pom.xml |  4 +--
 42 files changed, 159 insertions(+), 136 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/b04c976f/dev-support/hbase-personality.sh
--
diff --git a/dev-support/hbase-personality.sh b/dev-support/hbase-personality.sh
index c3f5668..da96019 100755
--- a/dev-support/hbase-personality.sh
+++ b/dev-support/hbase-personality.sh
@@ -661,24 +661,6 @@ function hbaseanti_patchfile
 ((result=result+1))
   fi
 
-  warnings=$(${GREP} -c 'import org.apache.hadoop.classification' 
"${patchfile}")
-  if [[ ${warnings} -gt 0 ]]; then
-add_vote_table -1 hbaseanti "" "The patch appears use Hadoop 
classification instead of HBase."
-((result=result+1))
-  fi
-
-  warnings=$(${GREP} -c 'import org.codehaus.jackson' "${patchfile}")
-  if [[ ${warnings} -gt 0 ]]; then
-add_vote_table -1 hbaseanti "" "The patch appears use Jackson 1 
classes/annotations."
-((result=result+1))
-  fi
-
-  warnings=$(${GREP} -cE 'org.apache.commons.logging.Log(Factory|;)' 
"${patchfile}")
-  if [[ ${warnings} -gt 0 ]]; then
-add_vote_table -1 hbaseanti "" "The patch appears to use commons-logging 
instead of slf4j."
-((result=result+1))
-  fi
-
   if [[ ${result} -gt 0 ]]; then
 return 1
   fi

http://git-wip-us.apache.org/repos/asf/hbase/blob/b04c976f/hbase-checkstyle/src/main/resources/hbase/checkstyle-suppressions.xml
--
diff --git

[4/6] hbase git commit: HBASE-19735 Create a client-tarball assembly

2018-06-18 Thread busbey

HBASE-19735 Create a client-tarball assembly

Provides an extra client descriptor to build a second
tarball with a reduced set of dependencies. Not of great
impact now, but will build the way for better in the future.

Signed-off-by: Sean Busbey 

 Conflicts:
hbase-assembly/pom.xml

 Conflicts:
hbase-spark/pom.xml


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/1725094e
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/1725094e
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/1725094e

Branch: refs/heads/branch-2
Commit: 1725094e6be7f58dea4f07f374c2001a486ec7b4
Parents: e19fdd3
Author: Josh Elser 
Authored: Wed Feb 7 18:37:39 2018 -0500
Committer: Sean Busbey 
Committed: Mon Jun 18 14:03:22 2018 -0700

--
 hbase-assembly/pom.xml  |  33 +++--
 .../src/main/assembly/client-components.xml |  92 +
 hbase-assembly/src/main/assembly/client.xml | 137 +++
 hbase-procedure/pom.xml |   4 +-
 pom.xml |  10 ++
 5 files changed, 262 insertions(+), 14 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/1725094e/hbase-assembly/pom.xml
--
diff --git a/hbase-assembly/pom.xml b/hbase-assembly/pom.xml
index 36a9afc..17d5693 100644
--- a/hbase-assembly/pom.xml
+++ b/hbase-assembly/pom.xml
@@ -79,6 +79,7 @@
   gnu
   
 ${assembly.file}
+src/main/assembly/client.xml
   
 
   
@@ -212,22 +213,22 @@
   test
 
 
-org.apache.hbase
-hbase-hadoop-compat
-
-  
-com.google.guava
-guava
-  
-
+  org.apache.hbase
+  hbase-hadoop-compat
+  
+
+  com.google.guava
+  guava
+
+  
 
 
-org.apache.hbase
-${compat.module}
+  org.apache.hbase
+  ${compat.module}
 
 
-   org.apache.hbase
-   hbase-shell
+  org.apache.hbase
+  hbase-shell
 
 
   org.apache.hbase
@@ -294,6 +295,14 @@
   jline
   jline
 
+
+  org.apache.hbase
+  hbase-shaded-client
+
+
+  org.apache.hbase
+  hbase-shaded-mapreduce
+
   
   
 

http://git-wip-us.apache.org/repos/asf/hbase/blob/1725094e/hbase-assembly/src/main/assembly/client-components.xml
--
diff --git a/hbase-assembly/src/main/assembly/client-components.xml 
b/hbase-assembly/src/main/assembly/client-components.xml
new file mode 100644
index 000..2fd1b57
--- /dev/null
+++ b/hbase-assembly/src/main/assembly/client-components.xml
@@ -0,0 +1,92 @@
+
+
+
+
+
+  
+
+
+  ${project.basedir}/../target/site
+  docs
+
+
+
+  ${project.basedir}/..
+  .
+  
+CHANGES.txt
+README.txt
+  
+  0644
+
+
+
+  ${project.basedir}/../conf
+  conf
+  0644
+  0755
+
+
+
+
+  ${project.basedir}/../bin
+  bin
+  
+get-active-master.rb
+hbase
+hbase-common.sh
+hbase-config.sh
+hbase-jruby
+hirb.rb
+
+  
+  0755
+  0755
+
+
+
+  ${project.basedir}/../bin
+  bin
+  
+hbase.cmd
+hbase-config.cmd
+  
+
+
+
+  ${project.basedir}/../hbase-shell/src/main/ruby
+  lib/ruby
+  0644
+  0755
+
+
+
+  ${project.basedir}/../hbase-server/target/native
+  lib/native
+  0755
+  0755
+  
+*.so
+*.dylib
+  
+
+  
+

http://git-wip-us.apache.org/repos/asf/hbase/blob/1725094e/hbase-assembly/src/main/assembly/client.xml
--
diff --git a/hbase-assembly/src/main/assembly/client.xml 
b/hbase-assembly/src/main/assembly/client.xml
new file mode 100644
index 000..7951961
--- /dev/null
+++ b/hbase-assembly/src/main/assembly/client.xml
@@ -0,0 +1,137 @@
+
+http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.1; 
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance; 
xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.1
 http://maven.apache.org/xsd/assembly-1.1.1.xsd;>
+
+
+  
+  client-bin
+  
+tar.gz
+  
+  
+  hbase-${project.version}-client
+  
+  
src/main/assembly/client-components.xml
+  
+  
+
+  true
+  
+
+org.apache.hbase:hbase-annotations
+org.apache.hbase:hbase-client
+org.apache.hbase:hbase-common
+org.apache.hbase:hbase-hadoop-compat
+

[6/6] hbase git commit: HBASE-20334 add a test that verifies basic client and MR integration

2018-06-18 Thread busbey

HBASE-20334 add a test that verifies basic client and MR integration

Signed-off-by: Josh Elser 


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/0a4ae9fa
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/0a4ae9fa
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/0a4ae9fa

Branch: refs/heads/branch-2
Commit: 0a4ae9fa943b86b398a974bf708b34efd0185ee2
Parents: 133892a
Author: Sean Busbey 
Authored: Tue May 1 14:28:52 2018 -0500
Committer: Sean Busbey 
Committed: Mon Jun 18 14:03:51 2018 -0700

--
 dev-support/Jenkinsfile | 233 +++--
 .../hbase_nightly_pseudo-distributed-test.sh| 516 +++
 dev-support/hbase_nightly_source-artifact.sh|  14 +-
 .../cache-apache-project-artifact.sh| 131 +
 4 files changed, 836 insertions(+), 58 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/0a4ae9fa/dev-support/Jenkinsfile
--
diff --git a/dev-support/Jenkinsfile b/dev-support/Jenkinsfile
index ce12d9a..af373e4 100644
--- a/dev-support/Jenkinsfile
+++ b/dev-support/Jenkinsfile
@@ -60,54 +60,109 @@ pipeline {
 booleanParam(name: 'DEBUG', defaultValue: false, description: 'Produce a 
lot more meta-information.')
   }
   stages {
-stage ('yetus install') {
+stage ('scm-checkout') {
   steps {
-sh  '''#!/usr/bin/env bash
-set -e
-echo "Ensure we have a copy of Apache Yetus."
-if [[ true !=  "${USE_YETUS_PRERELEASE}" ]]; then
-  YETUS_DIR="${WORKSPACE}/yetus-${YETUS_RELEASE}"
-  echo "Checking for Yetus ${YETUS_RELEASE} in '${YETUS_DIR}'"
-  if [ ! -d "${YETUS_DIR}" ]; then
-echo "New download of Apache Yetus version ${YETUS_RELEASE}."
-rm -rf "${WORKSPACE}/.gpg"
-mkdir -p "${WORKSPACE}/.gpg"
-chmod -R 700 "${WORKSPACE}/.gpg"
-
-echo "install yetus project KEYS"
-curl -L --fail -o "${WORKSPACE}/KEYS_YETUS" 
https://dist.apache.org/repos/dist/release/yetus/KEYS
-gpg --homedir "${WORKSPACE}/.gpg" --import "${WORKSPACE}/KEYS_YETUS"
-
-echo "download yetus release ${YETUS_RELEASE}"
-curl -L --fail -O 
"https://dist.apache.org/repos/dist/release/yetus/${YETUS_RELEASE}/yetus-${YETUS_RELEASE}-bin.tar.gz;
-curl -L --fail -O 
"https://dist.apache.org/repos/dist/release/yetus/${YETUS_RELEASE}/yetus-${YETUS_RELEASE}-bin.tar.gz.asc;
-echo "verifying yetus release"
-gpg --homedir "${WORKSPACE}/.gpg" --verify 
"yetus-${YETUS_RELEASE}-bin.tar.gz.asc"
-mv "yetus-${YETUS_RELEASE}-bin.tar.gz" yetus.tar.gz
-  else
-echo "Reusing cached download of Apache Yetus version ${YETUS_RELEASE}."
-  fi
-else
-  YETUS_DIR="${WORKSPACE}/yetus-git"
-  rm -rf "${YETUS_DIR}"
-  echo "downloading from github"
-  curl -L --fail https://api.github.com/repos/apache/yetus/tarball/HEAD -o 
yetus.tar.gz
-fi
-if [ ! -d "${YETUS_DIR}" ]; then
-  echo "unpacking yetus into '${YETUS_DIR}'"
-  mkdir -p "${YETUS_DIR}"
-  gunzip -c yetus.tar.gz | tar xpf - -C "${YETUS_DIR}" --strip-components 1
-fi
-'''
-// Set up the file we need at PERSONALITY_FILE location
-dir ("tools") {
-  sh """#!/usr/bin/env bash
-set -e
-echo "Downloading Project personality."
-curl -L  -o personality.sh "${env.PROJECT_PERSONALITY}"
-  """
+dir('component') {
+  checkout scm
+}
+  }
+}
+stage ('thirdparty installs') {
+  parallel {
+stage ('yetus install') {
+  steps {
+// directory must be unique for each parallel stage, because 
jenkins runs them in the same workspace :(
+dir('downloads-yetus') {
+  // can't just do a simple echo or the directory won't be 
created. :(
+  sh '''#!/usr/bin/env bash
+echo "Make sure we have a directory for downloading 
dependencies: $(pwd)"
+'''
+}
+sh  '''#!/usr/bin/env bash
+  set -e
+  echo "Ensure we have a copy of Apache Yetus."
+  if [[ true !=  "${USE_YETUS_PRERELEASE}" ]]; then
+YETUS_DIR="${WORKSPACE}/yetus-${YETUS_RELEASE}"
+echo "Checking for Yetus ${YETUS_RELEASE} in '${YETUS_DIR}'"
+if [ ! -d "${YETUS_DIR}" ]; then
+  
"${WORKSPACE}/component/dev-support/jenkins-scripts/cache-apache-project-artifact.sh"
 \
+  --working-dir "${WORKSPACE}/downloads-yetus" \
+  --keys 'https://www.apache.org/dist/yetus/KEYS' \
+  "${WORKSPACE}/yetus-${YETUS_RELEASE}-bin.tar.gz" \
+  
"yetus/${YETUS_RELEASE}/yetus-${YETUS_RELEASE}-bin.tar.gz"
+  mv "yetus-${YETUS_RELEASE}-bin.tar.gz" yetus.tar.gz
+else
+  echo "Reusing

[1/5] hbase git commit: HBASE-20332 shaded mapreduce module shouldn't include hadoop

2018-06-18 Thread busbey

Repository: hbase
Updated Branches:
  refs/heads/master ac5bb8155 -> 78da0e366


HBASE-20332 shaded mapreduce module shouldn't include hadoop

* modify the jar checking script to take args; make hadoop stuff optional
* separate out checking the artifacts that have hadoop vs those that don't.
* * Unfortunately means we need two modules for checking things
* * put in a safety check that the support script for checking jar contents is 
maintained in both modules
* * have to carve out an exception for o.a.hadoop.metrics2. :(
* fix duplicated class warning
* clean up dependencies in hbase-server and some modules that depend on it.
* allow Hadoop to have its own htrace where it needs it
* add a precommit check to make sure we're not using old htrace imports


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/f1b536ba
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/f1b536ba
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/f1b536ba

Branch: refs/heads/master
Commit: f1b536bad41023a2e017311196d32f8214bc3701
Parents: ac5bb81
Author: Sean Busbey 
Authored: Mon Apr 9 13:37:44 2018 -0500
Committer: Sean Busbey 
Committed: Mon Jun 18 11:31:04 2018 -0700

--
 hbase-backup/pom.xml|  12 +-
 .../resources/hbase/checkstyle-suppressions.xml |  11 +
 .../src/main/resources/hbase/checkstyle.xml |   9 +-
 hbase-client/pom.xml|   4 -
 hbase-common/pom.xml|  12 --
 hbase-endpoint/pom.xml  |  14 --
 hbase-examples/pom.xml  |  12 --
 hbase-external-blockcache/pom.xml   |   4 -
 hbase-hadoop2-compat/pom.xml|   6 -
 hbase-it/pom.xml|   6 -
 hbase-mapreduce/pom.xml |  30 +--
 hbase-replication/pom.xml   |   4 -
 hbase-rest/pom.xml  |  19 +-
 hbase-rsgroup/pom.xml   |   4 -
 hbase-server/pom.xml| 103 -
 .../hbase-shaded-check-invariants/pom.xml   |  54 +++--
 .../ensure-jars-have-correct-contents.sh|  92 ++--
 hbase-shaded/hbase-shaded-mapreduce/pom.xml | 190 +++-
 .../pom.xml | 215 +++
 .../ensure-jars-have-correct-contents.sh| 129 +++
 hbase-shaded/pom.xml|  13 ++
 hbase-shell/pom.xml |  14 --
 hbase-testing-util/pom.xml  |  16 --
 hbase-thrift/pom.xml|  16 --
 pom.xml |  65 +++---
 25 files changed, 789 insertions(+), 265 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/f1b536ba/hbase-backup/pom.xml
--
diff --git a/hbase-backup/pom.xml b/hbase-backup/pom.xml
index 7afd51e..00a996f 100644
--- a/hbase-backup/pom.xml
+++ b/hbase-backup/pom.xml
@@ -155,10 +155,6 @@
   hadoop-common
   
 
-  org.apache.htrace
-  htrace-core
-
-
   net.java.dev.jets3t
   jets3t
 
@@ -264,9 +260,6 @@
   3.0
 
   
-  
-3.0-SNAPSHOT
-  
   
 
   org.apache.hadoop
@@ -276,6 +269,11 @@
   org.apache.hadoop
   hadoop-mapreduce-client-core
 
+
+  org.apache.hadoop
+  hadoop-distcp
+  ${hadoop.version}
+
   
 
   

http://git-wip-us.apache.org/repos/asf/hbase/blob/f1b536ba/hbase-checkstyle/src/main/resources/hbase/checkstyle-suppressions.xml
--
diff --git 
a/hbase-checkstyle/src/main/resources/hbase/checkstyle-suppressions.xml 
b/hbase-checkstyle/src/main/resources/hbase/checkstyle-suppressions.xml
index ad79163..1679496 100644
--- a/hbase-checkstyle/src/main/resources/hbase/checkstyle-suppressions.xml
+++ b/hbase-checkstyle/src/main/resources/hbase/checkstyle-suppressions.xml
@@ -2,6 +2,13 @@
 http://www.puppycrawl.com/dtds/suppressions_1_0.dtd;>
+
   
 

http://git-wip-us.apache.org/repos/asf/hbase/blob/f1b536ba/hbase-checkstyle/src/main/resources/hbase/checkstyle.xml
--
diff --git a/hbase-checkstyle/src/main/resources/hbase/checkstyle.xml 
b/hbase-checkstyle/src/main/resources/hbase/checkstyle.xml
index 7ad797c..22864a3 100644
--- a/hbase-checkstyle/src/main/resources/hbase/checkstyle.xml
+++ b/hbase-checkstyle/src/main/resources/hbase/checkstyle.xml
@@ -77,6 +77,12 @@
   
 
 
+
   
+

[5/5] hbase git commit: HBASE-20334 add a test that verifies basic client and MR integration

2018-06-18 Thread busbey

HBASE-20334 add a test that verifies basic client and MR integration


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/78da0e36
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/78da0e36
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/78da0e36

Branch: refs/heads/master
Commit: 78da0e366970cf9bab6add9ba9fd74ca37c75be3
Parents: 10cc1da
Author: Sean Busbey 
Authored: Tue May 1 14:28:52 2018 -0500
Committer: Sean Busbey 
Committed: Mon Jun 18 11:31:13 2018 -0700

--
 dev-support/Jenkinsfile | 233 +++--
 .../hbase_nightly_pseudo-distributed-test.sh| 516 +++
 dev-support/hbase_nightly_source-artifact.sh|  14 +-
 .../cache-apache-project-artifact.sh| 131 +
 4 files changed, 836 insertions(+), 58 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/78da0e36/dev-support/Jenkinsfile
--
diff --git a/dev-support/Jenkinsfile b/dev-support/Jenkinsfile
index 2311e35..59d3227 100644
--- a/dev-support/Jenkinsfile
+++ b/dev-support/Jenkinsfile
@@ -60,54 +60,109 @@ pipeline {
 booleanParam(name: 'DEBUG', defaultValue: false, description: 'Produce a 
lot more meta-information.')
   }
   stages {
-stage ('yetus install') {
+stage ('scm-checkout') {
   steps {
-sh  '''#!/usr/bin/env bash
-set -e
-echo "Ensure we have a copy of Apache Yetus."
-if [[ true !=  "${USE_YETUS_PRERELEASE}" ]]; then
-  YETUS_DIR="${WORKSPACE}/yetus-${YETUS_RELEASE}"
-  echo "Checking for Yetus ${YETUS_RELEASE} in '${YETUS_DIR}'"
-  if [ ! -d "${YETUS_DIR}" ]; then
-echo "New download of Apache Yetus version ${YETUS_RELEASE}."
-rm -rf "${WORKSPACE}/.gpg"
-mkdir -p "${WORKSPACE}/.gpg"
-chmod -R 700 "${WORKSPACE}/.gpg"
-
-echo "install yetus project KEYS"
-curl -L --fail -o "${WORKSPACE}/KEYS_YETUS" 
https://dist.apache.org/repos/dist/release/yetus/KEYS
-gpg --homedir "${WORKSPACE}/.gpg" --import "${WORKSPACE}/KEYS_YETUS"
-
-echo "download yetus release ${YETUS_RELEASE}"
-curl -L --fail -O 
"https://dist.apache.org/repos/dist/release/yetus/${YETUS_RELEASE}/yetus-${YETUS_RELEASE}-bin.tar.gz;
-curl -L --fail -O 
"https://dist.apache.org/repos/dist/release/yetus/${YETUS_RELEASE}/yetus-${YETUS_RELEASE}-bin.tar.gz.asc;
-echo "verifying yetus release"
-gpg --homedir "${WORKSPACE}/.gpg" --verify 
"yetus-${YETUS_RELEASE}-bin.tar.gz.asc"
-mv "yetus-${YETUS_RELEASE}-bin.tar.gz" yetus.tar.gz
-  else
-echo "Reusing cached download of Apache Yetus version ${YETUS_RELEASE}."
-  fi
-else
-  YETUS_DIR="${WORKSPACE}/yetus-git"
-  rm -rf "${YETUS_DIR}"
-  echo "downloading from github"
-  curl -L --fail https://api.github.com/repos/apache/yetus/tarball/HEAD -o 
yetus.tar.gz
-fi
-if [ ! -d "${YETUS_DIR}" ]; then
-  echo "unpacking yetus into '${YETUS_DIR}'"
-  mkdir -p "${YETUS_DIR}"
-  gunzip -c yetus.tar.gz | tar xpf - -C "${YETUS_DIR}" --strip-components 1
-fi
-'''
-// Set up the file we need at PERSONALITY_FILE location
-dir ("tools") {
-  sh """#!/usr/bin/env bash
-set -e
-echo "Downloading Project personality."
-curl -L  -o personality.sh "${env.PROJECT_PERSONALITY}"
-  """
+dir('component') {
+  checkout scm
+}
+  }
+}
+stage ('thirdparty installs') {
+  parallel {
+stage ('yetus install') {
+  steps {
+// directory must be unique for each parallel stage, because 
jenkins runs them in the same workspace :(
+dir('downloads-yetus') {
+  // can't just do a simple echo or the directory won't be 
created. :(
+  sh '''#!/usr/bin/env bash
+echo "Make sure we have a directory for downloading 
dependencies: $(pwd)"
+'''
+}
+sh  '''#!/usr/bin/env bash
+  set -e
+  echo "Ensure we have a copy of Apache Yetus."
+  if [[ true !=  "${USE_YETUS_PRERELEASE}" ]]; then
+YETUS_DIR="${WORKSPACE}/yetus-${YETUS_RELEASE}"
+echo "Checking for Yetus ${YETUS_RELEASE} in '${YETUS_DIR}'"
+if [ ! -d "${YETUS_DIR}" ]; then
+  
"${WORKSPACE}/component/dev-support/jenkins-scripts/cache-apache-project-artifact.sh"
 \
+  --working-dir "${WORKSPACE}/downloads-yetus" \
+  --keys 'https://www.apache.org/dist/yetus/KEYS' \
+  "${WORKSPACE}/yetus-${YETUS_RELEASE}-bin.tar.gz" \
+  
"yetus/${YETUS_RELEASE}/yetus-${YETUS_RELEASE}-bin.tar.gz"
+  mv "yetus-${YETUS_RELEASE}-bin.tar.gz" yetus.tar.gz
+else
+  echo "Reusing cached install of Apache

[3/5] hbase git commit: HBASE-19735 Create a client-tarball assembly

2018-06-18 Thread busbey

HBASE-19735 Create a client-tarball assembly

Provides an extra client descriptor to build a second
tarball with a reduced set of dependencies. Not of great
impact now, but will build the way for better in the future.

Signed-off-by: Sean Busbey 

 Conflicts:
hbase-assembly/pom.xml


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/8710825a
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/8710825a
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/8710825a

Branch: refs/heads/master
Commit: 8710825a9a0c41288a56351dffb962448fed6326
Parents: b5151f5
Author: Josh Elser 
Authored: Wed Feb 7 18:37:39 2018 -0500
Committer: Sean Busbey 
Committed: Mon Jun 18 11:31:13 2018 -0700

--
 hbase-assembly/pom.xml  |  33 +++--
 .../src/main/assembly/client-components.xml |  92 +
 hbase-assembly/src/main/assembly/client.xml | 137 +++
 hbase-procedure/pom.xml |   4 +-
 hbase-spark/pom.xml |   6 +
 pom.xml |  10 ++
 6 files changed, 268 insertions(+), 14 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/8710825a/hbase-assembly/pom.xml
--
diff --git a/hbase-assembly/pom.xml b/hbase-assembly/pom.xml
index 327ce72..c0e577c 100644
--- a/hbase-assembly/pom.xml
+++ b/hbase-assembly/pom.xml
@@ -95,6 +95,7 @@
   gnu
   
 ${assembly.file}
+src/main/assembly/client.xml
   
 
   
@@ -228,22 +229,22 @@
   test
 
 
-org.apache.hbase
-hbase-hadoop-compat
-
-  
-com.google.guava
-guava
-  
-
+  org.apache.hbase
+  hbase-hadoop-compat
+  
+
+  com.google.guava
+  guava
+
+  
 
 
-org.apache.hbase
-${compat.module}
+  org.apache.hbase
+  ${compat.module}
 
 
-   org.apache.hbase
-   hbase-shell
+  org.apache.hbase
+  hbase-shell
 
 
   org.apache.hbase
@@ -323,6 +324,14 @@
   jline
   jline
 
+
+  org.apache.hbase
+  hbase-shaded-client
+
+
+  org.apache.hbase
+  hbase-shaded-mapreduce
+
   
   
 

http://git-wip-us.apache.org/repos/asf/hbase/blob/8710825a/hbase-assembly/src/main/assembly/client-components.xml
--
diff --git a/hbase-assembly/src/main/assembly/client-components.xml 
b/hbase-assembly/src/main/assembly/client-components.xml
new file mode 100644
index 000..2fd1b57
--- /dev/null
+++ b/hbase-assembly/src/main/assembly/client-components.xml
@@ -0,0 +1,92 @@
+
+
+
+
+
+  
+
+
+  ${project.basedir}/../target/site
+  docs
+
+
+
+  ${project.basedir}/..
+  .
+  
+CHANGES.txt
+README.txt
+  
+  0644
+
+
+
+  ${project.basedir}/../conf
+  conf
+  0644
+  0755
+
+
+
+
+  ${project.basedir}/../bin
+  bin
+  
+get-active-master.rb
+hbase
+hbase-common.sh
+hbase-config.sh
+hbase-jruby
+hirb.rb
+
+  
+  0755
+  0755
+
+
+
+  ${project.basedir}/../bin
+  bin
+  
+hbase.cmd
+hbase-config.cmd
+  
+
+
+
+  ${project.basedir}/../hbase-shell/src/main/ruby
+  lib/ruby
+  0644
+  0755
+
+
+
+  ${project.basedir}/../hbase-server/target/native
+  lib/native
+  0755
+  0755
+  
+*.so
+*.dylib
+  
+
+  
+

http://git-wip-us.apache.org/repos/asf/hbase/blob/8710825a/hbase-assembly/src/main/assembly/client.xml
--
diff --git a/hbase-assembly/src/main/assembly/client.xml 
b/hbase-assembly/src/main/assembly/client.xml
new file mode 100644
index 000..7951961
--- /dev/null
+++ b/hbase-assembly/src/main/assembly/client.xml
@@ -0,0 +1,137 @@
+
+http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.1; 
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance; 
xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.1
 http://maven.apache.org/xsd/assembly-1.1.1.xsd;>
+
+
+  
+  client-bin
+  
+tar.gz
+  
+  
+  hbase-${project.version}-client
+  
+  
src/main/assembly/client-components.xml
+  
+  
+
+  true
+  
+
+org.apache.hbase:hbase-annotations
+org.apache.hbase:hbase-client
+org.apache.hbase:hbase-common
+org.apache.hbase:hbase-hadoop-compat
+

[2/5] hbase git commit: HBASE-20333 Provide a shaded client that allows downstream to provide Hadoop needs.

2018-06-18 Thread busbey

HBASE-20333 Provide a shaded client that allows downstream to provide Hadoop 
needs.


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/b5151f57
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/b5151f57
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/b5151f57

Branch: refs/heads/master
Commit: b5151f57da3dea9088c0ce3fa4ada753eed14d33
Parents: f1b536b
Author: Sean Busbey 
Authored: Tue Apr 24 14:51:12 2018 -0500
Committer: Sean Busbey 
Committed: Mon Jun 18 11:31:12 2018 -0700

--
 .../hbase-shaded-check-invariants/pom.xml   |  5 ++
 .../hbase-shaded-client-byo-hadoop/pom.xml  | 70 
 hbase-shaded/hbase-shaded-client/pom.xml| 35 --
 hbase-shaded/hbase-shaded-mapreduce/pom.xml | 30 ++---
 hbase-shaded/pom.xml|  6 ++
 5 files changed, 115 insertions(+), 31 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/b5151f57/hbase-shaded/hbase-shaded-check-invariants/pom.xml
--
diff --git a/hbase-shaded/hbase-shaded-check-invariants/pom.xml 
b/hbase-shaded/hbase-shaded-check-invariants/pom.xml
index 7ba4a41..287a986 100644
--- a/hbase-shaded/hbase-shaded-check-invariants/pom.xml
+++ b/hbase-shaded/hbase-shaded-check-invariants/pom.xml
@@ -48,6 +48,11 @@
   hbase-shaded-mapreduce
   ${project.version}
 
+
+  org.apache.hbase
+  hbase-shaded-client-byo-hadoop
+  ${project.version}
+
 
 
   com.github.stephenc.findbugs

http://git-wip-us.apache.org/repos/asf/hbase/blob/b5151f57/hbase-shaded/hbase-shaded-client-byo-hadoop/pom.xml
--
diff --git a/hbase-shaded/hbase-shaded-client-byo-hadoop/pom.xml 
b/hbase-shaded/hbase-shaded-client-byo-hadoop/pom.xml
new file mode 100644
index 000..c51a1af
--- /dev/null
+++ b/hbase-shaded/hbase-shaded-client-byo-hadoop/pom.xml
@@ -0,0 +1,70 @@
+http://maven.apache.org/POM/4.0.0;
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance;
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 
http://maven.apache.org/xsd/maven-4.0.0.xsd;>
+
+4.0.0
+
+hbase-shaded
+org.apache.hbase
+3.0.0-SNAPSHOT
+..
+
+hbase-shaded-client-byo-hadoop
+Apache HBase - Shaded - Client
+
+
+
+org.apache.maven.plugins
+maven-site-plugin
+
+true
+
+
+
+
+maven-assembly-plugin
+
+true
+
+
+
+
+
+
+org.apache.hbase
+hbase-client
+
+
+
+
+
+release
+
+
+
+org.apache.maven.plugins
+maven-shade-plugin
+
+
+
+
+
+

http://git-wip-us.apache.org/repos/asf/hbase/blob/b5151f57/hbase-shaded/hbase-shaded-client/pom.xml
--
diff --git a/hbase-shaded/hbase-shaded-client/pom.xml 
b/hbase-shaded/hbase-shaded-client/pom.xml
index 72a5b60..6152aad 100644
--- a/hbase-shaded/hbase-shaded-client/pom.xml
+++ b/hbase-shaded/hbase-shaded-client/pom.xml
@@ -28,7 +28,7 @@
 ..
 
 hbase-shaded-client
-Apache HBase - Shaded - Client
+Apache HBase - Shaded - Client (with Hadoop bundled)
 
 
 
@@ -51,6 +51,7 @@
 
 org.apache.hbase
 hbase-client
+${project.version}
 
 
 
@@ -59,10 +60,34 @@
 release
 
 
-
-org.apache.maven.plugins
-maven-shade-plugin
-
+
+org.apache.maven.plugins
+maven-shade-plugin
+
+
+
aggregate-into-a-jar-with-relocated-third-parties
+
+
+
+
+
+
org.apache.hbase:hbase-resource-bundle
+org.slf4j:*
+
com.google.code.findbugs:*
+
com.github.stephenc.findbugs:*
+org.apache.htrace:*
+

[4/5] hbase git commit: HBASE-20615 emphasize shaded artifacts in client tarball.

2018-06-18 Thread busbey

HBASE-20615 emphasize shaded artifacts in client tarball.


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/10cc1da7
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/10cc1da7
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/10cc1da7

Branch: refs/heads/master
Commit: 10cc1da76a926684bae080e508e6a79b4e8fb1fd
Parents: 8710825
Author: Sean Busbey 
Authored: Fri May 18 11:11:42 2018 -0500
Committer: Sean Busbey 
Committed: Mon Jun 18 11:31:13 2018 -0700

--
 bin/hbase   | 226 ---
 bin/hbase-config.sh |  10 +
 hbase-assembly/pom.xml  |  21 +-
 .../src/main/assembly/client-components.xml |  43 +++-
 hbase-assembly/src/main/assembly/client.xml | 131 ++-
 hbase-assembly/src/main/assembly/components.xml |   3 +-
 .../src/main/assembly/hadoop-two-compat.xml |  80 ++-
 pom.xml |   6 +
 8 files changed, 417 insertions(+), 103 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/10cc1da7/bin/hbase
--
diff --git a/bin/hbase b/bin/hbase
index 4f1c854..559a02e 100755
--- a/bin/hbase
+++ b/bin/hbase
@@ -71,11 +71,18 @@ if [ -d "${HBASE_HOME}/target" ]; then
   in_dev_env=true
 fi
 
+# Detect if we are in the omnibus tarball
+in_omnibus_tarball="false"
+if [ -f "${HBASE_HOME}/bin/hbase-daemons.sh" ]; then
+  in_omnibus_tarball="true"
+fi
+
 read -d '' options_string << EOF
 Options:
-  --config DIR Configuration direction to use. Default: ./conf
-  --hosts HOSTSOverride the list in 'regionservers' file
-  --auth-as-server Authenticate to ZooKeeper using servers configuration
+  --config DIR Configuration direction to use. Default: ./conf
+  --hosts HOSTSOverride the list in 'regionservers' file
+  --auth-as-server Authenticate to ZooKeeper using servers configuration
+  --internal-classpath Skip attempting to use client facing jars (WARNING: 
unstable results between versions)
 EOF
 # if no args specified, show usage
 if [ $# = 0 ]; then
@@ -87,16 +94,18 @@ if [ $# = 0 ]; then
   echo "  shell   Run the HBase shell"
   echo "  hbckRun the hbase 'fsck' tool"
   echo "  snapshotTool for managing snapshots"
-  echo "  wal Write-ahead-log analyzer"
-  echo "  hfile   Store file analyzer"
-  echo "  zkcli   Run the ZooKeeper shell"
-  echo "  master  Run an HBase HMaster node"
-  echo "  regionserverRun an HBase HRegionServer node"
-  echo "  zookeeper   Run a ZooKeeper server"
-  echo "  restRun an HBase REST server"
-  echo "  thrift  Run the HBase Thrift server"
-  echo "  thrift2 Run the HBase Thrift2 server"
-  echo "  clean   Run the HBase clean up script"
+  if [ "${in_omnibus_tarball}" = "true" ]; then
+echo "  wal Write-ahead-log analyzer"
+echo "  hfile   Store file analyzer"
+echo "  zkcli   Run the ZooKeeper shell"
+echo "  master  Run an HBase HMaster node"
+echo "  regionserverRun an HBase HRegionServer node"
+echo "  zookeeper   Run a ZooKeeper server"
+echo "  restRun an HBase REST server"
+echo "  thrift  Run the HBase Thrift server"
+echo "  thrift2 Run the HBase Thrift2 server"
+echo "  clean   Run the HBase clean up script"
+  fi
   echo "  classpath   Dump hbase CLASSPATH"
   echo "  mapredcpDump CLASSPATH entries required by mapreduce"
   echo "  pe  Run PerformanceEvaluation"
@@ -186,9 +195,99 @@ for f in $HBASE_HOME/hbase-jars/hbase*.jar; do
   fi
 done
 
+#If avail, add Hadoop to the CLASSPATH and to the JAVA_LIBRARY_PATH
+# Allow this functionality to be disabled
+if [ "$HBASE_DISABLE_HADOOP_CLASSPATH_LOOKUP" != "true" ] ; then
+  HADOOP_IN_PATH=$(PATH="${HADOOP_HOME:-${HADOOP_PREFIX}}/bin:$PATH" which 
hadoop 2>/dev/null)
+fi
+
 # Add libs to CLASSPATH
-for f in $HBASE_HOME/lib/*.jar; do
-  CLASSPATH=${CLASSPATH}:$f;
+declare shaded_jar
+
+if [ "${INTERNAL_CLASSPATH}" != "true" ]; then
+  # find our shaded jars
+  declare shaded_client
+  declare shaded_client_byo_hadoop
+  declare shaded_mapreduce
+  for f in "${HBASE_HOME}"/lib/shaded-clients/hbase-shaded-client*.jar; do
+if [[ "${f}" =~ byo-hadoop ]]; then
+  shaded_client_byo_hadoop="${f}"
+else
+  shaded_client="${f}"
+fi
+  done
+  for f in "${HBASE_HOME}"/lib/shaded-clients/hbase-shaded-mapreduce*.jar; do
+shaded_mapreduce="${f}"
+  done
+
+  # If command can use our shaded client, use it
+  declare -a commands_in_client_jar=("classpath" "version")
+  for c in

[1/6] hbase git commit: WIP tune down build retention while troubleshooting. [Forced Update!]

2018-06-18 Thread busbey

Repository: hbase
Updated Branches:
  refs/heads/HBASE-20331 0ad9018b4 -> 1d14768d6 (forced update)


WIP tune down build retention while troubleshooting.


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/1d14768d
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/1d14768d
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/1d14768d

Branch: refs/heads/HBASE-20331
Commit: 1d14768d6ac3a64af13ce3c4614142c1549dfbd5
Parents: 8fe40b0
Author: Sean Busbey 
Authored: Wed May 23 14:14:45 2018 -0500
Committer: Sean Busbey 
Committed: Mon Jun 18 00:51:46 2018 -0700

--
 dev-support/Jenkinsfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/1d14768d/dev-support/Jenkinsfile
--
diff --git a/dev-support/Jenkinsfile b/dev-support/Jenkinsfile
index 59d3227..64144e9 100644
--- a/dev-support/Jenkinsfile
+++ b/dev-support/Jenkinsfile
@@ -24,7 +24,7 @@ pipeline {
 cron('@daily')
   }
   options {
-buildDiscarder(logRotator(numToKeepStr: '30'))
+buildDiscarder(logRotator(numToKeepStr: '1'))
 timeout (time: 9, unit: 'HOURS')
 timestamps()
 skipDefaultCheckout()

[3/6] hbase git commit: HBASE-20332 shaded mapreduce module shouldn't include hadoop

2018-06-18 Thread busbey

HBASE-20332 shaded mapreduce module shouldn't include hadoop

* modify the jar checking script to take args; make hadoop stuff optional
* separate out checking the artifacts that have hadoop vs those that don't.
* * Unfortunately means we need two modules for checking things
* * put in a safety check that the support script for checking jar contents is 
maintained in both modules
* * have to carve out an exception for o.a.hadoop.metrics2. :(
* fix duplicated class warning
* clean up dependencies in hbase-server and some modules that depend on it.
* allow Hadoop to have its own htrace where it needs it
* add a precommit check to make sure we're not using old htrace imports


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/7137c548
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/7137c548
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/7137c548

Branch: refs/heads/HBASE-20331
Commit: 7137c548ff387bf8762c8e46558cb7ac4bc84aa1
Parents: ac5bb81
Author: Sean Busbey 
Authored: Mon Apr 9 13:37:44 2018 -0500
Committer: Sean Busbey 
Committed: Mon Jun 18 00:51:46 2018 -0700

--
 hbase-backup/pom.xml|  12 +-
 .../resources/hbase/checkstyle-suppressions.xml |  11 +
 .../src/main/resources/hbase/checkstyle.xml |   9 +-
 hbase-client/pom.xml|   4 -
 hbase-common/pom.xml|  12 --
 hbase-endpoint/pom.xml  |  14 --
 hbase-examples/pom.xml  |  12 --
 hbase-external-blockcache/pom.xml   |   4 -
 hbase-hadoop2-compat/pom.xml|   6 -
 hbase-it/pom.xml|   6 -
 hbase-mapreduce/pom.xml |  30 +--
 hbase-replication/pom.xml   |   4 -
 hbase-rest/pom.xml  |  19 +-
 hbase-rsgroup/pom.xml   |   4 -
 hbase-server/pom.xml| 103 -
 .../hbase-shaded-check-invariants/pom.xml   |  54 +++--
 .../ensure-jars-have-correct-contents.sh|  92 ++--
 hbase-shaded/hbase-shaded-mapreduce/pom.xml | 190 +++-
 .../pom.xml | 215 +++
 .../ensure-jars-have-correct-contents.sh| 129 +++
 hbase-shaded/pom.xml|  13 ++
 hbase-shell/pom.xml |  14 --
 hbase-testing-util/pom.xml  |  16 --
 hbase-thrift/pom.xml|  16 --
 pom.xml |  65 +++---
 25 files changed, 789 insertions(+), 265 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/7137c548/hbase-backup/pom.xml
--
diff --git a/hbase-backup/pom.xml b/hbase-backup/pom.xml
index 7afd51e..00a996f 100644
--- a/hbase-backup/pom.xml
+++ b/hbase-backup/pom.xml
@@ -155,10 +155,6 @@
   hadoop-common
   
 
-  org.apache.htrace
-  htrace-core
-
-
   net.java.dev.jets3t
   jets3t
 
@@ -264,9 +260,6 @@
   3.0
 
   
-  
-3.0-SNAPSHOT
-  
   
 
   org.apache.hadoop
@@ -276,6 +269,11 @@
   org.apache.hadoop
   hadoop-mapreduce-client-core
 
+
+  org.apache.hadoop
+  hadoop-distcp
+  ${hadoop.version}
+
   
 
   

http://git-wip-us.apache.org/repos/asf/hbase/blob/7137c548/hbase-checkstyle/src/main/resources/hbase/checkstyle-suppressions.xml
--
diff --git 
a/hbase-checkstyle/src/main/resources/hbase/checkstyle-suppressions.xml 
b/hbase-checkstyle/src/main/resources/hbase/checkstyle-suppressions.xml
index ad79163..325ed3c 100644
--- a/hbase-checkstyle/src/main/resources/hbase/checkstyle-suppressions.xml
+++ b/hbase-checkstyle/src/main/resources/hbase/checkstyle-suppressions.xml
@@ -2,6 +2,13 @@
 http://www.puppycrawl.com/dtds/suppressions_1_0.dtd;>
+
   
 

http://git-wip-us.apache.org/repos/asf/hbase/blob/7137c548/hbase-checkstyle/src/main/resources/hbase/checkstyle.xml
--
diff --git a/hbase-checkstyle/src/main/resources/hbase/checkstyle.xml 
b/hbase-checkstyle/src/main/resources/hbase/checkstyle.xml
index 7ad797c..94472e1 100644
--- a/hbase-checkstyle/src/main/resources/hbase/checkstyle.xml
+++ b/hbase-checkstyle/src/main/resources/hbase/checkstyle.xml
@@ -77,6 +77,12 @@
   
 
 
+
   
+  org.codehaus.jackson,
+  org.htrace"/>

[6/6] hbase git commit: HBASE-19735 Create a client-tarball assembly

2018-06-18 Thread busbey

HBASE-19735 Create a client-tarball assembly

Provides an extra client descriptor to build a second
tarball with a reduced set of dependencies. Not of great
impact now, but will build the way for better in the future.

Signed-off-by: Sean Busbey 

 Conflicts:
hbase-assembly/pom.xml


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/08beff12
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/08beff12
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/08beff12

Branch: refs/heads/HBASE-20331
Commit: 08beff12e71d29ec28cecbdf44fbd3bca67db5cf
Parents: 7142b5d
Author: Josh Elser 
Authored: Wed Feb 7 18:37:39 2018 -0500
Committer: Sean Busbey 
Committed: Mon Jun 18 00:51:46 2018 -0700

--
 hbase-assembly/pom.xml  |  33 +++--
 .../src/main/assembly/client-components.xml |  92 +
 hbase-assembly/src/main/assembly/client.xml | 137 +++
 hbase-procedure/pom.xml |   4 +-
 hbase-spark/pom.xml |   6 +
 pom.xml |  10 ++
 6 files changed, 268 insertions(+), 14 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/08beff12/hbase-assembly/pom.xml
--
diff --git a/hbase-assembly/pom.xml b/hbase-assembly/pom.xml
index 327ce72..c0e577c 100644
--- a/hbase-assembly/pom.xml
+++ b/hbase-assembly/pom.xml
@@ -95,6 +95,7 @@
   gnu
   
 ${assembly.file}
+src/main/assembly/client.xml
   
 
   
@@ -228,22 +229,22 @@
   test
 
 
-org.apache.hbase
-hbase-hadoop-compat
-
-  
-com.google.guava
-guava
-  
-
+  org.apache.hbase
+  hbase-hadoop-compat
+  
+
+  com.google.guava
+  guava
+
+  
 
 
-org.apache.hbase
-${compat.module}
+  org.apache.hbase
+  ${compat.module}
 
 
-   org.apache.hbase
-   hbase-shell
+  org.apache.hbase
+  hbase-shell
 
 
   org.apache.hbase
@@ -323,6 +324,14 @@
   jline
   jline
 
+
+  org.apache.hbase
+  hbase-shaded-client
+
+
+  org.apache.hbase
+  hbase-shaded-mapreduce
+
   
   
 

http://git-wip-us.apache.org/repos/asf/hbase/blob/08beff12/hbase-assembly/src/main/assembly/client-components.xml
--
diff --git a/hbase-assembly/src/main/assembly/client-components.xml 
b/hbase-assembly/src/main/assembly/client-components.xml
new file mode 100644
index 000..2fd1b57
--- /dev/null
+++ b/hbase-assembly/src/main/assembly/client-components.xml
@@ -0,0 +1,92 @@
+
+
+
+
+
+  
+
+
+  ${project.basedir}/../target/site
+  docs
+
+
+
+  ${project.basedir}/..
+  .
+  
+CHANGES.txt
+README.txt
+  
+  0644
+
+
+
+  ${project.basedir}/../conf
+  conf
+  0644
+  0755
+
+
+
+
+  ${project.basedir}/../bin
+  bin
+  
+get-active-master.rb
+hbase
+hbase-common.sh
+hbase-config.sh
+hbase-jruby
+hirb.rb
+
+  
+  0755
+  0755
+
+
+
+  ${project.basedir}/../bin
+  bin
+  
+hbase.cmd
+hbase-config.cmd
+  
+
+
+
+  ${project.basedir}/../hbase-shell/src/main/ruby
+  lib/ruby
+  0644
+  0755
+
+
+
+  ${project.basedir}/../hbase-server/target/native
+  lib/native
+  0755
+  0755
+  
+*.so
+*.dylib
+  
+
+  
+

http://git-wip-us.apache.org/repos/asf/hbase/blob/08beff12/hbase-assembly/src/main/assembly/client.xml
--
diff --git a/hbase-assembly/src/main/assembly/client.xml 
b/hbase-assembly/src/main/assembly/client.xml
new file mode 100644
index 000..7951961
--- /dev/null
+++ b/hbase-assembly/src/main/assembly/client.xml
@@ -0,0 +1,137 @@
+
+http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.1; 
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance; 
xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.1
 http://maven.apache.org/xsd/assembly-1.1.1.xsd;>
+
+
+  
+  client-bin
+  
+tar.gz
+  
+  
+  hbase-${project.version}-client
+  
+  
src/main/assembly/client-components.xml
+  
+  
+
+  true
+  
+
+org.apache.hbase:hbase-annotations
+org.apache.hbase:hbase-client
+org.apache.hbase:hbase-common
+org.apache.hbase:hbase-hadoop-compat
+

[2/6] hbase git commit: HBASE-20615 emphasize shaded artifacts in client tarball.

2018-06-18 Thread busbey

HBASE-20615 emphasize shaded artifacts in client tarball.


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/68ff18aa
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/68ff18aa
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/68ff18aa

Branch: refs/heads/HBASE-20331
Commit: 68ff18aa273dafe08a791b384aa6a7f380cac2c6
Parents: 08beff1
Author: Sean Busbey 
Authored: Fri May 18 11:11:42 2018 -0500
Committer: Sean Busbey 
Committed: Mon Jun 18 00:51:46 2018 -0700

--
 bin/hbase   | 226 ---
 bin/hbase-config.sh |  10 +
 hbase-assembly/pom.xml  |  21 +-
 .../src/main/assembly/client-components.xml |  43 +++-
 hbase-assembly/src/main/assembly/client.xml | 131 ++-
 hbase-assembly/src/main/assembly/components.xml |   3 +-
 .../src/main/assembly/hadoop-two-compat.xml |  80 ++-
 pom.xml |   6 +
 8 files changed, 417 insertions(+), 103 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/68ff18aa/bin/hbase
--
diff --git a/bin/hbase b/bin/hbase
index 4f1c854..559a02e 100755
--- a/bin/hbase
+++ b/bin/hbase
@@ -71,11 +71,18 @@ if [ -d "${HBASE_HOME}/target" ]; then
   in_dev_env=true
 fi
 
+# Detect if we are in the omnibus tarball
+in_omnibus_tarball="false"
+if [ -f "${HBASE_HOME}/bin/hbase-daemons.sh" ]; then
+  in_omnibus_tarball="true"
+fi
+
 read -d '' options_string << EOF
 Options:
-  --config DIR Configuration direction to use. Default: ./conf
-  --hosts HOSTSOverride the list in 'regionservers' file
-  --auth-as-server Authenticate to ZooKeeper using servers configuration
+  --config DIR Configuration direction to use. Default: ./conf
+  --hosts HOSTSOverride the list in 'regionservers' file
+  --auth-as-server Authenticate to ZooKeeper using servers configuration
+  --internal-classpath Skip attempting to use client facing jars (WARNING: 
unstable results between versions)
 EOF
 # if no args specified, show usage
 if [ $# = 0 ]; then
@@ -87,16 +94,18 @@ if [ $# = 0 ]; then
   echo "  shell   Run the HBase shell"
   echo "  hbckRun the hbase 'fsck' tool"
   echo "  snapshotTool for managing snapshots"
-  echo "  wal Write-ahead-log analyzer"
-  echo "  hfile   Store file analyzer"
-  echo "  zkcli   Run the ZooKeeper shell"
-  echo "  master  Run an HBase HMaster node"
-  echo "  regionserverRun an HBase HRegionServer node"
-  echo "  zookeeper   Run a ZooKeeper server"
-  echo "  restRun an HBase REST server"
-  echo "  thrift  Run the HBase Thrift server"
-  echo "  thrift2 Run the HBase Thrift2 server"
-  echo "  clean   Run the HBase clean up script"
+  if [ "${in_omnibus_tarball}" = "true" ]; then
+echo "  wal Write-ahead-log analyzer"
+echo "  hfile   Store file analyzer"
+echo "  zkcli   Run the ZooKeeper shell"
+echo "  master  Run an HBase HMaster node"
+echo "  regionserverRun an HBase HRegionServer node"
+echo "  zookeeper   Run a ZooKeeper server"
+echo "  restRun an HBase REST server"
+echo "  thrift  Run the HBase Thrift server"
+echo "  thrift2 Run the HBase Thrift2 server"
+echo "  clean   Run the HBase clean up script"
+  fi
   echo "  classpath   Dump hbase CLASSPATH"
   echo "  mapredcpDump CLASSPATH entries required by mapreduce"
   echo "  pe  Run PerformanceEvaluation"
@@ -186,9 +195,99 @@ for f in $HBASE_HOME/hbase-jars/hbase*.jar; do
   fi
 done
 
+#If avail, add Hadoop to the CLASSPATH and to the JAVA_LIBRARY_PATH
+# Allow this functionality to be disabled
+if [ "$HBASE_DISABLE_HADOOP_CLASSPATH_LOOKUP" != "true" ] ; then
+  HADOOP_IN_PATH=$(PATH="${HADOOP_HOME:-${HADOOP_PREFIX}}/bin:$PATH" which 
hadoop 2>/dev/null)
+fi
+
 # Add libs to CLASSPATH
-for f in $HBASE_HOME/lib/*.jar; do
-  CLASSPATH=${CLASSPATH}:$f;
+declare shaded_jar
+
+if [ "${INTERNAL_CLASSPATH}" != "true" ]; then
+  # find our shaded jars
+  declare shaded_client
+  declare shaded_client_byo_hadoop
+  declare shaded_mapreduce
+  for f in "${HBASE_HOME}"/lib/shaded-clients/hbase-shaded-client*.jar; do
+if [[ "${f}" =~ byo-hadoop ]]; then
+  shaded_client_byo_hadoop="${f}"
+else
+  shaded_client="${f}"
+fi
+  done
+  for f in "${HBASE_HOME}"/lib/shaded-clients/hbase-shaded-mapreduce*.jar; do
+shaded_mapreduce="${f}"
+  done
+
+  # If command can use our shaded client, use it
+  declare -a commands_in_client_jar=("classpath" "version")
+  for c in

[4/6] hbase git commit: HBASE-20333 Provide a shaded client that allows downstream to provide Hadoop needs.

2018-06-18 Thread busbey

HBASE-20333 Provide a shaded client that allows downstream to provide Hadoop 
needs.


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/7142b5d5
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/7142b5d5
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/7142b5d5

Branch: refs/heads/HBASE-20331
Commit: 7142b5d5db1b37cb25e49068034fbc4fd19fa39a
Parents: 7137c54
Author: Sean Busbey 
Authored: Tue Apr 24 14:51:12 2018 -0500
Committer: Sean Busbey 
Committed: Mon Jun 18 00:51:46 2018 -0700

--
 .../hbase-shaded-check-invariants/pom.xml   |  5 ++
 .../hbase-shaded-client-byo-hadoop/pom.xml  | 70 
 hbase-shaded/hbase-shaded-client/pom.xml| 35 --
 hbase-shaded/hbase-shaded-mapreduce/pom.xml | 30 ++---
 hbase-shaded/pom.xml|  6 ++
 5 files changed, 115 insertions(+), 31 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/7142b5d5/hbase-shaded/hbase-shaded-check-invariants/pom.xml
--
diff --git a/hbase-shaded/hbase-shaded-check-invariants/pom.xml 
b/hbase-shaded/hbase-shaded-check-invariants/pom.xml
index 7ba4a41..287a986 100644
--- a/hbase-shaded/hbase-shaded-check-invariants/pom.xml
+++ b/hbase-shaded/hbase-shaded-check-invariants/pom.xml
@@ -48,6 +48,11 @@
   hbase-shaded-mapreduce
   ${project.version}
 
+
+  org.apache.hbase
+  hbase-shaded-client-byo-hadoop
+  ${project.version}
+
 
 
   com.github.stephenc.findbugs

http://git-wip-us.apache.org/repos/asf/hbase/blob/7142b5d5/hbase-shaded/hbase-shaded-client-byo-hadoop/pom.xml
--
diff --git a/hbase-shaded/hbase-shaded-client-byo-hadoop/pom.xml 
b/hbase-shaded/hbase-shaded-client-byo-hadoop/pom.xml
new file mode 100644
index 000..c51a1af
--- /dev/null
+++ b/hbase-shaded/hbase-shaded-client-byo-hadoop/pom.xml
@@ -0,0 +1,70 @@
+http://maven.apache.org/POM/4.0.0;
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance;
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 
http://maven.apache.org/xsd/maven-4.0.0.xsd;>
+
+4.0.0
+
+hbase-shaded
+org.apache.hbase
+3.0.0-SNAPSHOT
+..
+
+hbase-shaded-client-byo-hadoop
+Apache HBase - Shaded - Client
+
+
+
+org.apache.maven.plugins
+maven-site-plugin
+
+true
+
+
+
+
+maven-assembly-plugin
+
+true
+
+
+
+
+
+
+org.apache.hbase
+hbase-client
+
+
+
+
+
+release
+
+
+
+org.apache.maven.plugins
+maven-shade-plugin
+
+
+
+
+
+

http://git-wip-us.apache.org/repos/asf/hbase/blob/7142b5d5/hbase-shaded/hbase-shaded-client/pom.xml
--
diff --git a/hbase-shaded/hbase-shaded-client/pom.xml 
b/hbase-shaded/hbase-shaded-client/pom.xml
index 72a5b60..6152aad 100644
--- a/hbase-shaded/hbase-shaded-client/pom.xml
+++ b/hbase-shaded/hbase-shaded-client/pom.xml
@@ -28,7 +28,7 @@
 ..
 
 hbase-shaded-client
-Apache HBase - Shaded - Client
+Apache HBase - Shaded - Client (with Hadoop bundled)
 
 
 
@@ -51,6 +51,7 @@
 
 org.apache.hbase
 hbase-client
+${project.version}
 
 
 
@@ -59,10 +60,34 @@
 release
 
 
-
-org.apache.maven.plugins
-maven-shade-plugin
-
+
+org.apache.maven.plugins
+maven-shade-plugin
+
+
+
aggregate-into-a-jar-with-relocated-third-parties
+
+
+
+
+
+
org.apache.hbase:hbase-resource-bundle
+org.slf4j:*
+
com.google.code.findbugs:*
+
com.github.stephenc.findbugs:*
+org.apache.htrace:*
+

[5/6] hbase git commit: HBASE-20334 add a test that verifies basic client and MR integration

2018-06-18 Thread busbey

HBASE-20334 add a test that verifies basic client and MR integration


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/8fe40b0b
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/8fe40b0b
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/8fe40b0b

Branch: refs/heads/HBASE-20331
Commit: 8fe40b0b3bd2375a1609cb21848d9961c27f51fe
Parents: 68ff18a
Author: Sean Busbey 
Authored: Tue May 1 14:28:52 2018 -0500
Committer: Sean Busbey 
Committed: Mon Jun 18 00:51:46 2018 -0700

--
 dev-support/Jenkinsfile | 233 +++--
 .../hbase_nightly_pseudo-distributed-test.sh| 516 +++
 dev-support/hbase_nightly_source-artifact.sh|  14 +-
 .../cache-apache-project-artifact.sh| 131 +
 4 files changed, 836 insertions(+), 58 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/8fe40b0b/dev-support/Jenkinsfile
--
diff --git a/dev-support/Jenkinsfile b/dev-support/Jenkinsfile
index 2311e35..59d3227 100644
--- a/dev-support/Jenkinsfile
+++ b/dev-support/Jenkinsfile
@@ -60,54 +60,109 @@ pipeline {
 booleanParam(name: 'DEBUG', defaultValue: false, description: 'Produce a 
lot more meta-information.')
   }
   stages {
-stage ('yetus install') {
+stage ('scm-checkout') {
   steps {
-sh  '''#!/usr/bin/env bash
-set -e
-echo "Ensure we have a copy of Apache Yetus."
-if [[ true !=  "${USE_YETUS_PRERELEASE}" ]]; then
-  YETUS_DIR="${WORKSPACE}/yetus-${YETUS_RELEASE}"
-  echo "Checking for Yetus ${YETUS_RELEASE} in '${YETUS_DIR}'"
-  if [ ! -d "${YETUS_DIR}" ]; then
-echo "New download of Apache Yetus version ${YETUS_RELEASE}."
-rm -rf "${WORKSPACE}/.gpg"
-mkdir -p "${WORKSPACE}/.gpg"
-chmod -R 700 "${WORKSPACE}/.gpg"
-
-echo "install yetus project KEYS"
-curl -L --fail -o "${WORKSPACE}/KEYS_YETUS" 
https://dist.apache.org/repos/dist/release/yetus/KEYS
-gpg --homedir "${WORKSPACE}/.gpg" --import "${WORKSPACE}/KEYS_YETUS"
-
-echo "download yetus release ${YETUS_RELEASE}"
-curl -L --fail -O 
"https://dist.apache.org/repos/dist/release/yetus/${YETUS_RELEASE}/yetus-${YETUS_RELEASE}-bin.tar.gz;
-curl -L --fail -O 
"https://dist.apache.org/repos/dist/release/yetus/${YETUS_RELEASE}/yetus-${YETUS_RELEASE}-bin.tar.gz.asc;
-echo "verifying yetus release"
-gpg --homedir "${WORKSPACE}/.gpg" --verify 
"yetus-${YETUS_RELEASE}-bin.tar.gz.asc"
-mv "yetus-${YETUS_RELEASE}-bin.tar.gz" yetus.tar.gz
-  else
-echo "Reusing cached download of Apache Yetus version ${YETUS_RELEASE}."
-  fi
-else
-  YETUS_DIR="${WORKSPACE}/yetus-git"
-  rm -rf "${YETUS_DIR}"
-  echo "downloading from github"
-  curl -L --fail https://api.github.com/repos/apache/yetus/tarball/HEAD -o 
yetus.tar.gz
-fi
-if [ ! -d "${YETUS_DIR}" ]; then
-  echo "unpacking yetus into '${YETUS_DIR}'"
-  mkdir -p "${YETUS_DIR}"
-  gunzip -c yetus.tar.gz | tar xpf - -C "${YETUS_DIR}" --strip-components 1
-fi
-'''
-// Set up the file we need at PERSONALITY_FILE location
-dir ("tools") {
-  sh """#!/usr/bin/env bash
-set -e
-echo "Downloading Project personality."
-curl -L  -o personality.sh "${env.PROJECT_PERSONALITY}"
-  """
+dir('component') {
+  checkout scm
+}
+  }
+}
+stage ('thirdparty installs') {
+  parallel {
+stage ('yetus install') {
+  steps {
+// directory must be unique for each parallel stage, because 
jenkins runs them in the same workspace :(
+dir('downloads-yetus') {
+  // can't just do a simple echo or the directory won't be 
created. :(
+  sh '''#!/usr/bin/env bash
+echo "Make sure we have a directory for downloading 
dependencies: $(pwd)"
+'''
+}
+sh  '''#!/usr/bin/env bash
+  set -e
+  echo "Ensure we have a copy of Apache Yetus."
+  if [[ true !=  "${USE_YETUS_PRERELEASE}" ]]; then
+YETUS_DIR="${WORKSPACE}/yetus-${YETUS_RELEASE}"
+echo "Checking for Yetus ${YETUS_RELEASE} in '${YETUS_DIR}'"
+if [ ! -d "${YETUS_DIR}" ]; then
+  
"${WORKSPACE}/component/dev-support/jenkins-scripts/cache-apache-project-artifact.sh"
 \
+  --working-dir "${WORKSPACE}/downloads-yetus" \
+  --keys 'https://www.apache.org/dist/yetus/KEYS' \
+  "${WORKSPACE}/yetus-${YETUS_RELEASE}-bin.tar.gz" \
+  
"yetus/${YETUS_RELEASE}/yetus-${YETUS_RELEASE}-bin.tar.gz"
+  mv "yetus-${YETUS_RELEASE}-bin.tar.gz" yetus.tar.gz
+else
+  echo "Reusing cached install of Apache

[6/9] hbase git commit: HBASE-20615 emphasize shaded artifacts in client tarball.

2018-06-18 Thread busbey

HBASE-20615 emphasize shaded artifacts in client tarball.


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/305379d4
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/305379d4
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/305379d4

Branch: refs/heads/HBASE-20331
Commit: 305379d45e4a9577f795f8c795dad12e636f6764
Parents: 877781b
Author: Sean Busbey 
Authored: Fri May 18 11:11:42 2018 -0500
Committer: Sean Busbey 
Committed: Mon Jun 18 00:43:28 2018 -0700

--
 bin/hbase   | 226 ---
 bin/hbase-config.sh |  10 +
 hbase-assembly/pom.xml  |  21 +-
 .../src/main/assembly/client-components.xml |  43 +++-
 hbase-assembly/src/main/assembly/client.xml | 131 ++-
 hbase-assembly/src/main/assembly/components.xml |   3 +-
 .../src/main/assembly/hadoop-two-compat.xml |  80 ++-
 pom.xml |   6 +
 8 files changed, 417 insertions(+), 103 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/305379d4/bin/hbase
--
diff --git a/bin/hbase b/bin/hbase
index 4f1c854..559a02e 100755
--- a/bin/hbase
+++ b/bin/hbase
@@ -71,11 +71,18 @@ if [ -d "${HBASE_HOME}/target" ]; then
   in_dev_env=true
 fi
 
+# Detect if we are in the omnibus tarball
+in_omnibus_tarball="false"
+if [ -f "${HBASE_HOME}/bin/hbase-daemons.sh" ]; then
+  in_omnibus_tarball="true"
+fi
+
 read -d '' options_string << EOF
 Options:
-  --config DIR Configuration direction to use. Default: ./conf
-  --hosts HOSTSOverride the list in 'regionservers' file
-  --auth-as-server Authenticate to ZooKeeper using servers configuration
+  --config DIR Configuration direction to use. Default: ./conf
+  --hosts HOSTSOverride the list in 'regionservers' file
+  --auth-as-server Authenticate to ZooKeeper using servers configuration
+  --internal-classpath Skip attempting to use client facing jars (WARNING: 
unstable results between versions)
 EOF
 # if no args specified, show usage
 if [ $# = 0 ]; then
@@ -87,16 +94,18 @@ if [ $# = 0 ]; then
   echo "  shell   Run the HBase shell"
   echo "  hbckRun the hbase 'fsck' tool"
   echo "  snapshotTool for managing snapshots"
-  echo "  wal Write-ahead-log analyzer"
-  echo "  hfile   Store file analyzer"
-  echo "  zkcli   Run the ZooKeeper shell"
-  echo "  master  Run an HBase HMaster node"
-  echo "  regionserverRun an HBase HRegionServer node"
-  echo "  zookeeper   Run a ZooKeeper server"
-  echo "  restRun an HBase REST server"
-  echo "  thrift  Run the HBase Thrift server"
-  echo "  thrift2 Run the HBase Thrift2 server"
-  echo "  clean   Run the HBase clean up script"
+  if [ "${in_omnibus_tarball}" = "true" ]; then
+echo "  wal Write-ahead-log analyzer"
+echo "  hfile   Store file analyzer"
+echo "  zkcli   Run the ZooKeeper shell"
+echo "  master  Run an HBase HMaster node"
+echo "  regionserverRun an HBase HRegionServer node"
+echo "  zookeeper   Run a ZooKeeper server"
+echo "  restRun an HBase REST server"
+echo "  thrift  Run the HBase Thrift server"
+echo "  thrift2 Run the HBase Thrift2 server"
+echo "  clean   Run the HBase clean up script"
+  fi
   echo "  classpath   Dump hbase CLASSPATH"
   echo "  mapredcpDump CLASSPATH entries required by mapreduce"
   echo "  pe  Run PerformanceEvaluation"
@@ -186,9 +195,99 @@ for f in $HBASE_HOME/hbase-jars/hbase*.jar; do
   fi
 done
 
+#If avail, add Hadoop to the CLASSPATH and to the JAVA_LIBRARY_PATH
+# Allow this functionality to be disabled
+if [ "$HBASE_DISABLE_HADOOP_CLASSPATH_LOOKUP" != "true" ] ; then
+  HADOOP_IN_PATH=$(PATH="${HADOOP_HOME:-${HADOOP_PREFIX}}/bin:$PATH" which 
hadoop 2>/dev/null)
+fi
+
 # Add libs to CLASSPATH
-for f in $HBASE_HOME/lib/*.jar; do
-  CLASSPATH=${CLASSPATH}:$f;
+declare shaded_jar
+
+if [ "${INTERNAL_CLASSPATH}" != "true" ]; then
+  # find our shaded jars
+  declare shaded_client
+  declare shaded_client_byo_hadoop
+  declare shaded_mapreduce
+  for f in "${HBASE_HOME}"/lib/shaded-clients/hbase-shaded-client*.jar; do
+if [[ "${f}" =~ byo-hadoop ]]; then
+  shaded_client_byo_hadoop="${f}"
+else
+  shaded_client="${f}"
+fi
+  done
+  for f in "${HBASE_HOME}"/lib/shaded-clients/hbase-shaded-mapreduce*.jar; do
+shaded_mapreduce="${f}"
+  done
+
+  # If command can use our shaded client, use it
+  declare -a commands_in_client_jar=("classpath" "version")
+  for c in

[9/9] hbase git commit: HBASE-19735 Create a client-tarball assembly

2018-06-18 Thread busbey

HBASE-19735 Create a client-tarball assembly

Provides an extra client descriptor to build a second
tarball with a reduced set of dependencies. Not of great
impact now, but will build the way for better in the future.

Signed-off-by: Sean Busbey 

 Conflicts:
hbase-assembly/pom.xml


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/877781b6
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/877781b6
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/877781b6

Branch: refs/heads/HBASE-20331
Commit: 877781b65c7bd41cddba47b5b2ed087a0926f902
Parents: a2b91e1
Author: Josh Elser 
Authored: Wed Feb 7 18:37:39 2018 -0500
Committer: Sean Busbey 
Committed: Mon Jun 18 00:43:28 2018 -0700

--
 hbase-assembly/pom.xml  |  33 +++--
 .../src/main/assembly/client-components.xml |  92 +
 hbase-assembly/src/main/assembly/client.xml | 137 +++
 hbase-procedure/pom.xml |   4 +-
 hbase-spark/pom.xml |   6 +
 pom.xml |  10 ++
 6 files changed, 268 insertions(+), 14 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/877781b6/hbase-assembly/pom.xml
--
diff --git a/hbase-assembly/pom.xml b/hbase-assembly/pom.xml
index 327ce72..c0e577c 100644
--- a/hbase-assembly/pom.xml
+++ b/hbase-assembly/pom.xml
@@ -95,6 +95,7 @@
   gnu
   
 ${assembly.file}
+src/main/assembly/client.xml
   
 
   
@@ -228,22 +229,22 @@
   test
 
 
-org.apache.hbase
-hbase-hadoop-compat
-
-  
-com.google.guava
-guava
-  
-
+  org.apache.hbase
+  hbase-hadoop-compat
+  
+
+  com.google.guava
+  guava
+
+  
 
 
-org.apache.hbase
-${compat.module}
+  org.apache.hbase
+  ${compat.module}
 
 
-   org.apache.hbase
-   hbase-shell
+  org.apache.hbase
+  hbase-shell
 
 
   org.apache.hbase
@@ -323,6 +324,14 @@
   jline
   jline
 
+
+  org.apache.hbase
+  hbase-shaded-client
+
+
+  org.apache.hbase
+  hbase-shaded-mapreduce
+
   
   
 

http://git-wip-us.apache.org/repos/asf/hbase/blob/877781b6/hbase-assembly/src/main/assembly/client-components.xml
--
diff --git a/hbase-assembly/src/main/assembly/client-components.xml 
b/hbase-assembly/src/main/assembly/client-components.xml
new file mode 100644
index 000..2fd1b57
--- /dev/null
+++ b/hbase-assembly/src/main/assembly/client-components.xml
@@ -0,0 +1,92 @@
+
+
+
+
+
+  
+
+
+  ${project.basedir}/../target/site
+  docs
+
+
+
+  ${project.basedir}/..
+  .
+  
+CHANGES.txt
+README.txt
+  
+  0644
+
+
+
+  ${project.basedir}/../conf
+  conf
+  0644
+  0755
+
+
+
+
+  ${project.basedir}/../bin
+  bin
+  
+get-active-master.rb
+hbase
+hbase-common.sh
+hbase-config.sh
+hbase-jruby
+hirb.rb
+
+  
+  0755
+  0755
+
+
+
+  ${project.basedir}/../bin
+  bin
+  
+hbase.cmd
+hbase-config.cmd
+  
+
+
+
+  ${project.basedir}/../hbase-shell/src/main/ruby
+  lib/ruby
+  0644
+  0755
+
+
+
+  ${project.basedir}/../hbase-server/target/native
+  lib/native
+  0755
+  0755
+  
+*.so
+*.dylib
+  
+
+  
+

http://git-wip-us.apache.org/repos/asf/hbase/blob/877781b6/hbase-assembly/src/main/assembly/client.xml
--
diff --git a/hbase-assembly/src/main/assembly/client.xml 
b/hbase-assembly/src/main/assembly/client.xml
new file mode 100644
index 000..7951961
--- /dev/null
+++ b/hbase-assembly/src/main/assembly/client.xml
@@ -0,0 +1,137 @@
+
+http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.1; 
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance; 
xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.1
 http://maven.apache.org/xsd/assembly-1.1.1.xsd;>
+
+
+  
+  client-bin
+  
+tar.gz
+  
+  
+  hbase-${project.version}-client
+  
+  
src/main/assembly/client-components.xml
+  
+  
+
+  true
+  
+
+org.apache.hbase:hbase-annotations
+org.apache.hbase:hbase-client
+org.apache.hbase:hbase-common
+org.apache.hbase:hbase-hadoop-compat
+

[8/9] hbase git commit: HBASE-20334 add a test that verifies basic client and MR integration

2018-06-18 Thread busbey

HBASE-20334 add a test that verifies basic client and MR integration


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/ce259c54
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/ce259c54
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/ce259c54

Branch: refs/heads/HBASE-20331
Commit: ce259c54758362457159c395ba2e261e01c91b1e
Parents: 305379d
Author: Sean Busbey 
Authored: Tue May 1 14:28:52 2018 -0500
Committer: Sean Busbey 
Committed: Mon Jun 18 00:43:28 2018 -0700

--
 dev-support/Jenkinsfile | 233 +++--
 .../hbase_nightly_pseudo-distributed-test.sh| 516 +++
 dev-support/hbase_nightly_source-artifact.sh|  14 +-
 .../cache-apache-project-artifact.sh| 131 +
 4 files changed, 836 insertions(+), 58 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/ce259c54/dev-support/Jenkinsfile
--
diff --git a/dev-support/Jenkinsfile b/dev-support/Jenkinsfile
index 2311e35..59d3227 100644
--- a/dev-support/Jenkinsfile
+++ b/dev-support/Jenkinsfile
@@ -60,54 +60,109 @@ pipeline {
 booleanParam(name: 'DEBUG', defaultValue: false, description: 'Produce a 
lot more meta-information.')
   }
   stages {
-stage ('yetus install') {
+stage ('scm-checkout') {
   steps {
-sh  '''#!/usr/bin/env bash
-set -e
-echo "Ensure we have a copy of Apache Yetus."
-if [[ true !=  "${USE_YETUS_PRERELEASE}" ]]; then
-  YETUS_DIR="${WORKSPACE}/yetus-${YETUS_RELEASE}"
-  echo "Checking for Yetus ${YETUS_RELEASE} in '${YETUS_DIR}'"
-  if [ ! -d "${YETUS_DIR}" ]; then
-echo "New download of Apache Yetus version ${YETUS_RELEASE}."
-rm -rf "${WORKSPACE}/.gpg"
-mkdir -p "${WORKSPACE}/.gpg"
-chmod -R 700 "${WORKSPACE}/.gpg"
-
-echo "install yetus project KEYS"
-curl -L --fail -o "${WORKSPACE}/KEYS_YETUS" 
https://dist.apache.org/repos/dist/release/yetus/KEYS
-gpg --homedir "${WORKSPACE}/.gpg" --import "${WORKSPACE}/KEYS_YETUS"
-
-echo "download yetus release ${YETUS_RELEASE}"
-curl -L --fail -O 
"https://dist.apache.org/repos/dist/release/yetus/${YETUS_RELEASE}/yetus-${YETUS_RELEASE}-bin.tar.gz;
-curl -L --fail -O 
"https://dist.apache.org/repos/dist/release/yetus/${YETUS_RELEASE}/yetus-${YETUS_RELEASE}-bin.tar.gz.asc;
-echo "verifying yetus release"
-gpg --homedir "${WORKSPACE}/.gpg" --verify 
"yetus-${YETUS_RELEASE}-bin.tar.gz.asc"
-mv "yetus-${YETUS_RELEASE}-bin.tar.gz" yetus.tar.gz
-  else
-echo "Reusing cached download of Apache Yetus version ${YETUS_RELEASE}."
-  fi
-else
-  YETUS_DIR="${WORKSPACE}/yetus-git"
-  rm -rf "${YETUS_DIR}"
-  echo "downloading from github"
-  curl -L --fail https://api.github.com/repos/apache/yetus/tarball/HEAD -o 
yetus.tar.gz
-fi
-if [ ! -d "${YETUS_DIR}" ]; then
-  echo "unpacking yetus into '${YETUS_DIR}'"
-  mkdir -p "${YETUS_DIR}"
-  gunzip -c yetus.tar.gz | tar xpf - -C "${YETUS_DIR}" --strip-components 1
-fi
-'''
-// Set up the file we need at PERSONALITY_FILE location
-dir ("tools") {
-  sh """#!/usr/bin/env bash
-set -e
-echo "Downloading Project personality."
-curl -L  -o personality.sh "${env.PROJECT_PERSONALITY}"
-  """
+dir('component') {
+  checkout scm
+}
+  }
+}
+stage ('thirdparty installs') {
+  parallel {
+stage ('yetus install') {
+  steps {
+// directory must be unique for each parallel stage, because 
jenkins runs them in the same workspace :(
+dir('downloads-yetus') {
+  // can't just do a simple echo or the directory won't be 
created. :(
+  sh '''#!/usr/bin/env bash
+echo "Make sure we have a directory for downloading 
dependencies: $(pwd)"
+'''
+}
+sh  '''#!/usr/bin/env bash
+  set -e
+  echo "Ensure we have a copy of Apache Yetus."
+  if [[ true !=  "${USE_YETUS_PRERELEASE}" ]]; then
+YETUS_DIR="${WORKSPACE}/yetus-${YETUS_RELEASE}"
+echo "Checking for Yetus ${YETUS_RELEASE} in '${YETUS_DIR}'"
+if [ ! -d "${YETUS_DIR}" ]; then
+  
"${WORKSPACE}/component/dev-support/jenkins-scripts/cache-apache-project-artifact.sh"
 \
+  --working-dir "${WORKSPACE}/downloads-yetus" \
+  --keys 'https://www.apache.org/dist/yetus/KEYS' \
+  "${WORKSPACE}/yetus-${YETUS_RELEASE}-bin.tar.gz" \
+  
"yetus/${YETUS_RELEASE}/yetus-${YETUS_RELEASE}-bin.tar.gz"
+  mv "yetus-${YETUS_RELEASE}-bin.tar.gz" yetus.tar.gz
+else
+  echo "Reusing cached install of Apache

[5/9] hbase git commit: HBASE-20332 shaded mapreduce module shouldn't include hadoop

2018-06-18 Thread busbey

HBASE-20332 shaded mapreduce module shouldn't include hadoop

* modify the jar checking script to take args; make hadoop stuff optional
* separate out checking the artifacts that have hadoop vs those that don't.
* * Unfortunately means we need two modules for checking things
* * put in a safety check that the support script for checking jar contents is 
maintained in both modules
* * have to carve out an exception for o.a.hadoop.metrics2. :(
* fix duplicated class warning
* clean up dependencies in hbase-server and some modules that depend on it.
* allow Hadoop to have its own htrace where it needs it
* add a precommit check to make sure we're not using old htrace imports


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/3580dd61
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/3580dd61
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/3580dd61

Branch: refs/heads/HBASE-20331
Commit: 3580dd61ed9544775f10bc54632c8ee526526ab5
Parents: ac5bb81
Author: Sean Busbey 
Authored: Mon Apr 9 13:37:44 2018 -0500
Committer: Sean Busbey 
Committed: Mon Jun 18 00:43:27 2018 -0700

--
 hbase-backup/pom.xml|  12 +-
 .../resources/hbase/checkstyle-suppressions.xml |  11 +
 .../src/main/resources/hbase/checkstyle.xml |   9 +-
 hbase-client/pom.xml|   4 -
 hbase-common/pom.xml|  12 --
 hbase-endpoint/pom.xml  |  14 --
 hbase-examples/pom.xml  |  12 --
 hbase-external-blockcache/pom.xml   |   4 -
 hbase-hadoop2-compat/pom.xml|   6 -
 hbase-it/pom.xml|   6 -
 hbase-mapreduce/pom.xml |  30 +--
 hbase-replication/pom.xml   |   4 -
 hbase-rest/pom.xml  |  19 +-
 hbase-rsgroup/pom.xml   |   4 -
 hbase-server/pom.xml| 103 -
 .../hbase-shaded-check-invariants/pom.xml   |  54 +++--
 .../ensure-jars-have-correct-contents.sh|  92 ++--
 hbase-shaded/hbase-shaded-mapreduce/pom.xml | 190 +++-
 .../pom.xml | 215 +++
 .../ensure-jars-have-correct-contents.sh| 129 +++
 hbase-shaded/pom.xml|  13 ++
 hbase-shell/pom.xml |  14 --
 hbase-testing-util/pom.xml  |  16 --
 hbase-thrift/pom.xml|  16 --
 pom.xml |  65 +++---
 25 files changed, 789 insertions(+), 265 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/3580dd61/hbase-backup/pom.xml
--
diff --git a/hbase-backup/pom.xml b/hbase-backup/pom.xml
index 7afd51e..00a996f 100644
--- a/hbase-backup/pom.xml
+++ b/hbase-backup/pom.xml
@@ -155,10 +155,6 @@
   hadoop-common
   
 
-  org.apache.htrace
-  htrace-core
-
-
   net.java.dev.jets3t
   jets3t
 
@@ -264,9 +260,6 @@
   3.0
 
   
-  
-3.0-SNAPSHOT
-  
   
 
   org.apache.hadoop
@@ -276,6 +269,11 @@
   org.apache.hadoop
   hadoop-mapreduce-client-core
 
+
+  org.apache.hadoop
+  hadoop-distcp
+  ${hadoop.version}
+
   
 
   

http://git-wip-us.apache.org/repos/asf/hbase/blob/3580dd61/hbase-checkstyle/src/main/resources/hbase/checkstyle-suppressions.xml
--
diff --git 
a/hbase-checkstyle/src/main/resources/hbase/checkstyle-suppressions.xml 
b/hbase-checkstyle/src/main/resources/hbase/checkstyle-suppressions.xml
index ad79163..e3f79e6 100644
--- a/hbase-checkstyle/src/main/resources/hbase/checkstyle-suppressions.xml
+++ b/hbase-checkstyle/src/main/resources/hbase/checkstyle-suppressions.xml
@@ -2,6 +2,13 @@
 http://www.puppycrawl.com/dtds/suppressions_1_0.dtd;>
+
   
 

http://git-wip-us.apache.org/repos/asf/hbase/blob/3580dd61/hbase-checkstyle/src/main/resources/hbase/checkstyle.xml
--
diff --git a/hbase-checkstyle/src/main/resources/hbase/checkstyle.xml 
b/hbase-checkstyle/src/main/resources/hbase/checkstyle.xml
index 7ad797c..94472e1 100644
--- a/hbase-checkstyle/src/main/resources/hbase/checkstyle.xml
+++ b/hbase-checkstyle/src/main/resources/hbase/checkstyle.xml
@@ -77,6 +77,12 @@
   
 
 
+
   
+  org.codehaus.jackson,
+  org.htrace"/>

[1/9] hbase git commit: HBASE-20737 put collection into ArrayList instead of addAll function [Forced Update!]

2018-06-18 Thread busbey

Repository: hbase
Updated Branches:
  refs/heads/HBASE-20331 4f59a6edc -> 0ad9018b4 (forced update)


HBASE-20737 put collection into ArrayList instead of addAll function

Signed-off-by: Chia-Ping Tsai 


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/0e43abc7
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/0e43abc7
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/0e43abc7

Branch: refs/heads/HBASE-20331
Commit: 0e43abc78a5a55c84a7572125d498a950dabfb57
Parents: 30772ee
Author: taiynlee 
Authored: Fri Jun 15 09:34:43 2018 +
Committer: Chia-Ping Tsai 
Committed: Sat Jun 16 03:25:42 2018 +0800

--
 .../org/apache/hadoop/hbase/master/ClusterStatusPublisher.java| 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/0e43abc7/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ClusterStatusPublisher.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ClusterStatusPublisher.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ClusterStatusPublisher.java
index 21fa263..af35ce4 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ClusterStatusPublisher.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ClusterStatusPublisher.java
@@ -187,8 +187,7 @@ public class ClusterStatusPublisher extends ScheduledChore {
 }
 
 // We're sending the new deads first.
-List> entries = new ArrayList<>();
-entries.addAll(lastSent.entrySet());
+List> entries = new 
ArrayList<>(lastSent.entrySet());
 Collections.sort(entries, new Comparator>() 
{
   @Override
   public int compare(Map.Entry o1, 
Map.Entry o2) {

[2/9] hbase git commit: HBASE-20730 Add pv2 and amv2 chapters to refguide Signed-off-by: Mike Drob

2018-06-18 Thread busbey

HBASE-20730 Add pv2 and amv2 chapters to refguide
Signed-off-by: Mike Drob 


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/08254624
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/08254624
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/08254624

Branch: refs/heads/HBASE-20331
Commit: 082546243692a7a80760b88515fa6ec5c198f00c
Parents: 0e43abc
Author: Michael Stack 
Authored: Wed Jun 13 20:19:10 2018 -0700
Committer: Michael Stack 
Committed: Fri Jun 15 15:41:30 2018 -0700

--
 src/main/asciidoc/_chapters/amv2.adoc | 173 +
 src/main/asciidoc/_chapters/pv2.adoc  | 163 +++
 src/main/asciidoc/book.adoc   |   2 +
 3 files changed, 338 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/08254624/src/main/asciidoc/_chapters/amv2.adoc
--
diff --git a/src/main/asciidoc/_chapters/amv2.adoc 
b/src/main/asciidoc/_chapters/amv2.adoc
new file mode 100644
index 000..49841ce
--- /dev/null
+++ b/src/main/asciidoc/_chapters/amv2.adoc
@@ -0,0 +1,173 @@
+
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+[[amv2]]
+= AMv2 Description for Devs
+:doctype: book
+:numbered:
+:toc: left
+:icons: font
+:experimental:
+
+The AssignmentManager (AM) in HBase Master manages assignment of Regions over 
a cluster of RegionServers.
+
+The AMv2 project is a redo of Assignment in an attempt at addressing the root 
cause of many of our operational issues in production, namely slow assignment 
and problematic accounting such that Regions are misplaced stuck offline in the 
notorious _Regions-In-Transition (RIT)_ limbo state.
+
+Below are notes for devs on key aspects of AMv2 in no particular order.
+
+== Background
+
+Assignment in HBase 1.x has been problematic in operation. It is not hard to 
see why. Region state is kept at the other end of an RPC in ZooKeeper (Terminal 
states -- i.e. OPEN or CLOSED -- are published to the _hbase:meta_ table). In 
HBase-1.x.x, state has multiple writers with Master and RegionServers all able 
to make state edits concurrently (in _hbase:meta_ table and out on ZooKeeper). 
If clocks are awry or watchers missed, state changes can be skipped or 
overwritten. Locking of HBase Entities -- tables, regions -- is not 
comprehensive so a table operation -- disable/enable -- could clash with a 
region-level operation; a split or merge. Region state is distributed and hard 
to reason about and test. Assignment is slow in operation because each assign 
involves moving remote znodes through transitions. Cluster size tends to top 
out at a couple of hundred thousand regions; beyond this, cluster start/stop 
takes hours and is prone to corruption.
+
+AMv2 (AssignmentManager Version 2) is a refactor 
(https://issues.apache.org/jira/browse/HBASE-14350[HBASE-14350]) of the 
hbase-1.x AssignmentManager putting it up on a 
https://issues.apache.org/jira/browse/HBASE-12439[ProcedureV2 (HBASE-12439)] 
basis. ProcedureV2 (Pv2)__,__ is an awkwardly named system that allows 
describing and running multi-step state machines. It is performant and persists 
all state to a Store which is recoverable post crash. See the companion chapter 
on <>, to learn more about the ProcedureV2 system.
+
+In AMv2, all assignment, crash handling, splits and merges are recast as 
Procedures(v2).  ZooKeeper is purged from the mix. As before, the final 
assignment state gets published to _hbase:meta_ for non-Master participants to 
read (all-clients) with intermediate state kept in the local Pv2 WAL-based 
âstoreâ but only the active Master, a single-writer, evolves state. The 
Masterâs in-memory cluster image is the authority and if disagreement, 
RegionServers are forced to comply. Pv2 adds shared/exclusive locking of all 
core HBase Entities -- namespace, tables, and regions -- to ensure one actor at 
a time access and to prevent operations contending over resources (move/split, 
disable/assign, etc.).
+

[4/9] hbase git commit: HBASE-20333 Provide a shaded client that allows downstream to provide Hadoop needs.

2018-06-18 Thread busbey

HBASE-20333 Provide a shaded client that allows downstream to provide Hadoop 
needs.


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/a2b91e1d
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/a2b91e1d
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/a2b91e1d

Branch: refs/heads/HBASE-20331
Commit: a2b91e1da8ff66712ba333e226f1b4c63099a2f9
Parents: 3580dd6
Author: Sean Busbey 
Authored: Tue Apr 24 14:51:12 2018 -0500
Committer: Sean Busbey 
Committed: Mon Jun 18 00:43:27 2018 -0700

--
 .../hbase-shaded-check-invariants/pom.xml   |  5 ++
 .../hbase-shaded-client-byo-hadoop/pom.xml  | 70 
 hbase-shaded/hbase-shaded-client/pom.xml| 35 --
 hbase-shaded/hbase-shaded-mapreduce/pom.xml | 30 ++---
 hbase-shaded/pom.xml|  6 ++
 5 files changed, 115 insertions(+), 31 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/a2b91e1d/hbase-shaded/hbase-shaded-check-invariants/pom.xml
--
diff --git a/hbase-shaded/hbase-shaded-check-invariants/pom.xml 
b/hbase-shaded/hbase-shaded-check-invariants/pom.xml
index 7ba4a41..287a986 100644
--- a/hbase-shaded/hbase-shaded-check-invariants/pom.xml
+++ b/hbase-shaded/hbase-shaded-check-invariants/pom.xml
@@ -48,6 +48,11 @@
   hbase-shaded-mapreduce
   ${project.version}
 
+
+  org.apache.hbase
+  hbase-shaded-client-byo-hadoop
+  ${project.version}
+
 
 
   com.github.stephenc.findbugs

http://git-wip-us.apache.org/repos/asf/hbase/blob/a2b91e1d/hbase-shaded/hbase-shaded-client-byo-hadoop/pom.xml
--
diff --git a/hbase-shaded/hbase-shaded-client-byo-hadoop/pom.xml 
b/hbase-shaded/hbase-shaded-client-byo-hadoop/pom.xml
new file mode 100644
index 000..c51a1af
--- /dev/null
+++ b/hbase-shaded/hbase-shaded-client-byo-hadoop/pom.xml
@@ -0,0 +1,70 @@
+http://maven.apache.org/POM/4.0.0;
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance;
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 
http://maven.apache.org/xsd/maven-4.0.0.xsd;>
+
+4.0.0
+
+hbase-shaded
+org.apache.hbase
+3.0.0-SNAPSHOT
+..
+
+hbase-shaded-client-byo-hadoop
+Apache HBase - Shaded - Client
+
+
+
+org.apache.maven.plugins
+maven-site-plugin
+
+true
+
+
+
+
+maven-assembly-plugin
+
+true
+
+
+
+
+
+
+org.apache.hbase
+hbase-client
+
+
+
+
+
+release
+
+
+
+org.apache.maven.plugins
+maven-shade-plugin
+
+
+
+
+
+

http://git-wip-us.apache.org/repos/asf/hbase/blob/a2b91e1d/hbase-shaded/hbase-shaded-client/pom.xml
--
diff --git a/hbase-shaded/hbase-shaded-client/pom.xml 
b/hbase-shaded/hbase-shaded-client/pom.xml
index 72a5b60..6152aad 100644
--- a/hbase-shaded/hbase-shaded-client/pom.xml
+++ b/hbase-shaded/hbase-shaded-client/pom.xml
@@ -28,7 +28,7 @@
 ..
 
 hbase-shaded-client
-Apache HBase - Shaded - Client
+Apache HBase - Shaded - Client (with Hadoop bundled)
 
 
 
@@ -51,6 +51,7 @@
 
 org.apache.hbase
 hbase-client
+${project.version}
 
 
 
@@ -59,10 +60,34 @@
 release
 
 
-
-org.apache.maven.plugins
-maven-shade-plugin
-
+
+org.apache.maven.plugins
+maven-shade-plugin
+
+
+
aggregate-into-a-jar-with-relocated-third-parties
+
+
+
+
+
+
org.apache.hbase:hbase-resource-bundle
+org.slf4j:*
+
com.google.code.findbugs:*
+
com.github.stephenc.findbugs:*
+org.apache.htrace:*
+

[7/9] hbase git commit: WIP tune down build retention while troubleshooting.

2018-06-18 Thread busbey

WIP tune down build retention while troubleshooting.


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/0ad9018b
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/0ad9018b
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/0ad9018b

Branch: refs/heads/HBASE-20331
Commit: 0ad9018b4bf9091bb0abf72e428948e9e4bd90a5
Parents: ce259c5
Author: Sean Busbey 
Authored: Wed May 23 14:14:45 2018 -0500
Committer: Sean Busbey 
Committed: Mon Jun 18 00:43:28 2018 -0700

--
 dev-support/Jenkinsfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/0ad9018b/dev-support/Jenkinsfile
--
diff --git a/dev-support/Jenkinsfile b/dev-support/Jenkinsfile
index 59d3227..64144e9 100644
--- a/dev-support/Jenkinsfile
+++ b/dev-support/Jenkinsfile
@@ -24,7 +24,7 @@ pipeline {
 cron('@daily')
   }
   options {
-buildDiscarder(logRotator(numToKeepStr: '30'))
+buildDiscarder(logRotator(numToKeepStr: '1'))
 timeout (time: 9, unit: 'HOURS')
 timestamps()
 skipDefaultCheckout()

[3/9] hbase git commit: HBASE-20723 Custom hbase.wal.dir results in data loss because we write recovered edits into a different place than where the recovering region server looks for them

2018-06-18 Thread busbey

HBASE-20723 Custom hbase.wal.dir results in data loss because we write 
recovered edits into a different place than where the recovering region server 
looks for them


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/ac5bb815
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/ac5bb815
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/ac5bb815

Branch: refs/heads/HBASE-20331
Commit: ac5bb8155b618194fe9cf1131f0e72c99b7b534c
Parents: 0825462
Author: tedyu 
Authored: Fri Jun 15 19:40:48 2018 -0700
Committer: tedyu 
Committed: Fri Jun 15 19:40:48 2018 -0700

--
 .../apache/hadoop/hbase/wal/FSHLogProvider.java |  3 +-
 .../apache/hadoop/hbase/wal/WALSplitter.java| 55 +++-
 .../apache/hadoop/hbase/wal/TestWALFactory.java |  4 +-
 .../apache/hadoop/hbase/wal/TestWALSplit.java   |  8 +--
 4 files changed, 39 insertions(+), 31 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/ac5bb815/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/FSHLogProvider.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/FSHLogProvider.java 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/FSHLogProvider.java
index efcd377..44f692d 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/FSHLogProvider.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/FSHLogProvider.java
@@ -73,7 +73,8 @@ public class FSHLogProvider extends 
AbstractFSWALProvider {
 Writer writer = null;
 try {
   writer = logWriterClass.getDeclaredConstructor().newInstance();
-  writer.init(fs, path, conf, overwritable, blocksize);
+  FileSystem rootFs = FileSystem.get(path.toUri(), conf);
+  writer.init(rootFs, path, conf, overwritable, blocksize);
   return writer;
 } catch (Exception e) { 
   if (e instanceof CommonFSUtils.StreamLacksCapabilityException) {

http://git-wip-us.apache.org/repos/asf/hbase/blob/ac5bb815/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALSplitter.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALSplitter.java 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALSplitter.java
index dee5fb0..f020e7a 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALSplitter.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALSplitter.java
@@ -115,7 +115,7 @@ public class WALSplitter {
   public static final boolean SPLIT_SKIP_ERRORS_DEFAULT = false;
 
   // Parameters for split process
-  protected final Path rootDir;
+  protected final Path walDir;
   protected final FileSystem fs;
   protected final Configuration conf;
 
@@ -148,14 +148,14 @@ public class WALSplitter {
 
 
   @VisibleForTesting
-  WALSplitter(final WALFactory factory, Configuration conf, Path rootDir,
+  WALSplitter(final WALFactory factory, Configuration conf, Path walDir,
   FileSystem fs, LastSequenceId idChecker,
   SplitLogWorkerCoordination splitLogWorkerCoordination) {
 this.conf = HBaseConfiguration.create(conf);
 String codecClassName = conf
 .get(WALCellCodec.WAL_CELL_CODEC_CLASS_KEY, 
WALCellCodec.class.getName());
 this.conf.set(HConstants.RPC_CODEC_CONF_KEY, codecClassName);
-this.rootDir = rootDir;
+this.walDir = walDir;
 this.fs = fs;
 this.sequenceIdChecker = idChecker;
 this.splitLogWorkerCoordination = splitLogWorkerCoordination;
@@ -186,11 +186,11 @@ public class WALSplitter {
* 
* @return false if it is interrupted by the progress-able.
*/
-  public static boolean splitLogFile(Path rootDir, FileStatus logfile, 
FileSystem fs,
+  public static boolean splitLogFile(Path walDir, FileStatus logfile, 
FileSystem fs,
   Configuration conf, CancelableProgressable reporter, LastSequenceId 
idChecker,
   SplitLogWorkerCoordination splitLogWorkerCoordination, final WALFactory 
factory)
   throws IOException {
-WALSplitter s = new WALSplitter(factory, conf, rootDir, fs, idChecker,
+WALSplitter s = new WALSplitter(factory, conf, walDir, fs, idChecker,
 splitLogWorkerCoordination);
 return s.splitLogFile(logfile, reporter);
   }
@@ -322,10 +322,10 @@ public class WALSplitter {
   LOG.warn("Could not parse, corrupted WAL={}", logPath, e);
   if (splitLogWorkerCoordination != null) {
 // Some tests pass in a csm of null.
-splitLogWorkerCoordination.markCorrupted(rootDir, 
logfile.getPath().getName(), fs);
+splitLogWorkerCoordination.markCorrupted(walDir, 
logfile.getPath().getName(), fs);
   } else {
 // for tests only
-ZKSplitLog.markCorrupted(rootDir,

hbase git commit: HBASE-20727 Persist FlushedSequenceId to speed up WAL split after cluster restart

hbase-site git commit: INFRA-10751 Empty commit

[1/3] hbase-site git commit: Published site at 78da0e366970cf9bab6add9ba9fd74ca37c75be3.

[2/3] hbase-site git commit: Published site at 78da0e366970cf9bab6add9ba9fd74ca37c75be3.

[3/3] hbase-site git commit: Published site at 78da0e366970cf9bab6add9ba9fd74ca37c75be3.

[hbase] Git Push Summary

[3/6] hbase git commit: HBASE-20333 Provide a shaded client that allows downstream to provide Hadoop needs.

[2/6] hbase git commit: HBASE-20332 shaded mapreduce module shouldn't include hadoop

[5/6] hbase git commit: HBASE-20615 emphasize shaded artifacts in client tarball.

[1/6] hbase git commit: HBASE-20478 Update checkstyle to v8.2

[4/6] hbase git commit: HBASE-19735 Create a client-tarball assembly

[6/6] hbase git commit: HBASE-20334 add a test that verifies basic client and MR integration

[1/5] hbase git commit: HBASE-20332 shaded mapreduce module shouldn't include hadoop

[5/5] hbase git commit: HBASE-20334 add a test that verifies basic client and MR integration

[3/5] hbase git commit: HBASE-19735 Create a client-tarball assembly

[2/5] hbase git commit: HBASE-20333 Provide a shaded client that allows downstream to provide Hadoop needs.

[4/5] hbase git commit: HBASE-20615 emphasize shaded artifacts in client tarball.

[1/6] hbase git commit: WIP tune down build retention while troubleshooting. [Forced Update!]

[3/6] hbase git commit: HBASE-20332 shaded mapreduce module shouldn't include hadoop

[6/6] hbase git commit: HBASE-19735 Create a client-tarball assembly

[2/6] hbase git commit: HBASE-20615 emphasize shaded artifacts in client tarball.

[4/6] hbase git commit: HBASE-20333 Provide a shaded client that allows downstream to provide Hadoop needs.

[5/6] hbase git commit: HBASE-20334 add a test that verifies basic client and MR integration

[6/9] hbase git commit: HBASE-20615 emphasize shaded artifacts in client tarball.

[9/9] hbase git commit: HBASE-19735 Create a client-tarball assembly

[8/9] hbase git commit: HBASE-20334 add a test that verifies basic client and MR integration

[5/9] hbase git commit: HBASE-20332 shaded mapreduce module shouldn't include hadoop

[1/9] hbase git commit: HBASE-20737 put collection into ArrayList instead of addAll function [Forced Update!]

[2/9] hbase git commit: HBASE-20730 Add pv2 and amv2 chapters to refguide Signed-off-by: Mike Drob

[4/9] hbase git commit: HBASE-20333 Provide a shaded client that allows downstream to provide Hadoop needs.

[7/9] hbase git commit: WIP tune down build retention while troubleshooting.

[3/9] hbase git commit: HBASE-20723 Custom hbase.wal.dir results in data loss because we write recovered edits into a different place than where the recovering region server looks for them

32 matches

Site Navigation

Mail list logo

Footer information