hbase git commit: HBASE-18426 ADDENDUM leave the workspace around on pipeline failure so we can browse it.

[busbey]

Repository: hbase
Updated Branches:
  refs/heads/HBASE-18426 [created] 076a09347


HBASE-18426 ADDENDUM leave the workspace around on pipeline failure so we can 
browse it.


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/076a0934
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/076a0934
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/076a0934

Branch: refs/heads/HBASE-18426
Commit: 076a0934793fea3784d32550cbbffab1a4b49f75
Parents: 2a71745
Author: Sean Busbey 
Authored: Sun Aug 6 14:46:56 2017 -0500
Committer: Sean Busbey 
Committed: Sun Aug 6 14:46:56 2017 -0500

--
 dev-support/Jenkinsfile | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/076a0934/dev-support/Jenkinsfile
--
diff --git a/dev-support/Jenkinsfile b/dev-support/Jenkinsfile
index 94d8c3f..240a7f4 100644
--- a/dev-support/Jenkinsfile
+++ b/dev-support/Jenkinsfile
@@ -299,8 +299,8 @@ END
 }
   }
   post {
-failure {
-  deleteDir()
-}
+  //  failure {
+  //deleteDir()
+  //  }
   }
 }

[15/17] hbase git commit: HBASE-14220 nightly check that we can build a source tarball.

[busbey]

HBASE-14220 nightly check that we can build a source tarball.

Signed-off-by: Michael Stack 


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/cd683e61
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/cd683e61
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/cd683e61

Branch: refs/heads/branch-1.2
Commit: cd683e612637e5db596c109da20bd1884235b819
Parents: cb36ecf
Author: Sean Busbey 
Authored: Mon Jul 31 01:08:16 2017 -0500
Committer: Sean Busbey 
Committed: Sun Aug 6 12:20:54 2017 -0500

--
 dev-support/Jenkinsfile | 56 
 1 file changed, 56 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/cd683e61/dev-support/Jenkinsfile
--
diff --git a/dev-support/Jenkinsfile b/dev-support/Jenkinsfile
index b5c7660..cdb62b9 100644
--- a/dev-support/Jenkinsfile
+++ b/dev-support/Jenkinsfile
@@ -241,6 +241,62 @@ curl -L  -o personality.sh "${env.PROJET_PERSONALITY}"
 }
   }
 }
+// This is meant to mimic what a release manager will do to create RCs.
+// See http://hbase.apache.org/book.html#maven.release
+stage ('create source tarball') {
+  tools {
+maven 'Maven (latest)'
+// this needs to be set to the jdk that ought to be used to build 
releases on the branch the Jenkinsfile is stored in.
+jdk "JDK 1.7 (latest)"
+  }
+  steps {
+sh 'mvn --version --offline'
+dir ("component") {
+  sh '''#!/bin/bash -e
+echo "Clean..."
+mvn --batch-mode -DskipTests clean
+echo "Step 3 Build the source tarball"
+mvn -Prelease --batch-mode 
-Dmaven.repo.local="${WORKSPACE}/.m2-for-repo" install -DskipTests 
assembly:single -Dassembly.file=hbase-assembly/src/main/assembly/src.xml
+'''
+}
+dir ("unpacked_src_tarball") {
+  sh '''#!/bin/bash -e
+echo "Unpack the source tarball"
+tar --strip-components=1 -xzf 
../component/hbase-assembly/target/hbase-*-src.tar.gz
+echo "Building from source artifact."
+mvn -DskipTests -Prelease --batch-mode 
-Dmaven.repo.local="${WORKSPACE}/.m2-for-src" clean install assembly:single
+echo "Clean up after checking ability to build."
+mvn -DskipTests --batch-mode clean
+'''
+}
+dir ("component") {
+  sh '''#!/bin/bash -e
+echo "Clean up the source checkout"
+mvn -DskipTests --batch-mode clean
+echo "Diff against source tree"
+diff --binary --recursive . ../unpacked_src_tarball 
>../diff_output || true
+'''
+}
+// expectation check largely based on HBASE-14952
+sh '''#!/bin/bash -e
+  echo "Checking against things we don't expect to include in the 
source tarball (git related, hbase-native-client, etc.)"
+  cat >known_excluded  side."
+echo "In a given set of differences, '.' refers to the branch in 
the repo and 'unpacked_src_tarball' refers to what we pulled out of the 
tarball."
+diff known_excluded diff_output
+  else
+echo "Everything looks as expected."
+  fi
+'''
+  }
+}
   }
   post {
 failure {

[04/17] hbase git commit: HBASE-18426 nightly job should use independent stages to check supported jdks

[busbey]

HBASE-18426 nightly job should use independent stages to check supported jdks

* Jenkinsfile that works for all current branches.
* adds dev-support script for setting parameters for our yetus nightly 
invocation
* quiets the "zip test results" step

Signed-off-by: tedyu 


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/db18a00b
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/db18a00b
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/db18a00b

Branch: refs/heads/branch-2
Commit: db18a00b76154755d00d675779203bca1888cac2
Parents: 3c8b364
Author: Sean Busbey 
Authored: Thu Jul 20 01:01:59 2017 -0500
Committer: Sean Busbey 
Committed: Sun Aug 6 12:08:48 2017 -0500

--
 dev-support/Jenkinsfile| 256 +++-
 dev-support/hbase_nightly_yetus.sh |  86 +++
 2 files changed, 240 insertions(+), 102 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/db18a00b/dev-support/Jenkinsfile
--
diff --git a/dev-support/Jenkinsfile b/dev-support/Jenkinsfile
index 26f72d7..b5c7660 100644
--- a/dev-support/Jenkinsfile
+++ b/dev-support/Jenkinsfile
@@ -20,10 +20,6 @@ pipeline {
   label 'Hadoop'
 }
   }
-  // work around for YETUS-508, requires maven outside of the dockerfile
-  tools {
-maven 'Maven (latest)'
-  }
   triggers {
 cron('@daily')
   }
@@ -38,8 +34,23 @@ pipeline {
 BASEDIR = "${env.WORKSPACE}/component"
 YETUS_RELEASE = '0.5.0'
 // where we'll write everything from different steps.
-OUTPUT_RELATIVE = 'output'
-OUTPUTDIR = "${env.WORKSPACE}/output"
+OUTPUT_RELATIVE_GENERAL = 'output-general'
+OUTPUTDIR_GENERAL = "${env.WORKSPACE}/output-general"
+OUTPUT_RELATIVE_JDK7 = 'output-jdk7'
+OUTPUTDIR_JDK7 = "${env.WORKSPACE}/output-jdk7"
+OUTPUT_RELATIVE_JDK8 = 'output-jdk8'
+OUTPUTDIR_JDK8 = "${env.WORKSPACE}/output-jdk8"
+PROJECT = 'hbase'
+PROJET_PERSONALITY = 
'https://raw.githubusercontent.com/apache/hbase/master/dev-support/hbase-personality.sh'
+// This section of the docs tells folks not to use the javadoc tag. older 
branches have our old version of the check for said tag.
+AUTHOR_IGNORE_LIST = 
'src/main/asciidoc/_chapters/developer.adoc,dev-support/test-patch.sh'
+WHITESPACE_IGNORE_LIST = '.*/generated/.*'
+// output from surefire; sadly the archive function in yetus only works on 
file names.
+ARCHIVE_PATTERN_LIST = 'TEST-*.xml,org.apache.h*.txt'
+// These tests currently have known failures. Once they burn down to 0, 
remove from here so that new problems will cause a failure.
+TESTS_FILTER = 
'cc,checkstyle,javac,javadoc,pylint,shellcheck,whitespace,perlcritic,ruby-lint,rubocop'
+BRANCH_SPECIFIC_DOCKERFILE = "${env.BASEDIR}/dev-support/docker/Dockerfile"
+EXCLUDE_TESTS_URL = 
'https://builds.apache.org/job/HBase-Find-Flaky-Tests/lastSuccessfulBuild/artifact/excludes/'
   }
   parameters {
 booleanParam(name: 'USE_YETUS_PRERELEASE', defaultValue: false, 
description: '''Check to use the current HEAD of apache/yetus rather than our 
configured release.
@@ -48,24 +59,9 @@ pipeline {
 booleanParam(name: 'DEBUG', defaultValue: false, description: 'Produce a 
lot more meta-information.')
   }
   stages {
-stage ('yetus check') {
-  environment {
-PROJECT = 'hbase'
-PROJECT_PERSONALITY = 
'https://git-wip-us.apache.org/repos/asf?p=hbase.git;a=blob_plain;f=dev-support/hbase-personality.sh;hb=refs/heads/master'
-// This section of the docs tells folks not to use the javadoc tag. 
older branches have our old version of the check for said tag.
-AUTHOR_IGNORE_LIST = 
'src/main/asciidoc/_chapters/developer.adoc,dev-support/test-patch.sh'
-WHITESPACE_IGNORE_LIST = '.*/generated/.*'
-// output from surefire; sadly the archive function in yetus only 
works on file names.
-ARCHIVE_PATTERN_LIST = 
'TEST-*.xml,org.apache.h*-output.txt,org.apache.h*.txt'
-// These tests currently have known failures. Once they burn down to 0, remove 
from here so that new problems will cause a failure.
-TESTS_FILTER = 
'cc,checkstyle,javac,javadoc,pylint,shellcheck,whitespace,perlcritic,ruby-lint,rubocop'
-BRANCH_SPECIFIC_DOCKERFILE = 
"${env.BASEDIR}/dev-support/docker/Dockerfile"
-EXCLUDE_TESTS_URL = 
'https://builds.apache.org/job/HBase-Find-Flaky-Tests/lastSuccessfulBuild/artifact/excludes/'
-  }
+stage ('yetus install') {
   steps {
-// TODO we can move the yetus install into a different stage and then use 
stash to deploy it.
 sh  '''#!/usr/bin/env bash
-printenv
 echo "Ensure we have a copy of Apache Yetus."
 if [[ 

[02/17] hbase git commit: HBASE-14220 nightly check that we can build a source tarball.

[busbey]

HBASE-14220 nightly check that we can build a source tarball.

Signed-off-by: Michael Stack 


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/2a717459
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/2a717459
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/2a717459

Branch: refs/heads/master
Commit: 2a717459bdad0f8ab64ecc45300e9022cf210af9
Parents: 04424ad
Author: Sean Busbey 
Authored: Mon Jul 31 01:08:16 2017 -0500
Committer: Sean Busbey 
Committed: Sun Aug 6 12:01:47 2017 -0500

--
 dev-support/Jenkinsfile | 56 
 1 file changed, 56 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/2a717459/dev-support/Jenkinsfile
--
diff --git a/dev-support/Jenkinsfile b/dev-support/Jenkinsfile
index b5c7660..94d8c3f 100644
--- a/dev-support/Jenkinsfile
+++ b/dev-support/Jenkinsfile
@@ -241,6 +241,62 @@ curl -L  -o personality.sh "${env.PROJET_PERSONALITY}"
 }
   }
 }
+// This is meant to mimic what a release manager will do to create RCs.
+// See http://hbase.apache.org/book.html#maven.release
+stage ('create source tarball') {
+  tools {
+maven 'Maven (latest)'
+// this needs to be set to the jdk that ought to be used to build 
releases on the branch the Jenkinsfile is stored in.
+jdk "JDK 1.8 (latest)"
+  }
+  steps {
+sh 'mvn --version --offline'
+dir ("component") {
+  sh '''#!/bin/bash -e
+echo "Clean..."
+mvn --batch-mode -DskipTests clean
+echo "Step 3 Build the source tarball"
+mvn -Prelease --batch-mode 
-Dmaven.repo.local="${WORKSPACE}/.m2-for-repo" install -DskipTests 
assembly:single -Dassembly.file=hbase-assembly/src/main/assembly/src.xml
+'''
+}
+dir ("unpacked_src_tarball") {
+  sh '''#!/bin/bash -e
+echo "Unpack the source tarball"
+tar --strip-components=1 -xzf 
../component/hbase-assembly/target/hbase-*-src.tar.gz
+echo "Building from source artifact."
+mvn -DskipTests -Prelease --batch-mode 
-Dmaven.repo.local="${WORKSPACE}/.m2-for-src" clean install assembly:single
+echo "Clean up after checking ability to build."
+mvn -DskipTests --batch-mode clean
+'''
+}
+dir ("component") {
+  sh '''#!/bin/bash -e
+echo "Clean up the source checkout"
+mvn -DskipTests --batch-mode clean
+echo "Diff against source tree"
+diff --binary --recursive . ../unpacked_src_tarball 
>../diff_output || true
+'''
+}
+// expectation check largely based on HBASE-14952
+sh '''#!/bin/bash -e
+  echo "Checking against things we don't expect to include in the 
source tarball (git related, hbase-native-client, etc.)"
+  cat >known_excluded  side."
+echo "In a given set of differences, '.' refers to the branch in 
the repo and 'unpacked_src_tarball' refers to what we pulled out of the 
tarball."
+diff known_excluded diff_output
+  else
+echo "Everything looks as expected."
+  fi
+'''
+  }
+}
   }
   post {
 failure {

[17/17] hbase git commit: HBASE-14220 nightly check that we can build a source tarball.

[busbey]

HBASE-14220 nightly check that we can build a source tarball.

Signed-off-by: Michael Stack 


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/68fb6b96
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/68fb6b96
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/68fb6b96

Branch: refs/heads/branch-1.1
Commit: 68fb6b9603d03749b1fddc5cb86a371871f38fe1
Parents: de2951d
Author: Sean Busbey 
Authored: Mon Jul 31 01:08:16 2017 -0500
Committer: Sean Busbey 
Committed: Sun Aug 6 12:23:04 2017 -0500

--
 dev-support/Jenkinsfile | 56 
 1 file changed, 56 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/68fb6b96/dev-support/Jenkinsfile
--
diff --git a/dev-support/Jenkinsfile b/dev-support/Jenkinsfile
index b5c7660..cdb62b9 100644
--- a/dev-support/Jenkinsfile
+++ b/dev-support/Jenkinsfile
@@ -241,6 +241,62 @@ curl -L  -o personality.sh "${env.PROJET_PERSONALITY}"
 }
   }
 }
+// This is meant to mimic what a release manager will do to create RCs.
+// See http://hbase.apache.org/book.html#maven.release
+stage ('create source tarball') {
+  tools {
+maven 'Maven (latest)'
+// this needs to be set to the jdk that ought to be used to build 
releases on the branch the Jenkinsfile is stored in.
+jdk "JDK 1.7 (latest)"
+  }
+  steps {
+sh 'mvn --version --offline'
+dir ("component") {
+  sh '''#!/bin/bash -e
+echo "Clean..."
+mvn --batch-mode -DskipTests clean
+echo "Step 3 Build the source tarball"
+mvn -Prelease --batch-mode 
-Dmaven.repo.local="${WORKSPACE}/.m2-for-repo" install -DskipTests 
assembly:single -Dassembly.file=hbase-assembly/src/main/assembly/src.xml
+'''
+}
+dir ("unpacked_src_tarball") {
+  sh '''#!/bin/bash -e
+echo "Unpack the source tarball"
+tar --strip-components=1 -xzf 
../component/hbase-assembly/target/hbase-*-src.tar.gz
+echo "Building from source artifact."
+mvn -DskipTests -Prelease --batch-mode 
-Dmaven.repo.local="${WORKSPACE}/.m2-for-src" clean install assembly:single
+echo "Clean up after checking ability to build."
+mvn -DskipTests --batch-mode clean
+'''
+}
+dir ("component") {
+  sh '''#!/bin/bash -e
+echo "Clean up the source checkout"
+mvn -DskipTests --batch-mode clean
+echo "Diff against source tree"
+diff --binary --recursive . ../unpacked_src_tarball 
>../diff_output || true
+'''
+}
+// expectation check largely based on HBASE-14952
+sh '''#!/bin/bash -e
+  echo "Checking against things we don't expect to include in the 
source tarball (git related, hbase-native-client, etc.)"
+  cat >known_excluded  side."
+echo "In a given set of differences, '.' refers to the branch in 
the repo and 'unpacked_src_tarball' refers to what we pulled out of the 
tarball."
+diff known_excluded diff_output
+  else
+echo "Everything looks as expected."
+  fi
+'''
+  }
+}
   }
   post {
 failure {

[13/17] hbase git commit: HBASE-14220 nightly check that we can build a source tarball.

[busbey]

HBASE-14220 nightly check that we can build a source tarball.

Signed-off-by: Michael Stack 


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/f30b624e
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/f30b624e
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/f30b624e

Branch: refs/heads/branch-1.3
Commit: f30b624e1728cc8d703155182e82b7a40e8fd055
Parents: 879f7a5
Author: Sean Busbey 
Authored: Mon Jul 31 01:08:16 2017 -0500
Committer: Sean Busbey 
Committed: Sun Aug 6 12:19:51 2017 -0500

--
 dev-support/Jenkinsfile | 56 
 1 file changed, 56 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/f30b624e/dev-support/Jenkinsfile
--
diff --git a/dev-support/Jenkinsfile b/dev-support/Jenkinsfile
index b5c7660..cdb62b9 100644
--- a/dev-support/Jenkinsfile
+++ b/dev-support/Jenkinsfile
@@ -241,6 +241,62 @@ curl -L  -o personality.sh "${env.PROJET_PERSONALITY}"
 }
   }
 }
+// This is meant to mimic what a release manager will do to create RCs.
+// See http://hbase.apache.org/book.html#maven.release
+stage ('create source tarball') {
+  tools {
+maven 'Maven (latest)'
+// this needs to be set to the jdk that ought to be used to build 
releases on the branch the Jenkinsfile is stored in.
+jdk "JDK 1.7 (latest)"
+  }
+  steps {
+sh 'mvn --version --offline'
+dir ("component") {
+  sh '''#!/bin/bash -e
+echo "Clean..."
+mvn --batch-mode -DskipTests clean
+echo "Step 3 Build the source tarball"
+mvn -Prelease --batch-mode 
-Dmaven.repo.local="${WORKSPACE}/.m2-for-repo" install -DskipTests 
assembly:single -Dassembly.file=hbase-assembly/src/main/assembly/src.xml
+'''
+}
+dir ("unpacked_src_tarball") {
+  sh '''#!/bin/bash -e
+echo "Unpack the source tarball"
+tar --strip-components=1 -xzf 
../component/hbase-assembly/target/hbase-*-src.tar.gz
+echo "Building from source artifact."
+mvn -DskipTests -Prelease --batch-mode 
-Dmaven.repo.local="${WORKSPACE}/.m2-for-src" clean install assembly:single
+echo "Clean up after checking ability to build."
+mvn -DskipTests --batch-mode clean
+'''
+}
+dir ("component") {
+  sh '''#!/bin/bash -e
+echo "Clean up the source checkout"
+mvn -DskipTests --batch-mode clean
+echo "Diff against source tree"
+diff --binary --recursive . ../unpacked_src_tarball 
>../diff_output || true
+'''
+}
+// expectation check largely based on HBASE-14952
+sh '''#!/bin/bash -e
+  echo "Checking against things we don't expect to include in the 
source tarball (git related, hbase-native-client, etc.)"
+  cat >known_excluded  side."
+echo "In a given set of differences, '.' refers to the branch in 
the repo and 'unpacked_src_tarball' refers to what we pulled out of the 
tarball."
+diff known_excluded diff_output
+  else
+echo "Everything looks as expected."
+  fi
+'''
+  }
+}
   }
   post {
 failure {

[09/17] hbase git commit: HBASE-18426 nightly job should use independent stages to check supported jdks

[busbey]

HBASE-18426 nightly job should use independent stages to check supported jdks

* Jenkinsfile that works for all current branches.
* adds dev-support script for setting parameters for our yetus nightly 
invocation
* quiets the "zip test results" step

Signed-off-by: tedyu 


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/b0a81b21
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/b0a81b21
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/b0a81b21

Branch: refs/heads/branch-1.4
Commit: b0a81b21fcd4feeb25cee2b1472a18548a193f28
Parents: a7cd4a0
Author: Sean Busbey 
Authored: Thu Jul 20 01:01:59 2017 -0500
Committer: Sean Busbey 
Committed: Sun Aug 6 12:16:22 2017 -0500

--
 dev-support/Jenkinsfile| 256 +++-
 dev-support/hbase_nightly_yetus.sh |  86 +++
 2 files changed, 240 insertions(+), 102 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/b0a81b21/dev-support/Jenkinsfile
--
diff --git a/dev-support/Jenkinsfile b/dev-support/Jenkinsfile
index 26f72d7..b5c7660 100644
--- a/dev-support/Jenkinsfile
+++ b/dev-support/Jenkinsfile
@@ -20,10 +20,6 @@ pipeline {
   label 'Hadoop'
 }
   }
-  // work around for YETUS-508, requires maven outside of the dockerfile
-  tools {
-maven 'Maven (latest)'
-  }
   triggers {
 cron('@daily')
   }
@@ -38,8 +34,23 @@ pipeline {
 BASEDIR = "${env.WORKSPACE}/component"
 YETUS_RELEASE = '0.5.0'
 // where we'll write everything from different steps.
-OUTPUT_RELATIVE = 'output'
-OUTPUTDIR = "${env.WORKSPACE}/output"
+OUTPUT_RELATIVE_GENERAL = 'output-general'
+OUTPUTDIR_GENERAL = "${env.WORKSPACE}/output-general"
+OUTPUT_RELATIVE_JDK7 = 'output-jdk7'
+OUTPUTDIR_JDK7 = "${env.WORKSPACE}/output-jdk7"
+OUTPUT_RELATIVE_JDK8 = 'output-jdk8'
+OUTPUTDIR_JDK8 = "${env.WORKSPACE}/output-jdk8"
+PROJECT = 'hbase'
+PROJET_PERSONALITY = 
'https://raw.githubusercontent.com/apache/hbase/master/dev-support/hbase-personality.sh'
+// This section of the docs tells folks not to use the javadoc tag. older 
branches have our old version of the check for said tag.
+AUTHOR_IGNORE_LIST = 
'src/main/asciidoc/_chapters/developer.adoc,dev-support/test-patch.sh'
+WHITESPACE_IGNORE_LIST = '.*/generated/.*'
+// output from surefire; sadly the archive function in yetus only works on 
file names.
+ARCHIVE_PATTERN_LIST = 'TEST-*.xml,org.apache.h*.txt'
+// These tests currently have known failures. Once they burn down to 0, 
remove from here so that new problems will cause a failure.
+TESTS_FILTER = 
'cc,checkstyle,javac,javadoc,pylint,shellcheck,whitespace,perlcritic,ruby-lint,rubocop'
+BRANCH_SPECIFIC_DOCKERFILE = "${env.BASEDIR}/dev-support/docker/Dockerfile"
+EXCLUDE_TESTS_URL = 
'https://builds.apache.org/job/HBase-Find-Flaky-Tests/lastSuccessfulBuild/artifact/excludes/'
   }
   parameters {
 booleanParam(name: 'USE_YETUS_PRERELEASE', defaultValue: false, 
description: '''Check to use the current HEAD of apache/yetus rather than our 
configured release.
@@ -48,24 +59,9 @@ pipeline {
 booleanParam(name: 'DEBUG', defaultValue: false, description: 'Produce a 
lot more meta-information.')
   }
   stages {
-stage ('yetus check') {
-  environment {
-PROJECT = 'hbase'
-PROJECT_PERSONALITY = 
'https://git-wip-us.apache.org/repos/asf?p=hbase.git;a=blob_plain;f=dev-support/hbase-personality.sh;hb=refs/heads/master'
-// This section of the docs tells folks not to use the javadoc tag. 
older branches have our old version of the check for said tag.
-AUTHOR_IGNORE_LIST = 
'src/main/asciidoc/_chapters/developer.adoc,dev-support/test-patch.sh'
-WHITESPACE_IGNORE_LIST = '.*/generated/.*'
-// output from surefire; sadly the archive function in yetus only 
works on file names.
-ARCHIVE_PATTERN_LIST = 
'TEST-*.xml,org.apache.h*-output.txt,org.apache.h*.txt'
-// These tests currently have known failures. Once they burn down to 0, remove 
from here so that new problems will cause a failure.
-TESTS_FILTER = 
'cc,checkstyle,javac,javadoc,pylint,shellcheck,whitespace,perlcritic,ruby-lint,rubocop'
-BRANCH_SPECIFIC_DOCKERFILE = 
"${env.BASEDIR}/dev-support/docker/Dockerfile"
-EXCLUDE_TESTS_URL = 
'https://builds.apache.org/job/HBase-Find-Flaky-Tests/lastSuccessfulBuild/artifact/excludes/'
-  }
+stage ('yetus install') {
   steps {
-// TODO we can move the yetus install into a different stage and then use 
stash to deploy it.
 sh  '''#!/usr/bin/env bash
-printenv
 echo "Ensure we have a copy of Apache Yetus."
 if [[ 

[14/17] hbase git commit: HBASE-18426 nightly job should use independent stages to check supported jdks

[busbey]

HBASE-18426 nightly job should use independent stages to check supported jdks

* Jenkinsfile that works for all current branches.
* adds dev-support script for setting parameters for our yetus nightly 
invocation
* quiets the "zip test results" step

Signed-off-by: tedyu 


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/cb36ecfa
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/cb36ecfa
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/cb36ecfa

Branch: refs/heads/branch-1.2
Commit: cb36ecfa4a590c307fc376db6895bd97c1a807a3
Parents: 1813488
Author: Sean Busbey 
Authored: Thu Jul 20 01:01:59 2017 -0500
Committer: Sean Busbey 
Committed: Sun Aug 6 12:20:48 2017 -0500

--
 dev-support/Jenkinsfile| 256 +++-
 dev-support/hbase_nightly_yetus.sh |  86 +++
 2 files changed, 240 insertions(+), 102 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/cb36ecfa/dev-support/Jenkinsfile
--
diff --git a/dev-support/Jenkinsfile b/dev-support/Jenkinsfile
index 26f72d7..b5c7660 100644
--- a/dev-support/Jenkinsfile
+++ b/dev-support/Jenkinsfile
@@ -20,10 +20,6 @@ pipeline {
   label 'Hadoop'
 }
   }
-  // work around for YETUS-508, requires maven outside of the dockerfile
-  tools {
-maven 'Maven (latest)'
-  }
   triggers {
 cron('@daily')
   }
@@ -38,8 +34,23 @@ pipeline {
 BASEDIR = "${env.WORKSPACE}/component"
 YETUS_RELEASE = '0.5.0'
 // where we'll write everything from different steps.
-OUTPUT_RELATIVE = 'output'
-OUTPUTDIR = "${env.WORKSPACE}/output"
+OUTPUT_RELATIVE_GENERAL = 'output-general'
+OUTPUTDIR_GENERAL = "${env.WORKSPACE}/output-general"
+OUTPUT_RELATIVE_JDK7 = 'output-jdk7'
+OUTPUTDIR_JDK7 = "${env.WORKSPACE}/output-jdk7"
+OUTPUT_RELATIVE_JDK8 = 'output-jdk8'
+OUTPUTDIR_JDK8 = "${env.WORKSPACE}/output-jdk8"
+PROJECT = 'hbase'
+PROJET_PERSONALITY = 
'https://raw.githubusercontent.com/apache/hbase/master/dev-support/hbase-personality.sh'
+// This section of the docs tells folks not to use the javadoc tag. older 
branches have our old version of the check for said tag.
+AUTHOR_IGNORE_LIST = 
'src/main/asciidoc/_chapters/developer.adoc,dev-support/test-patch.sh'
+WHITESPACE_IGNORE_LIST = '.*/generated/.*'
+// output from surefire; sadly the archive function in yetus only works on 
file names.
+ARCHIVE_PATTERN_LIST = 'TEST-*.xml,org.apache.h*.txt'
+// These tests currently have known failures. Once they burn down to 0, 
remove from here so that new problems will cause a failure.
+TESTS_FILTER = 
'cc,checkstyle,javac,javadoc,pylint,shellcheck,whitespace,perlcritic,ruby-lint,rubocop'
+BRANCH_SPECIFIC_DOCKERFILE = "${env.BASEDIR}/dev-support/docker/Dockerfile"
+EXCLUDE_TESTS_URL = 
'https://builds.apache.org/job/HBase-Find-Flaky-Tests/lastSuccessfulBuild/artifact/excludes/'
   }
   parameters {
 booleanParam(name: 'USE_YETUS_PRERELEASE', defaultValue: false, 
description: '''Check to use the current HEAD of apache/yetus rather than our 
configured release.
@@ -48,24 +59,9 @@ pipeline {
 booleanParam(name: 'DEBUG', defaultValue: false, description: 'Produce a 
lot more meta-information.')
   }
   stages {
-stage ('yetus check') {
-  environment {
-PROJECT = 'hbase'
-PROJECT_PERSONALITY = 
'https://git-wip-us.apache.org/repos/asf?p=hbase.git;a=blob_plain;f=dev-support/hbase-personality.sh;hb=refs/heads/master'
-// This section of the docs tells folks not to use the javadoc tag. 
older branches have our old version of the check for said tag.
-AUTHOR_IGNORE_LIST = 
'src/main/asciidoc/_chapters/developer.adoc,dev-support/test-patch.sh'
-WHITESPACE_IGNORE_LIST = '.*/generated/.*'
-// output from surefire; sadly the archive function in yetus only 
works on file names.
-ARCHIVE_PATTERN_LIST = 
'TEST-*.xml,org.apache.h*-output.txt,org.apache.h*.txt'
-// These tests currently have known failures. Once they burn down to 0, remove 
from here so that new problems will cause a failure.
-TESTS_FILTER = 
'cc,checkstyle,javac,javadoc,pylint,shellcheck,whitespace,perlcritic,ruby-lint,rubocop'
-BRANCH_SPECIFIC_DOCKERFILE = 
"${env.BASEDIR}/dev-support/docker/Dockerfile"
-EXCLUDE_TESTS_URL = 
'https://builds.apache.org/job/HBase-Find-Flaky-Tests/lastSuccessfulBuild/artifact/excludes/'
-  }
+stage ('yetus install') {
   steps {
-// TODO we can move the yetus install into a different stage and then use 
stash to deploy it.
 sh  '''#!/usr/bin/env bash
-printenv
 echo "Ensure we have a copy of Apache Yetus."
 if [[ 

[11/17] hbase git commit: HBASE-14220 nightly check that we can build a source tarball.

[busbey]

HBASE-14220 nightly check that we can build a source tarball.

Signed-off-by: Michael Stack 


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/fc09b07c
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/fc09b07c
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/fc09b07c

Branch: refs/heads/branch-1.4
Commit: fc09b07c013a78765ed3a5dfbb59fdc8ce85
Parents: b0a81b2
Author: Sean Busbey 
Authored: Mon Jul 31 01:08:16 2017 -0500
Committer: Sean Busbey 
Committed: Sun Aug 6 12:18:50 2017 -0500

--
 dev-support/Jenkinsfile | 56 
 1 file changed, 56 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/fc09b07c/dev-support/Jenkinsfile
--
diff --git a/dev-support/Jenkinsfile b/dev-support/Jenkinsfile
index b5c7660..cdb62b9 100644
--- a/dev-support/Jenkinsfile
+++ b/dev-support/Jenkinsfile
@@ -241,6 +241,62 @@ curl -L  -o personality.sh "${env.PROJET_PERSONALITY}"
 }
   }
 }
+// This is meant to mimic what a release manager will do to create RCs.
+// See http://hbase.apache.org/book.html#maven.release
+stage ('create source tarball') {
+  tools {
+maven 'Maven (latest)'
+// this needs to be set to the jdk that ought to be used to build 
releases on the branch the Jenkinsfile is stored in.
+jdk "JDK 1.7 (latest)"
+  }
+  steps {
+sh 'mvn --version --offline'
+dir ("component") {
+  sh '''#!/bin/bash -e
+echo "Clean..."
+mvn --batch-mode -DskipTests clean
+echo "Step 3 Build the source tarball"
+mvn -Prelease --batch-mode 
-Dmaven.repo.local="${WORKSPACE}/.m2-for-repo" install -DskipTests 
assembly:single -Dassembly.file=hbase-assembly/src/main/assembly/src.xml
+'''
+}
+dir ("unpacked_src_tarball") {
+  sh '''#!/bin/bash -e
+echo "Unpack the source tarball"
+tar --strip-components=1 -xzf 
../component/hbase-assembly/target/hbase-*-src.tar.gz
+echo "Building from source artifact."
+mvn -DskipTests -Prelease --batch-mode 
-Dmaven.repo.local="${WORKSPACE}/.m2-for-src" clean install assembly:single
+echo "Clean up after checking ability to build."
+mvn -DskipTests --batch-mode clean
+'''
+}
+dir ("component") {
+  sh '''#!/bin/bash -e
+echo "Clean up the source checkout"
+mvn -DskipTests --batch-mode clean
+echo "Diff against source tree"
+diff --binary --recursive . ../unpacked_src_tarball 
>../diff_output || true
+'''
+}
+// expectation check largely based on HBASE-14952
+sh '''#!/bin/bash -e
+  echo "Checking against things we don't expect to include in the 
source tarball (git related, hbase-native-client, etc.)"
+  cat >known_excluded  side."
+echo "In a given set of differences, '.' refers to the branch in 
the repo and 'unpacked_src_tarball' refers to what we pulled out of the 
tarball."
+diff known_excluded diff_output
+  else
+echo "Everything looks as expected."
+  fi
+'''
+  }
+}
   }
   post {
 failure {

[10/17] hbase git commit: HBASE-14220 nightly check that we can build a source tarball.

[busbey]

HBASE-14220 nightly check that we can build a source tarball.

Signed-off-by: Michael Stack 


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/695b3031
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/695b3031
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/695b3031

Branch: refs/heads/branch-1
Commit: 695b30318edcbc775adfaa94ddafc93168b69623
Parents: 8c720ac
Author: Sean Busbey 
Authored: Mon Jul 31 01:08:16 2017 -0500
Committer: Sean Busbey 
Committed: Sun Aug 6 12:17:58 2017 -0500

--
 dev-support/Jenkinsfile | 56 
 1 file changed, 56 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/695b3031/dev-support/Jenkinsfile
--
diff --git a/dev-support/Jenkinsfile b/dev-support/Jenkinsfile
index b5c7660..cdb62b9 100644
--- a/dev-support/Jenkinsfile
+++ b/dev-support/Jenkinsfile
@@ -241,6 +241,62 @@ curl -L  -o personality.sh "${env.PROJET_PERSONALITY}"
 }
   }
 }
+// This is meant to mimic what a release manager will do to create RCs.
+// See http://hbase.apache.org/book.html#maven.release
+stage ('create source tarball') {
+  tools {
+maven 'Maven (latest)'
+// this needs to be set to the jdk that ought to be used to build 
releases on the branch the Jenkinsfile is stored in.
+jdk "JDK 1.7 (latest)"
+  }
+  steps {
+sh 'mvn --version --offline'
+dir ("component") {
+  sh '''#!/bin/bash -e
+echo "Clean..."
+mvn --batch-mode -DskipTests clean
+echo "Step 3 Build the source tarball"
+mvn -Prelease --batch-mode 
-Dmaven.repo.local="${WORKSPACE}/.m2-for-repo" install -DskipTests 
assembly:single -Dassembly.file=hbase-assembly/src/main/assembly/src.xml
+'''
+}
+dir ("unpacked_src_tarball") {
+  sh '''#!/bin/bash -e
+echo "Unpack the source tarball"
+tar --strip-components=1 -xzf 
../component/hbase-assembly/target/hbase-*-src.tar.gz
+echo "Building from source artifact."
+mvn -DskipTests -Prelease --batch-mode 
-Dmaven.repo.local="${WORKSPACE}/.m2-for-src" clean install assembly:single
+echo "Clean up after checking ability to build."
+mvn -DskipTests --batch-mode clean
+'''
+}
+dir ("component") {
+  sh '''#!/bin/bash -e
+echo "Clean up the source checkout"
+mvn -DskipTests --batch-mode clean
+echo "Diff against source tree"
+diff --binary --recursive . ../unpacked_src_tarball 
>../diff_output || true
+'''
+}
+// expectation check largely based on HBASE-14952
+sh '''#!/bin/bash -e
+  echo "Checking against things we don't expect to include in the 
source tarball (git related, hbase-native-client, etc.)"
+  cat >known_excluded  side."
+echo "In a given set of differences, '.' refers to the branch in 
the repo and 'unpacked_src_tarball' refers to what we pulled out of the 
tarball."
+diff known_excluded diff_output
+  else
+echo "Everything looks as expected."
+  fi
+'''
+  }
+}
   }
   post {
 failure {

[05/17] hbase git commit: HBASE-14220 nightly check that we can build a source tarball.

[busbey]

HBASE-14220 nightly check that we can build a source tarball.

Signed-off-by: Michael Stack 


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/69fddb58
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/69fddb58
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/69fddb58

Branch: refs/heads/branch-2
Commit: 69fddb58bda99b7f4ebcdddaacadd1ffaa93dc24
Parents: db18a00
Author: Sean Busbey 
Authored: Mon Jul 31 01:08:16 2017 -0500
Committer: Sean Busbey 
Committed: Sun Aug 6 12:08:51 2017 -0500

--
 dev-support/Jenkinsfile | 56 
 1 file changed, 56 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/69fddb58/dev-support/Jenkinsfile
--
diff --git a/dev-support/Jenkinsfile b/dev-support/Jenkinsfile
index b5c7660..94d8c3f 100644
--- a/dev-support/Jenkinsfile
+++ b/dev-support/Jenkinsfile
@@ -241,6 +241,62 @@ curl -L  -o personality.sh "${env.PROJET_PERSONALITY}"
 }
   }
 }
+// This is meant to mimic what a release manager will do to create RCs.
+// See http://hbase.apache.org/book.html#maven.release
+stage ('create source tarball') {
+  tools {
+maven 'Maven (latest)'
+// this needs to be set to the jdk that ought to be used to build 
releases on the branch the Jenkinsfile is stored in.
+jdk "JDK 1.8 (latest)"
+  }
+  steps {
+sh 'mvn --version --offline'
+dir ("component") {
+  sh '''#!/bin/bash -e
+echo "Clean..."
+mvn --batch-mode -DskipTests clean
+echo "Step 3 Build the source tarball"
+mvn -Prelease --batch-mode 
-Dmaven.repo.local="${WORKSPACE}/.m2-for-repo" install -DskipTests 
assembly:single -Dassembly.file=hbase-assembly/src/main/assembly/src.xml
+'''
+}
+dir ("unpacked_src_tarball") {
+  sh '''#!/bin/bash -e
+echo "Unpack the source tarball"
+tar --strip-components=1 -xzf 
../component/hbase-assembly/target/hbase-*-src.tar.gz
+echo "Building from source artifact."
+mvn -DskipTests -Prelease --batch-mode 
-Dmaven.repo.local="${WORKSPACE}/.m2-for-src" clean install assembly:single
+echo "Clean up after checking ability to build."
+mvn -DskipTests --batch-mode clean
+'''
+}
+dir ("component") {
+  sh '''#!/bin/bash -e
+echo "Clean up the source checkout"
+mvn -DskipTests --batch-mode clean
+echo "Diff against source tree"
+diff --binary --recursive . ../unpacked_src_tarball 
>../diff_output || true
+'''
+}
+// expectation check largely based on HBASE-14952
+sh '''#!/bin/bash -e
+  echo "Checking against things we don't expect to include in the 
source tarball (git related, hbase-native-client, etc.)"
+  cat >known_excluded  side."
+echo "In a given set of differences, '.' refers to the branch in 
the repo and 'unpacked_src_tarball' refers to what we pulled out of the 
tarball."
+diff known_excluded diff_output
+  else
+echo "Everything looks as expected."
+  fi
+'''
+  }
+}
   }
   post {
 failure {

[08/17] hbase git commit: HBASE-18426 nightly job should use independent stages to check supported jdks

[busbey]

HBASE-18426 nightly job should use independent stages to check supported jdks

* Jenkinsfile that works for all current branches.
* adds dev-support script for setting parameters for our yetus nightly 
invocation
* quiets the "zip test results" step

Signed-off-by: tedyu 


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/8c720ac6
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/8c720ac6
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/8c720ac6

Branch: refs/heads/branch-1
Commit: 8c720ac68828edf0c243cc7827ccfa7958f60fa5
Parents: cd7d5bd
Author: Sean Busbey 
Authored: Thu Jul 20 01:01:59 2017 -0500
Committer: Sean Busbey 
Committed: Sun Aug 6 12:12:58 2017 -0500

--
 dev-support/Jenkinsfile| 256 +++-
 dev-support/hbase_nightly_yetus.sh |  86 +++
 2 files changed, 240 insertions(+), 102 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/8c720ac6/dev-support/Jenkinsfile
--
diff --git a/dev-support/Jenkinsfile b/dev-support/Jenkinsfile
index 26f72d7..b5c7660 100644
--- a/dev-support/Jenkinsfile
+++ b/dev-support/Jenkinsfile
@@ -20,10 +20,6 @@ pipeline {
   label 'Hadoop'
 }
   }
-  // work around for YETUS-508, requires maven outside of the dockerfile
-  tools {
-maven 'Maven (latest)'
-  }
   triggers {
 cron('@daily')
   }
@@ -38,8 +34,23 @@ pipeline {
 BASEDIR = "${env.WORKSPACE}/component"
 YETUS_RELEASE = '0.5.0'
 // where we'll write everything from different steps.
-OUTPUT_RELATIVE = 'output'
-OUTPUTDIR = "${env.WORKSPACE}/output"
+OUTPUT_RELATIVE_GENERAL = 'output-general'
+OUTPUTDIR_GENERAL = "${env.WORKSPACE}/output-general"
+OUTPUT_RELATIVE_JDK7 = 'output-jdk7'
+OUTPUTDIR_JDK7 = "${env.WORKSPACE}/output-jdk7"
+OUTPUT_RELATIVE_JDK8 = 'output-jdk8'
+OUTPUTDIR_JDK8 = "${env.WORKSPACE}/output-jdk8"
+PROJECT = 'hbase'
+PROJET_PERSONALITY = 
'https://raw.githubusercontent.com/apache/hbase/master/dev-support/hbase-personality.sh'
+// This section of the docs tells folks not to use the javadoc tag. older 
branches have our old version of the check for said tag.
+AUTHOR_IGNORE_LIST = 
'src/main/asciidoc/_chapters/developer.adoc,dev-support/test-patch.sh'
+WHITESPACE_IGNORE_LIST = '.*/generated/.*'
+// output from surefire; sadly the archive function in yetus only works on 
file names.
+ARCHIVE_PATTERN_LIST = 'TEST-*.xml,org.apache.h*.txt'
+// These tests currently have known failures. Once they burn down to 0, 
remove from here so that new problems will cause a failure.
+TESTS_FILTER = 
'cc,checkstyle,javac,javadoc,pylint,shellcheck,whitespace,perlcritic,ruby-lint,rubocop'
+BRANCH_SPECIFIC_DOCKERFILE = "${env.BASEDIR}/dev-support/docker/Dockerfile"
+EXCLUDE_TESTS_URL = 
'https://builds.apache.org/job/HBase-Find-Flaky-Tests/lastSuccessfulBuild/artifact/excludes/'
   }
   parameters {
 booleanParam(name: 'USE_YETUS_PRERELEASE', defaultValue: false, 
description: '''Check to use the current HEAD of apache/yetus rather than our 
configured release.
@@ -48,24 +59,9 @@ pipeline {
 booleanParam(name: 'DEBUG', defaultValue: false, description: 'Produce a 
lot more meta-information.')
   }
   stages {
-stage ('yetus check') {
-  environment {
-PROJECT = 'hbase'
-PROJECT_PERSONALITY = 
'https://git-wip-us.apache.org/repos/asf?p=hbase.git;a=blob_plain;f=dev-support/hbase-personality.sh;hb=refs/heads/master'
-// This section of the docs tells folks not to use the javadoc tag. 
older branches have our old version of the check for said tag.
-AUTHOR_IGNORE_LIST = 
'src/main/asciidoc/_chapters/developer.adoc,dev-support/test-patch.sh'
-WHITESPACE_IGNORE_LIST = '.*/generated/.*'
-// output from surefire; sadly the archive function in yetus only 
works on file names.
-ARCHIVE_PATTERN_LIST = 
'TEST-*.xml,org.apache.h*-output.txt,org.apache.h*.txt'
-// These tests currently have known failures. Once they burn down to 0, remove 
from here so that new problems will cause a failure.
-TESTS_FILTER = 
'cc,checkstyle,javac,javadoc,pylint,shellcheck,whitespace,perlcritic,ruby-lint,rubocop'
-BRANCH_SPECIFIC_DOCKERFILE = 
"${env.BASEDIR}/dev-support/docker/Dockerfile"
-EXCLUDE_TESTS_URL = 
'https://builds.apache.org/job/HBase-Find-Flaky-Tests/lastSuccessfulBuild/artifact/excludes/'
-  }
+stage ('yetus install') {
   steps {
-// TODO we can move the yetus install into a different stage and then use 
stash to deploy it.
 sh  '''#!/usr/bin/env bash
-printenv
 echo "Ensure we have a copy of Apache Yetus."
 if [[ 

[06/17] hbase git commit: Revert "HBaseHBASE-18447 MetricRegistryInfo#hashCode uses hashCode instead of toHashCode"

[busbey]

Revert "HBaseHBASE-18447 MetricRegistryInfo#hashCode uses hashCode instead of 
toHashCode"

This reverts commit 1cb7d0e82ad64f37fbd6de950b74081b0d5eddf3.

bad commit message.


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/bcdc56ac
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/bcdc56ac
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/bcdc56ac

Branch: refs/heads/branch-1
Commit: bcdc56ac76e4a26e53faa8301a441e94ee8614d7
Parents: cad3d55
Author: Sean Busbey 
Authored: Sun Aug 6 12:12:01 2017 -0500
Committer: Sean Busbey 
Committed: Sun Aug 6 12:12:01 2017 -0500

--
 .../java/org/apache/hadoop/hbase/metrics/MetricRegistryInfo.java   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/bcdc56ac/hbase-metrics-api/src/main/java/org/apache/hadoop/hbase/metrics/MetricRegistryInfo.java
--
diff --git 
a/hbase-metrics-api/src/main/java/org/apache/hadoop/hbase/metrics/MetricRegistryInfo.java
 
b/hbase-metrics-api/src/main/java/org/apache/hadoop/hbase/metrics/MetricRegistryInfo.java
index 88ba232..58fcba7 100644
--- 
a/hbase-metrics-api/src/main/java/org/apache/hadoop/hbase/metrics/MetricRegistryInfo.java
+++ 
b/hbase-metrics-api/src/main/java/org/apache/hadoop/hbase/metrics/MetricRegistryInfo.java
@@ -107,6 +107,6 @@ public class MetricRegistryInfo {
 .append(metricsDescription)
 .append(metricsContext)
 .append(metricsJmxContext)
-.toHashCode();
+.hashCode();
   }
 }

[01/17] hbase git commit: HBASE-18426 nightly job should use independent stages to check supported jdks

[busbey]

Repository: hbase
Updated Branches:
  refs/heads/HBASE-14220 [deleted] 2ad70fcfe
  refs/heads/HBASE-18426 [deleted] 0bacb03ab
  refs/heads/branch-1 cad3d5589 -> 695b30318
  refs/heads/branch-1.1 7fcfd1c96 -> 68fb6b960
  refs/heads/branch-1.1-HBASE-18426 [deleted] adb2752e8
  refs/heads/branch-1.2 18134884f -> cd683e612
  refs/heads/branch-1.2-HBASE-18426 [deleted] aaee9
  refs/heads/branch-1.3 207db3d7a -> f30b624e1
  refs/heads/branch-1.4 a7cd4a053 -> fc09b07c0
  refs/heads/branch-2 0f20ec395 -> 69fddb58b
  refs/heads/master 637f7abf0 -> 2a717459b


HBASE-18426 nightly job should use independent stages to check supported jdks

* Jenkinsfile that works for all current branches.
* adds dev-support script for setting parameters for our yetus nightly 
invocation
* quiets the "zip test results" step

Signed-off-by: tedyu 


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/04424ad6
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/04424ad6
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/04424ad6

Branch: refs/heads/master
Commit: 04424ad65feae880d10fd3896e98a7201f76200b
Parents: 637f7ab
Author: Sean Busbey 
Authored: Thu Jul 20 01:01:59 2017 -0500
Committer: Sean Busbey 
Committed: Sun Aug 6 11:59:45 2017 -0500

--
 dev-support/Jenkinsfile| 256 +++-
 dev-support/hbase_nightly_yetus.sh |  86 +++
 2 files changed, 240 insertions(+), 102 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/04424ad6/dev-support/Jenkinsfile
--
diff --git a/dev-support/Jenkinsfile b/dev-support/Jenkinsfile
index 26f72d7..b5c7660 100644
--- a/dev-support/Jenkinsfile
+++ b/dev-support/Jenkinsfile
@@ -20,10 +20,6 @@ pipeline {
   label 'Hadoop'
 }
   }
-  // work around for YETUS-508, requires maven outside of the dockerfile
-  tools {
-maven 'Maven (latest)'
-  }
   triggers {
 cron('@daily')
   }
@@ -38,8 +34,23 @@ pipeline {
 BASEDIR = "${env.WORKSPACE}/component"
 YETUS_RELEASE = '0.5.0'
 // where we'll write everything from different steps.
-OUTPUT_RELATIVE = 'output'
-OUTPUTDIR = "${env.WORKSPACE}/output"
+OUTPUT_RELATIVE_GENERAL = 'output-general'
+OUTPUTDIR_GENERAL = "${env.WORKSPACE}/output-general"
+OUTPUT_RELATIVE_JDK7 = 'output-jdk7'
+OUTPUTDIR_JDK7 = "${env.WORKSPACE}/output-jdk7"
+OUTPUT_RELATIVE_JDK8 = 'output-jdk8'
+OUTPUTDIR_JDK8 = "${env.WORKSPACE}/output-jdk8"
+PROJECT = 'hbase'
+PROJET_PERSONALITY = 
'https://raw.githubusercontent.com/apache/hbase/master/dev-support/hbase-personality.sh'
+// This section of the docs tells folks not to use the javadoc tag. older 
branches have our old version of the check for said tag.
+AUTHOR_IGNORE_LIST = 
'src/main/asciidoc/_chapters/developer.adoc,dev-support/test-patch.sh'
+WHITESPACE_IGNORE_LIST = '.*/generated/.*'
+// output from surefire; sadly the archive function in yetus only works on 
file names.
+ARCHIVE_PATTERN_LIST = 'TEST-*.xml,org.apache.h*.txt'
+// These tests currently have known failures. Once they burn down to 0, 
remove from here so that new problems will cause a failure.
+TESTS_FILTER = 
'cc,checkstyle,javac,javadoc,pylint,shellcheck,whitespace,perlcritic,ruby-lint,rubocop'
+BRANCH_SPECIFIC_DOCKERFILE = "${env.BASEDIR}/dev-support/docker/Dockerfile"
+EXCLUDE_TESTS_URL = 
'https://builds.apache.org/job/HBase-Find-Flaky-Tests/lastSuccessfulBuild/artifact/excludes/'
   }
   parameters {
 booleanParam(name: 'USE_YETUS_PRERELEASE', defaultValue: false, 
description: '''Check to use the current HEAD of apache/yetus rather than our 
configured release.
@@ -48,24 +59,9 @@ pipeline {
 booleanParam(name: 'DEBUG', defaultValue: false, description: 'Produce a 
lot more meta-information.')
   }
   stages {
-stage ('yetus check') {
-  environment {
-PROJECT = 'hbase'
-PROJECT_PERSONALITY = 
'https://git-wip-us.apache.org/repos/asf?p=hbase.git;a=blob_plain;f=dev-support/hbase-personality.sh;hb=refs/heads/master'
-// This section of the docs tells folks not to use the javadoc tag. 
older branches have our old version of the check for said tag.
-AUTHOR_IGNORE_LIST = 
'src/main/asciidoc/_chapters/developer.adoc,dev-support/test-patch.sh'
-WHITESPACE_IGNORE_LIST = '.*/generated/.*'
-// output from surefire; sadly the archive function in yetus only 
works on file names.
-ARCHIVE_PATTERN_LIST = 
'TEST-*.xml,org.apache.h*-output.txt,org.apache.h*.txt'
-// These tests currently have known failures. Once they burn down to 0, remove 
from here so that new problems will cause a failure.
-TESTS_FILTER 

[07/17] hbase git commit: HBASE-18447 MetricRegistryInfo#hashCode uses hashCode instead of toHashCode

[busbey]

HBASE-18447 MetricRegistryInfo#hashCode uses hashCode instead of toHashCode

Signed-off-by: Chia-Ping Tsai 


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/cd7d5bd7
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/cd7d5bd7
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/cd7d5bd7

Branch: refs/heads/branch-1
Commit: cd7d5bd752e6c301e3e410e3bc136f994b7be88f
Parents: bcdc56a
Author: Peter Somogyi 
Authored: Tue Jul 25 13:15:48 2017 +0200
Committer: Sean Busbey 
Committed: Sun Aug 6 12:12:23 2017 -0500

--
 .../java/org/apache/hadoop/hbase/metrics/MetricRegistryInfo.java   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/cd7d5bd7/hbase-metrics-api/src/main/java/org/apache/hadoop/hbase/metrics/MetricRegistryInfo.java
--
diff --git 
a/hbase-metrics-api/src/main/java/org/apache/hadoop/hbase/metrics/MetricRegistryInfo.java
 
b/hbase-metrics-api/src/main/java/org/apache/hadoop/hbase/metrics/MetricRegistryInfo.java
index 58fcba7..88ba232 100644
--- 
a/hbase-metrics-api/src/main/java/org/apache/hadoop/hbase/metrics/MetricRegistryInfo.java
+++ 
b/hbase-metrics-api/src/main/java/org/apache/hadoop/hbase/metrics/MetricRegistryInfo.java
@@ -107,6 +107,6 @@ public class MetricRegistryInfo {
 .append(metricsDescription)
 .append(metricsContext)
 .append(metricsJmxContext)
-.hashCode();
+.toHashCode();
   }
 }

[16/17] hbase git commit: HBASE-18426 nightly job should use independent stages to check supported jdks

[busbey]

HBASE-18426 nightly job should use independent stages to check supported jdks

* Jenkinsfile that works for all current branches.
* adds dev-support script for setting parameters for our yetus nightly 
invocation
* quiets the "zip test results" step

Signed-off-by: tedyu 


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/de2951d1
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/de2951d1
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/de2951d1

Branch: refs/heads/branch-1.1
Commit: de2951d11b3ea17f29aca7523a82cb82146e93ee
Parents: 7fcfd1c
Author: Sean Busbey 
Authored: Thu Jul 20 01:01:59 2017 -0500
Committer: Sean Busbey 
Committed: Sun Aug 6 12:22:59 2017 -0500

--
 dev-support/Jenkinsfile| 256 +++-
 dev-support/hbase_nightly_yetus.sh |  86 +++
 2 files changed, 240 insertions(+), 102 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/de2951d1/dev-support/Jenkinsfile
--
diff --git a/dev-support/Jenkinsfile b/dev-support/Jenkinsfile
index 26f72d7..b5c7660 100644
--- a/dev-support/Jenkinsfile
+++ b/dev-support/Jenkinsfile
@@ -20,10 +20,6 @@ pipeline {
   label 'Hadoop'
 }
   }
-  // work around for YETUS-508, requires maven outside of the dockerfile
-  tools {
-maven 'Maven (latest)'
-  }
   triggers {
 cron('@daily')
   }
@@ -38,8 +34,23 @@ pipeline {
 BASEDIR = "${env.WORKSPACE}/component"
 YETUS_RELEASE = '0.5.0'
 // where we'll write everything from different steps.
-OUTPUT_RELATIVE = 'output'
-OUTPUTDIR = "${env.WORKSPACE}/output"
+OUTPUT_RELATIVE_GENERAL = 'output-general'
+OUTPUTDIR_GENERAL = "${env.WORKSPACE}/output-general"
+OUTPUT_RELATIVE_JDK7 = 'output-jdk7'
+OUTPUTDIR_JDK7 = "${env.WORKSPACE}/output-jdk7"
+OUTPUT_RELATIVE_JDK8 = 'output-jdk8'
+OUTPUTDIR_JDK8 = "${env.WORKSPACE}/output-jdk8"
+PROJECT = 'hbase'
+PROJET_PERSONALITY = 
'https://raw.githubusercontent.com/apache/hbase/master/dev-support/hbase-personality.sh'
+// This section of the docs tells folks not to use the javadoc tag. older 
branches have our old version of the check for said tag.
+AUTHOR_IGNORE_LIST = 
'src/main/asciidoc/_chapters/developer.adoc,dev-support/test-patch.sh'
+WHITESPACE_IGNORE_LIST = '.*/generated/.*'
+// output from surefire; sadly the archive function in yetus only works on 
file names.
+ARCHIVE_PATTERN_LIST = 'TEST-*.xml,org.apache.h*.txt'
+// These tests currently have known failures. Once they burn down to 0, 
remove from here so that new problems will cause a failure.
+TESTS_FILTER = 
'cc,checkstyle,javac,javadoc,pylint,shellcheck,whitespace,perlcritic,ruby-lint,rubocop'
+BRANCH_SPECIFIC_DOCKERFILE = "${env.BASEDIR}/dev-support/docker/Dockerfile"
+EXCLUDE_TESTS_URL = 
'https://builds.apache.org/job/HBase-Find-Flaky-Tests/lastSuccessfulBuild/artifact/excludes/'
   }
   parameters {
 booleanParam(name: 'USE_YETUS_PRERELEASE', defaultValue: false, 
description: '''Check to use the current HEAD of apache/yetus rather than our 
configured release.
@@ -48,24 +59,9 @@ pipeline {
 booleanParam(name: 'DEBUG', defaultValue: false, description: 'Produce a 
lot more meta-information.')
   }
   stages {
-stage ('yetus check') {
-  environment {
-PROJECT = 'hbase'
-PROJECT_PERSONALITY = 
'https://git-wip-us.apache.org/repos/asf?p=hbase.git;a=blob_plain;f=dev-support/hbase-personality.sh;hb=refs/heads/master'
-// This section of the docs tells folks not to use the javadoc tag. 
older branches have our old version of the check for said tag.
-AUTHOR_IGNORE_LIST = 
'src/main/asciidoc/_chapters/developer.adoc,dev-support/test-patch.sh'
-WHITESPACE_IGNORE_LIST = '.*/generated/.*'
-// output from surefire; sadly the archive function in yetus only 
works on file names.
-ARCHIVE_PATTERN_LIST = 
'TEST-*.xml,org.apache.h*-output.txt,org.apache.h*.txt'
-// These tests currently have known failures. Once they burn down to 0, remove 
from here so that new problems will cause a failure.
-TESTS_FILTER = 
'cc,checkstyle,javac,javadoc,pylint,shellcheck,whitespace,perlcritic,ruby-lint,rubocop'
-BRANCH_SPECIFIC_DOCKERFILE = 
"${env.BASEDIR}/dev-support/docker/Dockerfile"
-EXCLUDE_TESTS_URL = 
'https://builds.apache.org/job/HBase-Find-Flaky-Tests/lastSuccessfulBuild/artifact/excludes/'
-  }
+stage ('yetus install') {
   steps {
-// TODO we can move the yetus install into a different stage and then use 
stash to deploy it.
 sh  '''#!/usr/bin/env bash
-printenv
 echo "Ensure we have a copy of Apache Yetus."
 if [[ 

[12/17] hbase git commit: HBASE-18426 nightly job should use independent stages to check supported jdks

[busbey]

HBASE-18426 nightly job should use independent stages to check supported jdks

* Jenkinsfile that works for all current branches.
* adds dev-support script for setting parameters for our yetus nightly 
invocation
* quiets the "zip test results" step

Signed-off-by: tedyu 


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/879f7a51
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/879f7a51
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/879f7a51

Branch: refs/heads/branch-1.3
Commit: 879f7a517c2ddd794e26b06382697bbf3409cba8
Parents: 207db3d
Author: Sean Busbey 
Authored: Thu Jul 20 01:01:59 2017 -0500
Committer: Sean Busbey 
Committed: Sun Aug 6 12:19:45 2017 -0500

--
 dev-support/Jenkinsfile| 256 +++-
 dev-support/hbase_nightly_yetus.sh |  86 +++
 2 files changed, 240 insertions(+), 102 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/879f7a51/dev-support/Jenkinsfile
--
diff --git a/dev-support/Jenkinsfile b/dev-support/Jenkinsfile
index 26f72d7..b5c7660 100644
--- a/dev-support/Jenkinsfile
+++ b/dev-support/Jenkinsfile
@@ -20,10 +20,6 @@ pipeline {
   label 'Hadoop'
 }
   }
-  // work around for YETUS-508, requires maven outside of the dockerfile
-  tools {
-maven 'Maven (latest)'
-  }
   triggers {
 cron('@daily')
   }
@@ -38,8 +34,23 @@ pipeline {
 BASEDIR = "${env.WORKSPACE}/component"
 YETUS_RELEASE = '0.5.0'
 // where we'll write everything from different steps.
-OUTPUT_RELATIVE = 'output'
-OUTPUTDIR = "${env.WORKSPACE}/output"
+OUTPUT_RELATIVE_GENERAL = 'output-general'
+OUTPUTDIR_GENERAL = "${env.WORKSPACE}/output-general"
+OUTPUT_RELATIVE_JDK7 = 'output-jdk7'
+OUTPUTDIR_JDK7 = "${env.WORKSPACE}/output-jdk7"
+OUTPUT_RELATIVE_JDK8 = 'output-jdk8'
+OUTPUTDIR_JDK8 = "${env.WORKSPACE}/output-jdk8"
+PROJECT = 'hbase'
+PROJET_PERSONALITY = 
'https://raw.githubusercontent.com/apache/hbase/master/dev-support/hbase-personality.sh'
+// This section of the docs tells folks not to use the javadoc tag. older 
branches have our old version of the check for said tag.
+AUTHOR_IGNORE_LIST = 
'src/main/asciidoc/_chapters/developer.adoc,dev-support/test-patch.sh'
+WHITESPACE_IGNORE_LIST = '.*/generated/.*'
+// output from surefire; sadly the archive function in yetus only works on 
file names.
+ARCHIVE_PATTERN_LIST = 'TEST-*.xml,org.apache.h*.txt'
+// These tests currently have known failures. Once they burn down to 0, 
remove from here so that new problems will cause a failure.
+TESTS_FILTER = 
'cc,checkstyle,javac,javadoc,pylint,shellcheck,whitespace,perlcritic,ruby-lint,rubocop'
+BRANCH_SPECIFIC_DOCKERFILE = "${env.BASEDIR}/dev-support/docker/Dockerfile"
+EXCLUDE_TESTS_URL = 
'https://builds.apache.org/job/HBase-Find-Flaky-Tests/lastSuccessfulBuild/artifact/excludes/'
   }
   parameters {
 booleanParam(name: 'USE_YETUS_PRERELEASE', defaultValue: false, 
description: '''Check to use the current HEAD of apache/yetus rather than our 
configured release.
@@ -48,24 +59,9 @@ pipeline {
 booleanParam(name: 'DEBUG', defaultValue: false, description: 'Produce a 
lot more meta-information.')
   }
   stages {
-stage ('yetus check') {
-  environment {
-PROJECT = 'hbase'
-PROJECT_PERSONALITY = 
'https://git-wip-us.apache.org/repos/asf?p=hbase.git;a=blob_plain;f=dev-support/hbase-personality.sh;hb=refs/heads/master'
-// This section of the docs tells folks not to use the javadoc tag. 
older branches have our old version of the check for said tag.
-AUTHOR_IGNORE_LIST = 
'src/main/asciidoc/_chapters/developer.adoc,dev-support/test-patch.sh'
-WHITESPACE_IGNORE_LIST = '.*/generated/.*'
-// output from surefire; sadly the archive function in yetus only 
works on file names.
-ARCHIVE_PATTERN_LIST = 
'TEST-*.xml,org.apache.h*-output.txt,org.apache.h*.txt'
-// These tests currently have known failures. Once they burn down to 0, remove 
from here so that new problems will cause a failure.
-TESTS_FILTER = 
'cc,checkstyle,javac,javadoc,pylint,shellcheck,whitespace,perlcritic,ruby-lint,rubocop'
-BRANCH_SPECIFIC_DOCKERFILE = 
"${env.BASEDIR}/dev-support/docker/Dockerfile"
-EXCLUDE_TESTS_URL = 
'https://builds.apache.org/job/HBase-Find-Flaky-Tests/lastSuccessfulBuild/artifact/excludes/'
-  }
+stage ('yetus install') {
   steps {
-// TODO we can move the yetus install into a different stage and then use 
stash to deploy it.
 sh  '''#!/usr/bin/env bash
-printenv
 echo "Ensure we have a copy of Apache Yetus."
 if [[ 

[03/17] hbase git commit: HBASE-18147 POC jenkinsfile for nightly checks.

[busbey]

HBASE-18147 POC jenkinsfile for nightly checks.

* adds ruby tools to dockerfile
* adds rubocop to dockerfile
* adds ruby-lint to dockerfile
* adds perlcritic to dockerfile

Signed-off-by: Alex Leblang 
Signed-off-by: Josh Elser 


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/3c8b3643
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/3c8b3643
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/3c8b3643

Branch: refs/heads/branch-2
Commit: 3c8b3643797a469fc8308861b296e1aa96a64945
Parents: 0f20ec3
Author: Sean Busbey 
Authored: Thu Jul 20 17:07:33 2017 -0500
Committer: Sean Busbey 
Committed: Sun Aug 6 12:08:34 2017 -0500

--
 dev-support/Jenkinsfile   | 198 +
 dev-support/docker/Dockerfile |  29 ++
 2 files changed, 227 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/3c8b3643/dev-support/Jenkinsfile
--
diff --git a/dev-support/Jenkinsfile b/dev-support/Jenkinsfile
new file mode 100644
index 000..26f72d7
--- /dev/null
+++ b/dev-support/Jenkinsfile
@@ -0,0 +1,198 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+pipeline {
+  agent {
+node {
+  label 'Hadoop'
+}
+  }
+  // work around for YETUS-508, requires maven outside of the dockerfile
+  tools {
+maven 'Maven (latest)'
+  }
+  triggers {
+cron('@daily')
+  }
+  options {
+buildDiscarder(logRotator(numToKeepStr: '30'))
+timeout (time: 6, unit: 'HOURS')
+timestamps()
+  }
+  environment {
+TOOLS = "${env.WORKSPACE}/tools"
+// where we check out to across stages
+BASEDIR = "${env.WORKSPACE}/component"
+YETUS_RELEASE = '0.5.0'
+// where we'll write everything from different steps.
+OUTPUT_RELATIVE = 'output'
+OUTPUTDIR = "${env.WORKSPACE}/output"
+  }
+  parameters {
+booleanParam(name: 'USE_YETUS_PRERELEASE', defaultValue: false, 
description: '''Check to use the current HEAD of apache/yetus rather than our 
configured release.
+
+Should only be used manually when e.g. there is some non-work-aroundable 
issue in yetus we are checking a fix for.''')
+booleanParam(name: 'DEBUG', defaultValue: false, description: 'Produce a 
lot more meta-information.')
+  }
+  stages {
+stage ('yetus check') {
+  environment {
+PROJECT = 'hbase'
+PROJECT_PERSONALITY = 
'https://git-wip-us.apache.org/repos/asf?p=hbase.git;a=blob_plain;f=dev-support/hbase-personality.sh;hb=refs/heads/master'
+// This section of the docs tells folks not to use the javadoc tag. 
older branches have our old version of the check for said tag.
+AUTHOR_IGNORE_LIST = 
'src/main/asciidoc/_chapters/developer.adoc,dev-support/test-patch.sh'
+WHITESPACE_IGNORE_LIST = '.*/generated/.*'
+// output from surefire; sadly the archive function in yetus only 
works on file names.
+ARCHIVE_PATTERN_LIST = 
'TEST-*.xml,org.apache.h*-output.txt,org.apache.h*.txt'
+// These tests currently have known failures. Once they burn down to 0, remove 
from here so that new problems will cause a failure.
+TESTS_FILTER = 
'cc,checkstyle,javac,javadoc,pylint,shellcheck,whitespace,perlcritic,ruby-lint,rubocop'
+BRANCH_SPECIFIC_DOCKERFILE = 
"${env.BASEDIR}/dev-support/docker/Dockerfile"
+EXCLUDE_TESTS_URL = 
'https://builds.apache.org/job/HBase-Find-Flaky-Tests/lastSuccessfulBuild/artifact/excludes/'
+  }
+  steps {
+// TODO we can move the yetus install into a different stage and then use 
stash to deploy it.
+sh  '''#!/usr/bin/env bash
+printenv
+echo "Ensure we have a copy of Apache Yetus."
+if [[ true !=  "${USE_YETUS_PRERELEASE}" ]]; then
+  YETUS_DIR="${WORKSPACE}/yetus-${YETUS_RELEASE}"
+  echo "Checking for Yetus ${YETUS_RELEASE} in '${YETUS_DIR}'"
+  if [ ! -d "${YETUS_DIR}" ]; then
+echo "New download of Apache Yetus version ${YETUS_RELEASE}."
+rm -rf 

hbase git commit: HBASE-18426 ADDENDUM leave the workspace around on pipeline failure so we can browse it. [Forced Update!]

[busbey]

Repository: hbase
Updated Branches:
  refs/heads/HBASE-18426 076a09347 -> 0bea97e1e (forced update)


HBASE-18426 ADDENDUM leave the workspace around on pipeline failure so we can 
browse it.


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/0bea97e1
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/0bea97e1
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/0bea97e1

Branch: refs/heads/HBASE-18426
Commit: 0bea97e1e01e4dfa924e6f8a8d99b9dffb6d2477
Parents: 2a71745
Author: Sean Busbey 
Authored: Sun Aug 6 14:46:56 2017 -0500
Committer: Sean Busbey 
Committed: Sun Aug 6 15:10:55 2017 -0500

--
 dev-support/Jenkinsfile | 10 +-
 1 file changed, 5 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/0bea97e1/dev-support/Jenkinsfile
--
diff --git a/dev-support/Jenkinsfile b/dev-support/Jenkinsfile
index 94d8c3f..96854c2 100644
--- a/dev-support/Jenkinsfile
+++ b/dev-support/Jenkinsfile
@@ -298,9 +298,9 @@ END
   }
 }
   }
-  post {
-failure {
-  deleteDir()
-}
-  }
+//  post {
+  //  failure {
+  //deleteDir()
+  //  }
+//  }
 }

hbase git commit: HBASE-18485 Performance issue: ClientAsyncPrefetchScanner is slower than ClientSimpleScanner

[zghao]

Repository: hbase
Updated Branches:
  refs/heads/master 2a717459b -> 5915d73a7


HBASE-18485 Performance issue: ClientAsyncPrefetchScanner is slower than 
ClientSimpleScanner


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/5915d73a
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/5915d73a
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/5915d73a

Branch: refs/heads/master
Commit: 5915d73a70eb69adc639062f372559c9fc5130be
Parents: 2a71745
Author: Guanghao Zhang 
Authored: Fri Aug 4 23:10:18 2017 +0800
Committer: Guanghao Zhang 
Committed: Mon Aug 7 10:35:19 2017 +0800

--
 .../client/ClientAsyncPrefetchScanner.java  | 181 ++-
 .../hadoop/hbase/client/ClientScanner.java  |   6 +-
 .../client/TestScannersFromClientSide.java  |   1 +
 3 files changed, 64 insertions(+), 124 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/5915d73a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ClientAsyncPrefetchScanner.java
--
diff --git 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ClientAsyncPrefetchScanner.java
 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ClientAsyncPrefetchScanner.java
index 34c5620..e8da18f 100644
--- 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ClientAsyncPrefetchScanner.java
+++ 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ClientAsyncPrefetchScanner.java
@@ -18,15 +18,20 @@
 package org.apache.hadoop.hbase.client;
 
 import 
org.apache.hadoop.hbase.shaded.com.google.common.annotations.VisibleForTesting;
+
 import static org.apache.hadoop.hbase.client.ConnectionUtils.calcEstimatedSize;
 
 import java.io.IOException;
+import java.io.InterruptedIOException;
 import java.util.Queue;
 import java.util.concurrent.ConcurrentLinkedQueue;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.LinkedBlockingQueue;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.locks.Condition;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReentrantLock;
 import java.util.function.Consumer;
 
 import org.apache.hadoop.conf.Configuration;
@@ -48,25 +53,18 @@ import org.apache.hadoop.hbase.util.Threads;
 @InterfaceAudience.Private
 public class ClientAsyncPrefetchScanner extends ClientSimpleScanner {
 
-  private static final int ESTIMATED_SINGLE_RESULT_SIZE = 1024;
-  private static final int DEFAULT_QUEUE_CAPACITY = 1024;
-
-  private int cacheCapacity;
+  private long maxCacheSize;
   private AtomicLong cacheSizeInBytes;
   // exception queue (from prefetch to main scan execution)
   private Queue exceptionsQueue;
-  // prefetch runnable object to be executed asynchronously
-  private PrefetchRunnable prefetchRunnable;
-  // Boolean flag to ensure only a single prefetch is running (per scan)
-  // We use atomic boolean to allow multiple concurrent threads to
-  // consume records from the same cache, but still have a single prefetcher 
thread.
-  // For a single consumer thread this can be replace with a native boolean.
-  private AtomicBoolean prefetchRunning;
-  // an attribute for synchronizing close between scanner and prefetch threads
-  private AtomicLong closingThreadId;
+  // prefetch thread to be executed asynchronously
+  private Thread prefetcher;
   // used for testing
   private Consumer prefetchListener;
-  private static final int NO_THREAD = -1;
+
+  private final Lock lock = new ReentrantLock();
+  private final Condition notEmpty = lock.newCondition();
+  private final Condition notFull = lock.newCondition();
 
   public ClientAsyncPrefetchScanner(Configuration configuration, Scan scan, 
TableName name,
   ClusterConnection connection, RpcRetryingCallerFactory rpcCallerFactory,
@@ -84,82 +82,56 @@ public class ClientAsyncPrefetchScanner extends 
ClientSimpleScanner {
   @Override
   protected void initCache() {
 // concurrent cache
-cacheCapacity = calcCacheCapacity();
+maxCacheSize = resultSize2CacheSize(maxScannerResultSize);
 cache = new LinkedBlockingQueue<>();
 cacheSizeInBytes = new AtomicLong(0);
 exceptionsQueue = new ConcurrentLinkedQueue<>();
-prefetchRunnable = new PrefetchRunnable();
-prefetchRunning = new AtomicBoolean(false);
-closingThreadId = new AtomicLong(NO_THREAD);
+prefetcher = new Thread(new PrefetchRunnable());
+Threads.setDaemonThreadRunning(prefetcher, tableName + ".asyncPrefetcher");
+  }
+
+  private long resultSize2CacheSize(long maxResultSize) {
+// * 2 if possible
+return maxResultSize > Long.MAX_VALUE / 2 ? maxResultSize : maxResultSize 
* 2;
   }
 
   

hbase git commit: HBASE-18485 Performance issue: ClientAsyncPrefetchScanner is slower than ClientSimpleScanner

[zghao]

Repository: hbase
Updated Branches:
  refs/heads/branch-2 69fddb58b -> be034c26b


HBASE-18485 Performance issue: ClientAsyncPrefetchScanner is slower than 
ClientSimpleScanner


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/be034c26
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/be034c26
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/be034c26

Branch: refs/heads/branch-2
Commit: be034c26b4ba820651d3255798ef01dc958608d5
Parents: 69fddb5
Author: Guanghao Zhang 
Authored: Fri Aug 4 23:10:18 2017 +0800
Committer: Guanghao Zhang 
Committed: Mon Aug 7 10:43:03 2017 +0800

--
 .../client/ClientAsyncPrefetchScanner.java  | 181 ++-
 .../hadoop/hbase/client/ClientScanner.java  |   6 +-
 .../client/TestScannersFromClientSide.java  |   1 +
 3 files changed, 64 insertions(+), 124 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/be034c26/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ClientAsyncPrefetchScanner.java
--
diff --git 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ClientAsyncPrefetchScanner.java
 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ClientAsyncPrefetchScanner.java
index 34c5620..e8da18f 100644
--- 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ClientAsyncPrefetchScanner.java
+++ 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ClientAsyncPrefetchScanner.java
@@ -18,15 +18,20 @@
 package org.apache.hadoop.hbase.client;
 
 import 
org.apache.hadoop.hbase.shaded.com.google.common.annotations.VisibleForTesting;
+
 import static org.apache.hadoop.hbase.client.ConnectionUtils.calcEstimatedSize;
 
 import java.io.IOException;
+import java.io.InterruptedIOException;
 import java.util.Queue;
 import java.util.concurrent.ConcurrentLinkedQueue;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.LinkedBlockingQueue;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.locks.Condition;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReentrantLock;
 import java.util.function.Consumer;
 
 import org.apache.hadoop.conf.Configuration;
@@ -48,25 +53,18 @@ import org.apache.hadoop.hbase.util.Threads;
 @InterfaceAudience.Private
 public class ClientAsyncPrefetchScanner extends ClientSimpleScanner {
 
-  private static final int ESTIMATED_SINGLE_RESULT_SIZE = 1024;
-  private static final int DEFAULT_QUEUE_CAPACITY = 1024;
-
-  private int cacheCapacity;
+  private long maxCacheSize;
   private AtomicLong cacheSizeInBytes;
   // exception queue (from prefetch to main scan execution)
   private Queue exceptionsQueue;
-  // prefetch runnable object to be executed asynchronously
-  private PrefetchRunnable prefetchRunnable;
-  // Boolean flag to ensure only a single prefetch is running (per scan)
-  // We use atomic boolean to allow multiple concurrent threads to
-  // consume records from the same cache, but still have a single prefetcher 
thread.
-  // For a single consumer thread this can be replace with a native boolean.
-  private AtomicBoolean prefetchRunning;
-  // an attribute for synchronizing close between scanner and prefetch threads
-  private AtomicLong closingThreadId;
+  // prefetch thread to be executed asynchronously
+  private Thread prefetcher;
   // used for testing
   private Consumer prefetchListener;
-  private static final int NO_THREAD = -1;
+
+  private final Lock lock = new ReentrantLock();
+  private final Condition notEmpty = lock.newCondition();
+  private final Condition notFull = lock.newCondition();
 
   public ClientAsyncPrefetchScanner(Configuration configuration, Scan scan, 
TableName name,
   ClusterConnection connection, RpcRetryingCallerFactory rpcCallerFactory,
@@ -84,82 +82,56 @@ public class ClientAsyncPrefetchScanner extends 
ClientSimpleScanner {
   @Override
   protected void initCache() {
 // concurrent cache
-cacheCapacity = calcCacheCapacity();
+maxCacheSize = resultSize2CacheSize(maxScannerResultSize);
 cache = new LinkedBlockingQueue<>();
 cacheSizeInBytes = new AtomicLong(0);
 exceptionsQueue = new ConcurrentLinkedQueue<>();
-prefetchRunnable = new PrefetchRunnable();
-prefetchRunning = new AtomicBoolean(false);
-closingThreadId = new AtomicLong(NO_THREAD);
+prefetcher = new Thread(new PrefetchRunnable());
+Threads.setDaemonThreadRunning(prefetcher, tableName + ".asyncPrefetcher");
+  }
+
+  private long resultSize2CacheSize(long maxResultSize) {
+// * 2 if possible
+return maxResultSize > Long.MAX_VALUE / 2 ? maxResultSize : maxResultSize 
* 2;
   }
 
   

hbase git commit: HBASE-18515 Introduce Delete.add as a replacement for Delete#addDeleteMarker

[chia7712]

Repository: hbase
Updated Branches:
  refs/heads/master 5915d73a7 -> a7014ce46


HBASE-18515 Introduce Delete.add as a replacement for Delete#addDeleteMarker

Signed-off-by: Chia-Ping Tsai 


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/a7014ce4
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/a7014ce4
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/a7014ce4

Branch: refs/heads/master
Commit: a7014ce46c15ad5293cc5b124df3463b3c5fa62d
Parents: 5915d73
Author: no_apologies 
Authored: Mon Aug 7 11:04:31 2017 +0800
Committer: Chia-Ping Tsai 
Committed: Mon Aug 7 11:05:23 2017 +0800

--
 .../org/apache/hadoop/hbase/client/Delete.java| 18 ++
 .../hadoop/hbase/protobuf/ProtobufUtil.java   |  2 +-
 .../hbase/shaded/protobuf/ProtobufUtil.java   |  2 +-
 .../coprocessor/TestRowProcessorEndpoint.java |  2 +-
 .../org/apache/hadoop/hbase/mapreduce/Import.java |  4 ++--
 .../apache/hadoop/hbase/mapreduce/WALPlayer.java  |  2 +-
 .../replication/regionserver/ReplicationSink.java |  2 +-
 .../security/visibility/VisibilityController.java |  2 +-
 .../org/apache/hadoop/hbase/wal/WALSplitter.java  |  2 +-
 9 files changed, 23 insertions(+), 13 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/a7014ce4/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Delete.java
--
diff --git 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Delete.java 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Delete.java
index 351d8a6..395c277 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Delete.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Delete.java
@@ -151,15 +151,26 @@ public class Delete extends Mutation implements 
Comparable {
   }
 
   /**
-   * Advanced use only.
-   * Add an existing delete marker to this Delete object.
+   * Advanced use only. Add an existing delete marker to this Delete object.
* @param kv An existing KeyValue of type "delete".
* @return this for invocation chaining
* @throws IOException
+   * @deprecated As of release 2.0.0, this will be removed in HBase 3.0.0. Use 
{@link #add(Cell)}
+   * instead
*/
   @SuppressWarnings("unchecked")
+  @Deprecated
   public Delete addDeleteMarker(Cell kv) throws IOException {
-// TODO: Deprecate and rename 'add' so it matches how we add KVs to Puts.
+return this.add(kv);
+  }
+
+  /**
+   * Add an existing delete marker to this Delete object.
+   * @param kv An existing KeyValue of type "delete".
+   * @return this for invocation chaining
+   * @throws IOException
+   */
+  public Delete add(Cell kv) throws IOException {
 if (!CellUtil.isDelete(kv)) {
   throw new IOException("The recently added KeyValue is not of type "
   + "delete. Rowkey: " + Bytes.toStringBinary(this.row));
@@ -178,7 +189,6 @@ public class Delete extends Mutation implements 
Comparable {
 return this;
   }
 
-
   /**
* Delete all versions of all columns of the specified family.
* 

http://git-wip-us.apache.org/repos/asf/hbase/blob/a7014ce4/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java
--
diff --git 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java
index fcf2c34..01ba0e0 100644
--- 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java
+++ 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java
@@ -579,7 +579,7 @@ public final class ProtobufUtil {
   delete =
 new Delete(cell.getRowArray(), cell.getRowOffset(), 
cell.getRowLength(), timestamp);
 }
-delete.addDeleteMarker(cell);
+delete.add(cell);
   }
 } else {
   if (delete == null) {

http://git-wip-us.apache.org/repos/asf/hbase/blob/a7014ce4/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java
--
diff --git 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java
 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java
index 542ade9..65e95b6 100644
--- 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java
+++ 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java
@@ -730,7 +730,7 @@ public final class ProtobufUtil {
   delete =
 new Delete(cell.getRowArray(), 

[2/2] hbase git commit: HBASE-18502 Change MasterObserver to use TableDescriptor and ColumnFamilyDescriptor

[chia7712]

HBASE-18502 Change MasterObserver to use TableDescriptor and 
ColumnFamilyDescriptor


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/afd5866d
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/afd5866d
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/afd5866d

Branch: refs/heads/branch-2
Commit: afd5866de04ec5b06ee8e75c38653a62d9b9db89
Parents: 2c06769
Author: Chia-Ping Tsai 
Authored: Mon Aug 7 11:25:45 2017 +0800
Committer: Chia-Ping Tsai 
Committed: Mon Aug 7 11:25:45 2017 +0800

--
 .../ExampleMasterObserverWithMetrics.java   |   6 +-
 .../hbase/rsgroup/RSGroupAdminEndpoint.java |   7 +-
 .../hbase/coprocessor/MasterObserver.java   | 122 ++-
 .../hadoop/hbase/coprocessor/package-info.java  |  17 +--
 .../org/apache/hadoop/hbase/master/HMaster.java |  19 +--
 .../hbase/master/MasterCoprocessorHost.java |  10 +-
 .../hadoop/hbase/master/MasterRpcServices.java  |   7 +-
 .../hadoop/hbase/master/MasterServices.java |   3 +-
 .../security/access/AccessControlLists.java |   3 +-
 .../hbase/security/access/AccessController.java |  41 +++
 .../CoprocessorWhitelistMasterObserver.java |  14 +--
 .../visibility/VisibilityController.java|   8 +-
 .../backup/TestBackupDeleteWithFailures.java|   4 +-
 .../hadoop/hbase/client/TestEnableTable.java|   2 +-
 .../coprocessor/TestCoprocessorMetrics.java |   5 +-
 ...TestMasterCoprocessorExceptionWithAbort.java |   3 +-
 ...estMasterCoprocessorExceptionWithRemove.java |   3 +-
 .../hbase/coprocessor/TestMasterObserver.java   |  54 
 .../hbase/master/MockNoopMasterServices.java|   3 +-
 .../hbase/namespace/TestNamespaceAuditor.java   |   5 +-
 .../hbase/security/access/SecureTestUtil.java   |   3 +-
 .../access/TestWithDisabledAuthorization.java   |   6 +-
 .../snapshot/TestSnapshotClientRetries.java |   6 +-
 .../hadoop/hbase/util/BaseTestHBaseFsck.java|   4 +-
 24 files changed, 184 insertions(+), 171 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/afd5866d/hbase-examples/src/main/java/org/apache/hadoop/hbase/coprocessor/example/ExampleMasterObserverWithMetrics.java
--
diff --git 
a/hbase-examples/src/main/java/org/apache/hadoop/hbase/coprocessor/example/ExampleMasterObserverWithMetrics.java
 
b/hbase-examples/src/main/java/org/apache/hadoop/hbase/coprocessor/example/ExampleMasterObserverWithMetrics.java
index 6649162..8535d05 100644
--- 
a/hbase-examples/src/main/java/org/apache/hadoop/hbase/coprocessor/example/ExampleMasterObserverWithMetrics.java
+++ 
b/hbase-examples/src/main/java/org/apache/hadoop/hbase/coprocessor/example/ExampleMasterObserverWithMetrics.java
@@ -24,8 +24,8 @@ import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.hbase.CoprocessorEnvironment;
 import org.apache.hadoop.hbase.HRegionInfo;
-import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.TableDescriptor;
 import org.apache.hadoop.hbase.coprocessor.MasterObserver;
 import org.apache.hadoop.hbase.coprocessor.MasterCoprocessorEnvironment;
 import org.apache.hadoop.hbase.coprocessor.ObserverContext;
@@ -68,7 +68,7 @@ public class ExampleMasterObserverWithMetrics implements 
MasterObserver {
 
   @Override
   public void preCreateTable(ObserverContext ctx,
- HTableDescriptor desc, HRegionInfo[] regions) 
throws IOException {
+ TableDescriptor desc, HRegionInfo[] regions) 
throws IOException {
 // we rely on the fact that there is only 1 instance of our 
MasterObserver. We keep track of
 // when the operation starts before the operation is executing.
 this.createTableStartTime = System.currentTimeMillis();
@@ -76,7 +76,7 @@ public class ExampleMasterObserverWithMetrics implements 
MasterObserver {
 
   @Override
   public void postCreateTable(ObserverContext 
ctx,
-  HTableDescriptor desc, HRegionInfo[] regions) 
throws IOException {
+  TableDescriptor desc, HRegionInfo[] regions) 
throws IOException {
 if (this.createTableStartTime > 0) {
   long time = System.currentTimeMillis() - this.createTableStartTime;
   LOG.info("Create table took: " + time);

http://git-wip-us.apache.org/repos/asf/hbase/blob/afd5866d/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminEndpoint.java
--
diff --git 
a/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminEndpoint.java
 

[2/2] hbase git commit: HBASE-18502 Change MasterObserver to use TableDescriptor and ColumnFamilyDescriptor

[chia7712]

HBASE-18502 Change MasterObserver to use TableDescriptor and 
ColumnFamilyDescriptor


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/fd76eb39
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/fd76eb39
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/fd76eb39

Branch: refs/heads/master
Commit: fd76eb39d727f2794cf9eaa7c81d27dc2bf1036b
Parents: a7014ce
Author: Chia-Ping Tsai 
Authored: Mon Aug 7 11:26:15 2017 +0800
Committer: Chia-Ping Tsai 
Committed: Mon Aug 7 11:26:15 2017 +0800

--
 .../ExampleMasterObserverWithMetrics.java   |   6 +-
 .../hbase/rsgroup/RSGroupAdminEndpoint.java |   7 +-
 .../hbase/coprocessor/MasterObserver.java   | 122 ++-
 .../hadoop/hbase/coprocessor/package-info.java  |  17 +--
 .../org/apache/hadoop/hbase/master/HMaster.java |  19 +--
 .../hbase/master/MasterCoprocessorHost.java |  10 +-
 .../hadoop/hbase/master/MasterRpcServices.java  |   7 +-
 .../hadoop/hbase/master/MasterServices.java |   3 +-
 .../security/access/AccessControlLists.java |   3 +-
 .../hbase/security/access/AccessController.java |  41 +++
 .../CoprocessorWhitelistMasterObserver.java |  14 +--
 .../visibility/VisibilityController.java|   8 +-
 .../backup/TestBackupDeleteWithFailures.java|   4 +-
 .../hadoop/hbase/client/TestEnableTable.java|   2 +-
 .../coprocessor/TestCoprocessorMetrics.java |   5 +-
 ...TestMasterCoprocessorExceptionWithAbort.java |   3 +-
 ...estMasterCoprocessorExceptionWithRemove.java |   3 +-
 .../hbase/coprocessor/TestMasterObserver.java   |  54 
 .../hbase/master/MockNoopMasterServices.java|   3 +-
 .../hbase/namespace/TestNamespaceAuditor.java   |   5 +-
 .../hbase/security/access/SecureTestUtil.java   |   3 +-
 .../access/TestWithDisabledAuthorization.java   |   6 +-
 .../snapshot/TestSnapshotClientRetries.java |   6 +-
 .../hadoop/hbase/util/BaseTestHBaseFsck.java|   4 +-
 24 files changed, 184 insertions(+), 171 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/fd76eb39/hbase-examples/src/main/java/org/apache/hadoop/hbase/coprocessor/example/ExampleMasterObserverWithMetrics.java
--
diff --git 
a/hbase-examples/src/main/java/org/apache/hadoop/hbase/coprocessor/example/ExampleMasterObserverWithMetrics.java
 
b/hbase-examples/src/main/java/org/apache/hadoop/hbase/coprocessor/example/ExampleMasterObserverWithMetrics.java
index 6649162..8535d05 100644
--- 
a/hbase-examples/src/main/java/org/apache/hadoop/hbase/coprocessor/example/ExampleMasterObserverWithMetrics.java
+++ 
b/hbase-examples/src/main/java/org/apache/hadoop/hbase/coprocessor/example/ExampleMasterObserverWithMetrics.java
@@ -24,8 +24,8 @@ import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.hbase.CoprocessorEnvironment;
 import org.apache.hadoop.hbase.HRegionInfo;
-import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.TableDescriptor;
 import org.apache.hadoop.hbase.coprocessor.MasterObserver;
 import org.apache.hadoop.hbase.coprocessor.MasterCoprocessorEnvironment;
 import org.apache.hadoop.hbase.coprocessor.ObserverContext;
@@ -68,7 +68,7 @@ public class ExampleMasterObserverWithMetrics implements 
MasterObserver {
 
   @Override
   public void preCreateTable(ObserverContext ctx,
- HTableDescriptor desc, HRegionInfo[] regions) 
throws IOException {
+ TableDescriptor desc, HRegionInfo[] regions) 
throws IOException {
 // we rely on the fact that there is only 1 instance of our 
MasterObserver. We keep track of
 // when the operation starts before the operation is executing.
 this.createTableStartTime = System.currentTimeMillis();
@@ -76,7 +76,7 @@ public class ExampleMasterObserverWithMetrics implements 
MasterObserver {
 
   @Override
   public void postCreateTable(ObserverContext 
ctx,
-  HTableDescriptor desc, HRegionInfo[] regions) 
throws IOException {
+  TableDescriptor desc, HRegionInfo[] regions) 
throws IOException {
 if (this.createTableStartTime > 0) {
   long time = System.currentTimeMillis() - this.createTableStartTime;
   LOG.info("Create table took: " + time);

http://git-wip-us.apache.org/repos/asf/hbase/blob/fd76eb39/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminEndpoint.java
--
diff --git 
a/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminEndpoint.java
 

[1/2] hbase git commit: HBASE-18502 Change MasterObserver to use TableDescriptor and ColumnFamilyDescriptor

[chia7712]

Repository: hbase
Updated Branches:
  refs/heads/branch-2 2c067695d -> afd5866de


http://git-wip-us.apache.org/repos/asf/hbase/blob/afd5866d/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestSnapshotClientRetries.java
--
diff --git 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestSnapshotClientRetries.java
 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestSnapshotClientRetries.java
index bd9d5dc..5da0d94 100644
--- 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestSnapshotClientRetries.java
+++ 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestSnapshotClientRetries.java
@@ -26,8 +26,8 @@ import java.util.concurrent.atomic.AtomicInteger;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.hbase.HBaseTestingUtility;
-import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.TableDescriptor;
 import org.apache.hadoop.hbase.coprocessor.MasterObserver;
 import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
 import org.apache.hadoop.hbase.coprocessor.MasterCoprocessorEnvironment;
@@ -80,7 +80,7 @@ public class TestSnapshotClientRetries {
 
 @Override
 public void preSnapshot(final 
ObserverContext ctx,
-final SnapshotDescription snapshot, final HTableDescriptor 
hTableDescriptor)
+final SnapshotDescription snapshot, final TableDescriptor 
hTableDescriptor)
 throws IOException {
   if (snapshotCount != null) {
 snapshotCount.incrementAndGet();
@@ -89,7 +89,7 @@ public class TestSnapshotClientRetries {
 
 @Override
 public void preCloneSnapshot(final 
ObserverContext ctx,
-final SnapshotDescription snapshot, final HTableDescriptor 
hTableDescriptor)
+final SnapshotDescription snapshot, final TableDescriptor 
hTableDescriptor)
 throws IOException {
   if (cloneCount != null) {
 cloneCount.incrementAndGet();

http://git-wip-us.apache.org/repos/asf/hbase/blob/afd5866d/hbase-server/src/test/java/org/apache/hadoop/hbase/util/BaseTestHBaseFsck.java
--
diff --git 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/BaseTestHBaseFsck.java
 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/BaseTestHBaseFsck.java
index f6e328e..c18d6d0 100644
--- 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/BaseTestHBaseFsck.java
+++ 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/BaseTestHBaseFsck.java
@@ -55,9 +55,9 @@ import org.apache.hadoop.hbase.client.ConnectionFactory;
 import org.apache.hadoop.hbase.client.Delete;
 import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.client.RegionLocator;
-import org.apache.hadoop.hbase.client.ResultScanner;
 import org.apache.hadoop.hbase.client.Scan;
 import org.apache.hadoop.hbase.client.Table;
+import org.apache.hadoop.hbase.client.TableDescriptor;
 import org.apache.hadoop.hbase.coprocessor.MasterObserver;
 import org.apache.hadoop.hbase.coprocessor.MasterCoprocessorEnvironment;
 import org.apache.hadoop.hbase.coprocessor.ObserverContext;
@@ -598,7 +598,7 @@ public class BaseTestHBaseFsck {
 @Override
 public void postCompletedCreateTableAction(
 final ObserverContext ctx,
-final HTableDescriptor desc,
+final TableDescriptor desc,
 final HRegionInfo[] regions) throws IOException {
   // the AccessController test, some times calls only and directly the
   // postCompletedCreateTableAction()

[1/2] hbase git commit: HBASE-18502 Change MasterObserver to use TableDescriptor and ColumnFamilyDescriptor

[chia7712]

Repository: hbase
Updated Branches:
  refs/heads/master a7014ce46 -> fd76eb39d


http://git-wip-us.apache.org/repos/asf/hbase/blob/fd76eb39/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestSnapshotClientRetries.java
--
diff --git 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestSnapshotClientRetries.java
 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestSnapshotClientRetries.java
index bd9d5dc..5da0d94 100644
--- 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestSnapshotClientRetries.java
+++ 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestSnapshotClientRetries.java
@@ -26,8 +26,8 @@ import java.util.concurrent.atomic.AtomicInteger;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.hbase.HBaseTestingUtility;
-import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.TableDescriptor;
 import org.apache.hadoop.hbase.coprocessor.MasterObserver;
 import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
 import org.apache.hadoop.hbase.coprocessor.MasterCoprocessorEnvironment;
@@ -80,7 +80,7 @@ public class TestSnapshotClientRetries {
 
 @Override
 public void preSnapshot(final 
ObserverContext ctx,
-final SnapshotDescription snapshot, final HTableDescriptor 
hTableDescriptor)
+final SnapshotDescription snapshot, final TableDescriptor 
hTableDescriptor)
 throws IOException {
   if (snapshotCount != null) {
 snapshotCount.incrementAndGet();
@@ -89,7 +89,7 @@ public class TestSnapshotClientRetries {
 
 @Override
 public void preCloneSnapshot(final 
ObserverContext ctx,
-final SnapshotDescription snapshot, final HTableDescriptor 
hTableDescriptor)
+final SnapshotDescription snapshot, final TableDescriptor 
hTableDescriptor)
 throws IOException {
   if (cloneCount != null) {
 cloneCount.incrementAndGet();

http://git-wip-us.apache.org/repos/asf/hbase/blob/fd76eb39/hbase-server/src/test/java/org/apache/hadoop/hbase/util/BaseTestHBaseFsck.java
--
diff --git 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/BaseTestHBaseFsck.java
 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/BaseTestHBaseFsck.java
index f6e328e..c18d6d0 100644
--- 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/BaseTestHBaseFsck.java
+++ 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/BaseTestHBaseFsck.java
@@ -55,9 +55,9 @@ import org.apache.hadoop.hbase.client.ConnectionFactory;
 import org.apache.hadoop.hbase.client.Delete;
 import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.client.RegionLocator;
-import org.apache.hadoop.hbase.client.ResultScanner;
 import org.apache.hadoop.hbase.client.Scan;
 import org.apache.hadoop.hbase.client.Table;
+import org.apache.hadoop.hbase.client.TableDescriptor;
 import org.apache.hadoop.hbase.coprocessor.MasterObserver;
 import org.apache.hadoop.hbase.coprocessor.MasterCoprocessorEnvironment;
 import org.apache.hadoop.hbase.coprocessor.ObserverContext;
@@ -598,7 +598,7 @@ public class BaseTestHBaseFsck {
 @Override
 public void postCompletedCreateTableAction(
 final ObserverContext ctx,
-final HTableDescriptor desc,
+final TableDescriptor desc,
 final HRegionInfo[] regions) throws IOException {
   // the AccessController test, some times calls only and directly the
   // postCompletedCreateTableAction()

hbase git commit: HBASE-18515 Introduce Delete.add as a replacement for Delete#addDeleteMarker

[chia7712]

Repository: hbase
Updated Branches:
  refs/heads/branch-2 be034c26b -> 2c067695d


HBASE-18515 Introduce Delete.add as a replacement for Delete#addDeleteMarker

Signed-off-by: Chia-Ping Tsai 


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/2c067695
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/2c067695
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/2c067695

Branch: refs/heads/branch-2
Commit: 2c067695dd1819ef832335bb481fac57b0de6e3b
Parents: be034c2
Author: no_apologies 
Authored: Mon Aug 7 11:04:31 2017 +0800
Committer: Chia-Ping Tsai 
Committed: Mon Aug 7 11:13:12 2017 +0800

--
 .../org/apache/hadoop/hbase/client/Delete.java| 18 ++
 .../hadoop/hbase/protobuf/ProtobufUtil.java   |  2 +-
 .../hbase/shaded/protobuf/ProtobufUtil.java   |  2 +-
 .../coprocessor/TestRowProcessorEndpoint.java |  2 +-
 .../org/apache/hadoop/hbase/mapreduce/Import.java |  4 ++--
 .../apache/hadoop/hbase/mapreduce/WALPlayer.java  |  2 +-
 .../replication/regionserver/ReplicationSink.java |  2 +-
 .../security/visibility/VisibilityController.java |  2 +-
 .../org/apache/hadoop/hbase/wal/WALSplitter.java  |  2 +-
 9 files changed, 23 insertions(+), 13 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/2c067695/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Delete.java
--
diff --git 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Delete.java 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Delete.java
index 351d8a6..395c277 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Delete.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Delete.java
@@ -151,15 +151,26 @@ public class Delete extends Mutation implements 
Comparable {
   }
 
   /**
-   * Advanced use only.
-   * Add an existing delete marker to this Delete object.
+   * Advanced use only. Add an existing delete marker to this Delete object.
* @param kv An existing KeyValue of type "delete".
* @return this for invocation chaining
* @throws IOException
+   * @deprecated As of release 2.0.0, this will be removed in HBase 3.0.0. Use 
{@link #add(Cell)}
+   * instead
*/
   @SuppressWarnings("unchecked")
+  @Deprecated
   public Delete addDeleteMarker(Cell kv) throws IOException {
-// TODO: Deprecate and rename 'add' so it matches how we add KVs to Puts.
+return this.add(kv);
+  }
+
+  /**
+   * Add an existing delete marker to this Delete object.
+   * @param kv An existing KeyValue of type "delete".
+   * @return this for invocation chaining
+   * @throws IOException
+   */
+  public Delete add(Cell kv) throws IOException {
 if (!CellUtil.isDelete(kv)) {
   throw new IOException("The recently added KeyValue is not of type "
   + "delete. Rowkey: " + Bytes.toStringBinary(this.row));
@@ -178,7 +189,6 @@ public class Delete extends Mutation implements 
Comparable {
 return this;
   }
 
-
   /**
* Delete all versions of all columns of the specified family.
* 

http://git-wip-us.apache.org/repos/asf/hbase/blob/2c067695/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java
--
diff --git 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java
index bc03699..6f9238a 100644
--- 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java
+++ 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java
@@ -580,7 +580,7 @@ public final class ProtobufUtil {
   delete =
 new Delete(cell.getRowArray(), cell.getRowOffset(), 
cell.getRowLength(), timestamp);
 }
-delete.addDeleteMarker(cell);
+delete.add(cell);
   }
 } else {
   if (delete == null) {

http://git-wip-us.apache.org/repos/asf/hbase/blob/2c067695/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java
--
diff --git 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java
 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java
index 542ade9..65e95b6 100644
--- 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java
+++ 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java
@@ -730,7 +730,7 @@ public final class ProtobufUtil {
   delete =
 new Delete(cell.getRowArray(),