hbase git commit: HBASE-14548 Expand how table coprocessor jar and dependency path can be specified (Xiang Li)

2016-07-09 Thread jerryjch 
Repository: hbase
Updated Branches:
  refs/heads/branch-1 9563ab4ca -> fe915e10d


HBASE-14548 Expand how table coprocessor jar and dependency path can be 
specified (Xiang Li)


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/fe915e10
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/fe915e10
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/fe915e10

Branch: refs/heads/branch-1
Commit: fe915e10d6f021357e53984742bcb36091306ea1
Parents: 9563ab4
Author: Jerry He 
Authored: Sat Jul 9 18:18:22 2016 -0700
Committer: Jerry He 
Committed: Sat Jul 9 18:18:22 2016 -0700

--
 .../hbase/util/CoprocessorClassLoader.java  | 71 +---
 .../hbase/util/TestCoprocessorClassLoader.java  | 43 
 2 files changed, 90 insertions(+), 24 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/fe915e10/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CoprocessorClassLoader.java
--
diff --git 
a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CoprocessorClassLoader.java
 
b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CoprocessorClassLoader.java
index 5f949d9..56fab75 100644
--- 
a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CoprocessorClassLoader.java
+++ 
b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CoprocessorClassLoader.java
@@ -18,6 +18,7 @@
 package org.apache.hadoop.hbase.util;
 
 import java.io.File;
+import java.io.FileNotFoundException;
 import java.io.FileOutputStream;
 import java.io.IOException;
 import java.net.URL;
@@ -39,6 +40,9 @@ import 
org.apache.hadoop.hbase.classification.InterfaceAudience;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
 import org.apache.hadoop.io.IOUtils;
 
 import com.google.common.base.Preconditions;
@@ -155,7 +159,7 @@ public class CoprocessorClassLoader extends ClassLoaderBase 
{
 super(parent);
   }
 
-  private void init(Path path, String pathPrefix,
+  private void init(Path pathPattern, String pathPrefix,
   Configuration conf) throws IOException {
 // Copy the jar to the local filesystem
 String parentDirStr =
@@ -173,31 +177,50 @@ public class CoprocessorClassLoader extends 
ClassLoaderBase {
   }
 }
 
-FileSystem fs = path.getFileSystem(conf);
-File dst = new File(parentDirStr, "." + pathPrefix + "."
-  + path.getName() + "." + System.currentTimeMillis() + ".jar");
-fs.copyToLocalFile(path, new Path(dst.toString()));
-dst.deleteOnExit();
-
-addURL(dst.getCanonicalFile().toURI().toURL());
+FileSystem fs = pathPattern.getFileSystem(conf);
+Path pathPattern1 = fs.isDirectory(pathPattern) ?
+  new Path(pathPattern, "*.jar") : pathPattern;  // append "*.jar" if a 
directory is specified
+FileStatus[] fileStatuses = fs.globStatus(pathPattern1);  // return all 
files that match the pattern
+if (fileStatuses == null || fileStatuses.length == 0) {  // if no one 
matches
+  throw new FileNotFoundException(pathPattern1.toString());
+} else {
+  boolean validFileEncountered = false;
+  for (Path path : FileUtil.stat2Paths(fileStatuses)) {  // for each file 
that match the pattern
+if (fs.isFile(path)) {  // only process files, skip for directories
+  File dst = new File(parentDirStr, "." + pathPrefix + "."
++ path.getName() + "." + System.currentTimeMillis() + ".jar");
+  fs.copyToLocalFile(path, new Path(dst.toString()));
+  dst.deleteOnExit();
+
+  addURL(dst.getCanonicalFile().toURI().toURL());
+
+  JarFile jarFile = new JarFile(dst.toString());
+  try {
+Enumeration entries = jarFile.entries();  // get entries 
inside a jar file
+while (entries.hasMoreElements()) {
+  JarEntry entry = entries.nextElement();
+  Matcher m = libJarPattern.matcher(entry.getName());
+  if (m.matches()) {
+File file = new File(parentDirStr, "." + pathPrefix + "."
+  + path.getName() + "." + System.currentTimeMillis() + "." + 
m.group(1));
+try (FileOutputStream outStream = new FileOutputStream(file)) {
+  IOUtils.copyBytes(jarFile.getInputStream(entry),
+outStream, conf, true);
+}
+file.deleteOnExit();
+addURL(file.toURI().toURL());
+  }
+}
+  } finally {
+jarFile.close();
+  }
 
-

hbase git commit: HBASE-14548 Expand how table coprocessor jar and dependency path can be specified (Xiang Li)

2016-07-09 Thread jerryjch 
Repository: hbase
Updated Branches:
  refs/heads/master 496fd9837 -> 632969787


HBASE-14548 Expand how table coprocessor jar and dependency path can be 
specified (Xiang Li)


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/63296978
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/63296978
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/63296978

Branch: refs/heads/master
Commit: 632969787aa9e9e1d73b83449b93e21f91110b73
Parents: 496fd98
Author: Jerry He 
Authored: Sat Jul 9 18:00:41 2016 -0700
Committer: Jerry He 
Committed: Sat Jul 9 18:01:49 2016 -0700

--
 .../hbase/util/CoprocessorClassLoader.java  | 72 +---
 .../hbase/util/TestCoprocessorClassLoader.java  | 43 
 src/main/asciidoc/_chapters/cp.adoc |  9 ++-
 3 files changed, 97 insertions(+), 27 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/63296978/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CoprocessorClassLoader.java
--
diff --git 
a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CoprocessorClassLoader.java
 
b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CoprocessorClassLoader.java
index 11016c3..c3635cb 100644
--- 
a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CoprocessorClassLoader.java
+++ 
b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CoprocessorClassLoader.java
@@ -18,6 +18,7 @@
 package org.apache.hadoop.hbase.util;
 
 import java.io.File;
+import java.io.FileNotFoundException;
 import java.io.FileOutputStream;
 import java.io.IOException;
 import java.net.URL;
@@ -38,6 +39,8 @@ import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileUtil;
 import org.apache.hadoop.hbase.classification.InterfaceAudience;
 import org.apache.hadoop.io.IOUtils;
 
@@ -142,7 +145,7 @@ public class CoprocessorClassLoader extends ClassLoaderBase 
{
 super(parent);
   }
 
-  private void init(Path path, String pathPrefix,
+  private void init(Path pathPattern, String pathPrefix,
   Configuration conf) throws IOException {
 // Copy the jar to the local filesystem
 String parentDirStr =
@@ -160,33 +163,50 @@ public class CoprocessorClassLoader extends 
ClassLoaderBase {
   }
 }
 
-FileSystem fs = path.getFileSystem(conf);
-File dst = new File(parentDirStr, "." + pathPrefix + "."
-  + path.getName() + "." + System.currentTimeMillis() + ".jar");
-fs.copyToLocalFile(path, new Path(dst.toString()));
-dst.deleteOnExit();
-
-addURL(dst.getCanonicalFile().toURI().toURL());
-
-JarFile jarFile = new JarFile(dst.toString());
-try {
-  Enumeration entries = jarFile.entries();
-  while (entries.hasMoreElements()) {
-JarEntry entry = entries.nextElement();
-Matcher m = libJarPattern.matcher(entry.getName());
-if (m.matches()) {
-  File file = new File(parentDirStr, "." + pathPrefix + "."
-+ path.getName() + "." + System.currentTimeMillis() + "." + 
m.group(1));
-  try (FileOutputStream outStream = new FileOutputStream(file)) {
-IOUtils.copyBytes(jarFile.getInputStream(entry),
-  outStream, conf, true);
+FileSystem fs = pathPattern.getFileSystem(conf);
+Path pathPattern1 = fs.isDirectory(pathPattern) ?
+  new Path(pathPattern, "*.jar") : pathPattern;  // append "*.jar" if a 
directory is specified
+FileStatus[] fileStatuses = fs.globStatus(pathPattern1);  // return all 
files that match the pattern
+if (fileStatuses == null || fileStatuses.length == 0) {  // if no one 
matches
+  throw new FileNotFoundException(pathPattern1.toString());
+} else {
+  boolean validFileEncountered = false;
+  for (Path path : FileUtil.stat2Paths(fileStatuses)) {  // for each file 
that match the pattern
+if (fs.isFile(path)) {  // only process files, skip for directories
+  File dst = new File(parentDirStr, "." + pathPrefix + "."
++ path.getName() + "." + System.currentTimeMillis() + ".jar");
+  fs.copyToLocalFile(path, new Path(dst.toString()));
+  dst.deleteOnExit();
+
+  addURL(dst.getCanonicalFile().toURI().toURL());
+
+  JarFile jarFile = new JarFile(dst.toString());
+  try {
+Enumeration entries = jarFile.entries();  // get entries 
inside a jar file
+while (entries.hasMoreElements()) {
+  JarEntry entry = entries.nextElement();
+  Matcher m =

hbase git commit: HBASE-16176 Bug fixes/improvements on HBASE-15650 Remove TimeRangeTracker as point of contention when many threads reading a StoreFile Fixes HBASE-16074 ITBLL fails, reports lost big

2016-07-09 Thread stack 
Repository: hbase
Updated Branches:
  refs/heads/master 3c39cbd92 -> 496fd9837


HBASE-16176 Bug fixes/improvements on HBASE-15650 Remove
 TimeRangeTracker as point of contention when many threads reading a StoreFile
 Fixes HBASE-16074 ITBLL fails, reports lost big or tiny families broken
 scanning because of a side effect of a clean up in  HBASE-15650 to make
 TimeRange construction consistent exposed a latent issue in
 TimeRange#compare. See HBASE-16074 for more detail.

Also change HFile Writer constructor so we pass in the TimeRangeTracker, if one,
on construction rather than set later (the flag and reference were not volatile
so could have made for issues in concurrent case). And make sure the 
construction
of a TimeRange from a TimeRangeTracer on open of an HFile Reader never makes a
bad minimum value, one that would preclude us reading any values from a file
(set min to 0)

M hbase-common/src/main/java/org/apache/hadoop/hbase/io/TimeRange.java
 Call through to next constructor (if minStamp was 0, we'd skip setting
 allTime=true). Add asserts that timestamps are not < 0 cos it messes
 us up if they are (we already were checking for < 0 on construction but
 assert passed in timestamps are not < 0).

M hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java
 Add constructor override that takes a TimeRangeTracker (set when flushing
 but not when compacting)

M hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
 Add override creating an HFile in tmp that takes a TimeRangeTracker

M hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
 Add override for HFile Writer that takes a TimeRangeTracker Take it on
 construction instead of having it passed by a setter later (flags and
 reference set by the setter were not volatile... could have been prob
 in concurrent case)

M 
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/TimeRangeTracker.java
 Log WARN if bad initial TimeRange value (and then 'fix' it)

M 
hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestTimeRangeTracker.java
 A few tests to prove serialization works as expected and that we'll get a bad 
min if not constructed properly.

M 
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java
 Handle OLDEST_TIMESTAMP explictly. Don't expect TimeRange to do it.

M 
hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestQueryMatcher.java
 Refactor from junit3 to junit4 and add test for this weird case.


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/496fd983
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/496fd983
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/496fd983

Branch: refs/heads/master
Commit: 496fd9837a0fb199a516758a632fecfe59b0b480
Parents: 3c39cbd
Author: stack 
Authored: Fri Jul 8 18:18:01 2016 -0700
Committer: stack 
Committed: Fri Jul 8 18:20:31 2016 -0700

--
 .../org/apache/hadoop/hbase/client/Query.java   |  4 +-
 .../org/apache/hadoop/hbase/HConstants.java |  5 +
 .../org/apache/hadoop/hbase/KeyValueUtil.java   |  2 +-
 .../org/apache/hadoop/hbase/io/TimeRange.java   | 99 
 .../hbase/mob/DefaultMobStoreFlusher.java   |  5 +-
 .../hbase/regionserver/DefaultStoreFlusher.java |  4 +-
 .../regionserver/ExplicitColumnTracker.java |  4 +-
 .../hadoop/hbase/regionserver/HStore.java   | 27 +-
 .../hbase/regionserver/ScanQueryMatcher.java| 11 ++-
 .../apache/hadoop/hbase/regionserver/Store.java | 20 +++-
 .../hadoop/hbase/regionserver/StoreFile.java|  1 -
 .../hbase/regionserver/StoreFileReader.java |  2 +-
 .../hbase/regionserver/StoreFileWriter.java | 61 
 .../hadoop/hbase/regionserver/StoreScanner.java |  3 +-
 .../hbase/regionserver/StripeStoreFlusher.java  |  4 +-
 .../hbase/regionserver/TimeRangeTracker.java| 23 ++---
 .../hbase/regionserver/TestQueryMatcher.java| 35 +--
 .../regionserver/TestTimeRangeTracker.java  | 40 +++-
 18 files changed, 250 insertions(+), 100 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/496fd983/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Query.java
--
diff --git 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Query.java 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Query.java
index 53062a0..53dd2c1 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Query.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Query.java
@@ -203,6 +203,4 @@ public abstract class Query extends OperationWithAttributes 
{
   public Map getColumnFamilyTimeRange() {