Repository: crunch Updated Branches: refs/heads/master 2c9e8306c -> c88ce4718
CRUNCH-386: HBase 0.98.1 upgrade/fixes Project: http://git-wip-us.apache.org/repos/asf/crunch/repo Commit: http://git-wip-us.apache.org/repos/asf/crunch/commit/c88ce471 Tree: http://git-wip-us.apache.org/repos/asf/crunch/tree/c88ce471 Diff: http://git-wip-us.apache.org/repos/asf/crunch/diff/c88ce471 Branch: refs/heads/master Commit: c88ce47187b0a06ceba1e12ddee3077bf3f16b36 Parents: 2c9e830 Author: Josh Wills <[email protected]> Authored: Wed Apr 30 09:50:57 2014 -0700 Committer: Josh Wills <[email protected]> Committed: Thu May 1 09:56:42 2014 -0700 ---------------------------------------------------------------------- .../org/apache/crunch/io/hbase/HFileSourceIT.java | 2 ++ .../org/apache/crunch/io/hbase/HFileTargetIT.java | 6 +++--- .../org/apache/crunch/io/hbase/HFileInputFormat.java | 2 +- .../crunch/io/hbase/HFileOutputFormatForCrunch.java | 14 +++++++++----- .../apache/crunch/io/hbase/HFileReaderFactory.java | 2 +- .../java/org/apache/crunch/io/hbase/HFileUtils.java | 5 +---- pom.xml | 4 ++-- 7 files changed, 19 insertions(+), 16 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/crunch/blob/c88ce471/crunch-hbase/src/it/java/org/apache/crunch/io/hbase/HFileSourceIT.java ---------------------------------------------------------------------- diff --git a/crunch-hbase/src/it/java/org/apache/crunch/io/hbase/HFileSourceIT.java b/crunch-hbase/src/it/java/org/apache/crunch/io/hbase/HFileSourceIT.java index 05c6a42..e82102b 100644 --- a/crunch-hbase/src/it/java/org/apache/crunch/io/hbase/HFileSourceIT.java +++ b/crunch-hbase/src/it/java/org/apache/crunch/io/hbase/HFileSourceIT.java @@ -38,6 +38,7 @@ import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.io.hfile.CacheConfig; import org.apache.hadoop.hbase.io.hfile.HFile; +import org.apache.hadoop.hbase.io.hfile.HFileContext; import org.apache.hadoop.hbase.util.Bytes; import org.junit.Before; import org.junit.Rule; @@ -304,6 +305,7 @@ public class HFileSourceIT implements Serializable { w = HFile.getWriterFactory(conf, new CacheConfig(conf)) .withPath(fs, inputPath) .withComparator(KeyValue.COMPARATOR) + .withFileContext(new HFileContext()) .create(); for (KeyValue kv : sortedKVs) { w.append(kv); http://git-wip-us.apache.org/repos/asf/crunch/blob/c88ce471/crunch-hbase/src/it/java/org/apache/crunch/io/hbase/HFileTargetIT.java ---------------------------------------------------------------------- diff --git a/crunch-hbase/src/it/java/org/apache/crunch/io/hbase/HFileTargetIT.java b/crunch-hbase/src/it/java/org/apache/crunch/io/hbase/HFileTargetIT.java index 7dd035e..25cec98 100644 --- a/crunch-hbase/src/it/java/org/apache/crunch/io/hbase/HFileTargetIT.java +++ b/crunch-hbase/src/it/java/org/apache/crunch/io/hbase/HFileTargetIT.java @@ -259,8 +259,8 @@ public class HFileTargetIT implements Serializable { } HFile.Reader reader = null; try { - reader = HFile.createReader(fs, f, new CacheConfig(conf)); - assertEquals(DataBlockEncoding.PREFIX, reader.getEncodingOnDisk()); + reader = HFile.createReader(fs, f, new CacheConfig(conf), conf); + assertEquals(DataBlockEncoding.PREFIX, reader.getDataBlockEncoding()); } finally { reader.close(); } @@ -328,7 +328,7 @@ public class HFileTargetIT implements Serializable { fs, f, new CacheConfig(fs.getConf()), - DataBlockEncoding.NONE); + fs.getConf()); StoreFileScanner scanner = reader.getStoreFileScanner(false, false); scanner.seek(fakeKV); // have to call seek of each underlying scanner, otherwise KeyValueHeap won't work scanners.add(scanner); http://git-wip-us.apache.org/repos/asf/crunch/blob/c88ce471/crunch-hbase/src/main/java/org/apache/crunch/io/hbase/HFileInputFormat.java ---------------------------------------------------------------------- diff --git a/crunch-hbase/src/main/java/org/apache/crunch/io/hbase/HFileInputFormat.java b/crunch-hbase/src/main/java/org/apache/crunch/io/hbase/HFileInputFormat.java index 9ced0ac..43005ab 100644 --- a/crunch-hbase/src/main/java/org/apache/crunch/io/hbase/HFileInputFormat.java +++ b/crunch-hbase/src/main/java/org/apache/crunch/io/hbase/HFileInputFormat.java @@ -96,7 +96,7 @@ public class HFileInputFormat extends FileInputFormat<NullWritable, KeyValue> { Path path = fileSplit.getPath(); FileSystem fs = path.getFileSystem(conf); LOG.info("Initialize HFileRecordReader for " + path); - this.in = HFile.createReader(fs, path, new CacheConfig(conf)); + this.in = HFile.createReader(fs, path, new CacheConfig(conf), conf); // The file info must be loaded before the scanner can be used. // This seems like a bug in HBase, but it's easily worked around. http://git-wip-us.apache.org/repos/asf/crunch/blob/c88ce471/crunch-hbase/src/main/java/org/apache/crunch/io/hbase/HFileOutputFormatForCrunch.java ---------------------------------------------------------------------- diff --git a/crunch-hbase/src/main/java/org/apache/crunch/io/hbase/HFileOutputFormatForCrunch.java b/crunch-hbase/src/main/java/org/apache/crunch/io/hbase/HFileOutputFormatForCrunch.java index 9d5f6ed..ad68da4 100644 --- a/crunch-hbase/src/main/java/org/apache/crunch/io/hbase/HFileOutputFormatForCrunch.java +++ b/crunch-hbase/src/main/java/org/apache/crunch/io/hbase/HFileOutputFormatForCrunch.java @@ -30,6 +30,7 @@ import org.apache.hadoop.hbase.HColumnDescriptor; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.io.hfile.HFile; +import org.apache.hadoop.hbase.io.hfile.HFileContext; import org.apache.hadoop.hbase.io.hfile.HFileDataBlockEncoderImpl; import org.apache.hadoop.hbase.regionserver.HStore; import org.apache.hadoop.hbase.regionserver.StoreFile; @@ -89,12 +90,8 @@ public class HFileOutputFormatForCrunch extends FileOutputFormat<Object, KeyValu LOG.info("HColumnDescriptor: " + hcol.toString()); final HFile.Writer writer = HFile.getWriterFactoryNoCache(conf) .withPath(fs, outputPath) - .withBlockSize(hcol.getBlocksize()) - .withCompression(hcol.getCompression()) .withComparator(KeyValue.COMPARATOR) - .withDataBlockEncoder(new HFileDataBlockEncoderImpl(hcol.getDataBlockEncoding())) - .withChecksumType(HStore.getChecksumType(conf)) - .withBytesPerChecksum(HStore.getBytesPerChecksum(conf)) + .withFileContext(getContext(hcol)) .create(); return new RecordWriter<Object, KeyValue>() { @@ -124,4 +121,11 @@ public class HFileOutputFormatForCrunch extends FileOutputFormat<Object, KeyValu } }; } + + private HFileContext getContext(HColumnDescriptor desc) { + HFileContext ctxt = new HFileContext(); + ctxt.setDataBlockEncoding(desc.getDataBlockEncoding()); + ctxt.setCompression(desc.getCompression()); + return ctxt; + } } http://git-wip-us.apache.org/repos/asf/crunch/blob/c88ce471/crunch-hbase/src/main/java/org/apache/crunch/io/hbase/HFileReaderFactory.java ---------------------------------------------------------------------- diff --git a/crunch-hbase/src/main/java/org/apache/crunch/io/hbase/HFileReaderFactory.java b/crunch-hbase/src/main/java/org/apache/crunch/io/hbase/HFileReaderFactory.java index f5db516..6189775 100644 --- a/crunch-hbase/src/main/java/org/apache/crunch/io/hbase/HFileReaderFactory.java +++ b/crunch-hbase/src/main/java/org/apache/crunch/io/hbase/HFileReaderFactory.java @@ -39,7 +39,7 @@ public class HFileReaderFactory implements FileReaderFactory<KeyValue> { Configuration conf = fs.getConf(); CacheConfig cacheConfig = new CacheConfig(conf); try { - HFile.Reader hfr = HFile.createReader(fs, path, cacheConfig); + HFile.Reader hfr = HFile.createReader(fs, path, cacheConfig, conf); HFileScanner scanner = hfr.getScanner( conf.getBoolean(HFILE_SCANNER_CACHE_BLOCKS, false), conf.getBoolean(HFILE_SCANNER_PREAD, false)); http://git-wip-us.apache.org/repos/asf/crunch/blob/c88ce471/crunch-hbase/src/main/java/org/apache/crunch/io/hbase/HFileUtils.java ---------------------------------------------------------------------- diff --git a/crunch-hbase/src/main/java/org/apache/crunch/io/hbase/HFileUtils.java b/crunch-hbase/src/main/java/org/apache/crunch/io/hbase/HFileUtils.java index cb82dd8..63d2286 100644 --- a/crunch-hbase/src/main/java/org/apache/crunch/io/hbase/HFileUtils.java +++ b/crunch-hbase/src/main/java/org/apache/crunch/io/hbase/HFileUtils.java @@ -86,7 +86,7 @@ public final class HFileUtils { if ((cmp = compareType(l, r)) != 0) { return cmp; } - return compareMemstoreTS(l, r); + return 0; } private int compareFamily(KeyValue l, KeyValue r) { @@ -111,9 +111,6 @@ public final class HFileUtils { return (int) r.getType() - (int) l.getType(); } - private int compareMemstoreTS(KeyValue l, KeyValue r) { - return Longs.compare(l.getMemstoreTS(), r.getMemstoreTS()); - } }; private static class FilterByFamilyFn extends FilterFn<KeyValue> { http://git-wip-us.apache.org/repos/asf/crunch/blob/c88ce471/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index aafe113..1787830 100644 --- a/pom.xml +++ b/pom.xml @@ -88,7 +88,7 @@ under the License. <!-- Can be overridden by hadoop-2 profile, but these are the default values --> <hadoop.version>1.1.2</hadoop.version> - <hbase.version>0.96.0-hadoop1</hbase.version> + <hbase.version>0.98.1-hadoop1</hbase.version> <hbase.midfix>hadoop1</hbase.midfix> <scala.base.version>2.10</scala.base.version> @@ -457,7 +457,7 @@ under the License. </activation> <properties> <hadoop.version>2.2.0</hadoop.version> - <hbase.version>0.96.0-hadoop2</hbase.version> + <hbase.version>0.98.1-hadoop2</hbase.version> <commons-lang.version>2.5</commons-lang.version> <hbase.midfix>hadoop2</hbase.midfix> </properties>
