Author: apurtell
Date: Wed Sep 15 02:05:22 2010
New Revision: 997169
URL: http://svn.apache.org/viewvc?rev=997169&view=rev
Log:
HBASE-2988 Support alternate compression for major compactions
Modified:
hbase/branches/0.20/CHANGES.txt
hbase/branches/0.20/bin/HBase.rb
hbase/branches/0.20/src/java/org/apache/hadoop/hbase/HColumnDescriptor.java
hbase/branches/0.20/src/java/org/apache/hadoop/hbase/regionserver/Store.java
Modified: hbase/branches/0.20/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hbase/branches/0.20/CHANGES.txt?rev=997169&r1=997168&r2=997169&view=diff
==============================================================================
--- hbase/branches/0.20/CHANGES.txt (original)
+++ hbase/branches/0.20/CHANGES.txt Wed Sep 15 02:05:22 2010
@@ -9,6 +9,7 @@ Release 0.20.7 - Unreleased
HBaseObjectWritable (Gary Helmling via Andrew Purtell)
IMPROVEMENTS
+ HBASE-2988 Support alternate compression for major compactions
Release 0.20.6 - July 30, 2010
BUG FIXES
Modified: hbase/branches/0.20/bin/HBase.rb
URL:
http://svn.apache.org/viewvc/hbase/branches/0.20/bin/HBase.rb?rev=997169&r1=997168&r2=997169&view=diff
==============================================================================
--- hbase/branches/0.20/bin/HBase.rb (original)
+++ hbase/branches/0.20/bin/HBase.rb Wed Sep 15 02:05:22 2010
@@ -225,7 +225,11 @@ module HBase
else
raise TypeError.new(arg.class.to_s + " of " + arg.to_s + " is not of
Hash type") \
unless arg.instance_of? Hash
- htd.addFamily(hcd(arg))
+ descriptor = hcd(arg)
+ if arg[COMPRESSION_COMPACT]
+ descriptor.setValue(COMPRESSION_COMPACT, arg[COMPRESSION_COMPACT])
+ end
+ htd.addFamily(descriptor)
end
end
@admin.createTable(htd)
@@ -253,7 +257,10 @@ module HBase
end
@admin.modifyTable(tableName.to_java_bytes, htd)
else
- descriptor = hcd(args)
+ descriptor = hcd(args)
+ if args[COMPRESSION_COMPACT]
+ descriptor.setValue(COMPRESSION_COMPACT, args[COMPRESSION_COMPACT])
+ end
if (htd.hasFamily(descriptor.getNameAsString().to_java_bytes))
@admin.modifyColumn(tableName, descriptor.getNameAsString(),
descriptor);
Modified:
hbase/branches/0.20/src/java/org/apache/hadoop/hbase/HColumnDescriptor.java
URL:
http://svn.apache.org/viewvc/hbase/branches/0.20/src/java/org/apache/hadoop/hbase/HColumnDescriptor.java?rev=997169&r1=997168&r2=997169&view=diff
==============================================================================
--- hbase/branches/0.20/src/java/org/apache/hadoop/hbase/HColumnDescriptor.java
(original)
+++ hbase/branches/0.20/src/java/org/apache/hadoop/hbase/HColumnDescriptor.java
Wed Sep 15 02:05:22 2010
@@ -72,6 +72,7 @@ public class HColumnDescriptor implement
}
public static final String COMPRESSION = "COMPRESSION";
+ public static final String COMPRESSION_COMPACT = "COMPRESSION_COMPACT";
public static final String BLOCKCACHE = "BLOCKCACHE";
public static final String BLOCKSIZE = "BLOCKSIZE";
public static final String LENGTH = "LENGTH";
@@ -396,9 +397,22 @@ public class HColumnDescriptor implement
/** @return compression type being used for the column family */
public Compression.Algorithm getCompression() {
String n = getValue(COMPRESSION);
+ if (n == null) {
+ return Compression.Algorithm.NONE;
+ }
return Compression.Algorithm.valueOf(n.toUpperCase());
}
-
+
+ /** @return compression type being used for the column family for major
+ compression */
+ public Compression.Algorithm getCompactionCompression() {
+ String n = getValue(COMPRESSION_COMPACT);
+ if (n == null) {
+ return getCompression();
+ }
+ return Compression.Algorithm.valueOf(n.toUpperCase());
+ }
+
/** @return maximum number of versions */
public synchronized int getMaxVersions() {
if (this.cachedMaxVersions == -1) {
@@ -461,6 +475,30 @@ public class HColumnDescriptor implement
}
/**
+ * @return Compression type setting.
+ */
+ public Compression.Algorithm getCompactionCompressionType() {
+ return getCompactionCompression();
+ }
+
+ /**
+ * Compression types supported in hbase.
+ * LZO is not bundled as part of the hbase distribution.
+ * See <a href="http://wiki.apache.org/hadoop/UsingLzoCompression";>LZO
Compression</a>
+ * for how to enable it.
+ * @param type Compression type setting.
+ */
+ public void setCompactionCompressionType(Compression.Algorithm type) {
+ String compressionType;
+ switch (type) {
+ case LZO: compressionType = "LZO"; break;
+ case GZ: compressionType = "GZ"; break;
+ default: compressionType = "NONE"; break;
+ }
+ setValue(COMPRESSION_COMPACT, compressionType);
+ }
+
+ /**
* @return True if we are to keep all in use HRegionServer cache.
*/
public boolean isInMemory() {
Modified:
hbase/branches/0.20/src/java/org/apache/hadoop/hbase/regionserver/Store.java
URL:
http://svn.apache.org/viewvc/hbase/branches/0.20/src/java/org/apache/hadoop/hbase/regionserver/Store.java?rev=997169&r1=997168&r2=997169&view=diff
==============================================================================
---
hbase/branches/0.20/src/java/org/apache/hadoop/hbase/regionserver/Store.java
(original)
+++
hbase/branches/0.20/src/java/org/apache/hadoop/hbase/regionserver/Store.java
Wed Sep 15 02:05:22 2010
@@ -138,7 +138,10 @@ public class Store implements HConstants
private final int compactionThreshold;
private final int blocksize;
private final boolean blockcache;
+ /** Compression algorithm for flush files and minor compaction */
private final Compression.Algorithm compression;
+ /** Compression algorithm for major compaction */
+ private final Compression.Algorithm compactionCompression;
// Comparing KeyValues
final KeyValue.KVComparator comparator;
@@ -172,6 +175,11 @@ public class Store implements HConstants
this.blockcache = family.isBlockCacheEnabled();
this.blocksize = family.getBlocksize();
this.compression = family.getCompression();
+ // avoid overriding compression setting for major compactions if the user
+ // has not specified it separately
+ this.compactionCompression =
+ (family.getCompactionCompression() != Compression.Algorithm.NONE) ?
+ family.getCompactionCompression() : this.compression;
this.comparator = info.getComparator();
this.comparatorIgnoringType = this.comparator.getComparatorIgnoringType();
// getTimeToLive returns ttl in seconds. Convert to milliseconds.
@@ -587,6 +595,18 @@ public class Store implements HConstants
}
/*
+ * @return Writer for this store.
+ * @param basedir Directory to put writer in.
+ * @param compression Compression algorithm.
+ * @throws IOException
+ */
+ private HFile.Writer getWriter(final Path basedir,
+ Compression.Algorithm compression) throws IOException {
+ return StoreFile.getWriter(this.fs, basedir, blocksize, compression,
+ this.comparator.getRawComparator());
+ }
+
+ /*
* Change storefiles adding into place the Reader produced by this new flush.
* @param logCacheFlushId
* @param sf
@@ -903,7 +923,8 @@ public class Store implements HConstants
// output to writer:
for (KeyValue kv : kvs) {
if (writer == null) {
- writer = getWriter(this.regionCompactionDir);
+ writer = getWriter(this.regionCompactionDir,
+ this.compactionCompression);
}
writer.append(kv);
}
