Author: stack
Date: Tue Nov 24 20:10:16 2009
New Revision: 883845
URL: http://svn.apache.org/viewvc?rev=883845&view=rev
Log:
HBASE-1970 Export does one version only; make it configurable how many it does
Modified:
hadoop/hbase/trunk/CHANGES.txt
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/client/Scan.java
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/mapreduce/Export.java
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/mapreduce/TableInputFormatBase.java
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java
hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/regionserver/TestStoreScanner.java
Modified: hadoop/hbase/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/hbase/trunk/CHANGES.txt?rev=883845&r1=883844&r2=883845&view=diff
==============================================================================
--- hadoop/hbase/trunk/CHANGES.txt (original)
+++ hadoop/hbase/trunk/CHANGES.txt Tue Nov 24 20:10:16 2009
@@ -198,6 +198,8 @@
(Jeremiah Jacquet via Stack)
HBASE-1961 HBase EC2 scripts
HBASE-1971 Unit test the full WAL replay cycle
+ HBASE-1970 Export does one version only; make it configurable how many
+ it does
NEW FEATURES
HBASE-1901 "General" partitioner for "hbase-48" bulk (behind the api, write
Modified: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/client/Scan.java
URL:
http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/client/Scan.java?rev=883845&r1=883844&r2=883845&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/client/Scan.java
(original)
+++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/client/Scan.java Tue
Nov 24 20:10:16 2009
@@ -76,7 +76,6 @@
*/
public class Scan implements Writable {
private static final byte SCAN_VERSION = (byte)1;
-
private byte [] startRow = HConstants.EMPTY_START_ROW;
private byte [] stopRow = HConstants.EMPTY_END_ROW;
private int maxVersions = 1;
@@ -180,10 +179,14 @@
/**
* Get versions of columns only within the specified timestamp range,
- * [minStamp, maxStamp).
+ * [minStamp, maxStamp). Note, default maximum versions to return is 1. If
+ * your time range spans more than one version and you want all versions
+ * returned, up the number of versions beyond the defaut.
* @param minStamp minimum timestamp value, inclusive
* @param maxStamp maximum timestamp value, exclusive
* @throws IOException if invalid time range
+ * @see {...@link #setMaxVersions()}
+ * @see {...@link #setMaxVersions(int)}
*/
public Scan setTimeRange(long minStamp, long maxStamp)
throws IOException {
@@ -192,8 +195,13 @@
}
/**
- * Get versions of columns with the specified timestamp.
- * @param timestamp version timestamp
+ * Get versions of columns with the specified timestamp. Note, default
maximum
+ * versions to return is 1. If your time range spans more than one version
+ * and you want all versions returned, up the number of versions beyond the
+ * defaut.
+ * @param timestamp version timestamp
+ * @see {...@link #setMaxVersions()}
+ * @see {...@link #setMaxVersions(int)}
*/
public Scan setTimeStamp(long timestamp) {
try {
Modified:
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/mapreduce/Export.java
URL:
http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/mapreduce/Export.java?rev=883845&r1=883844&r2=883845&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/mapreduce/Export.java
(original)
+++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/mapreduce/Export.java
Tue Nov 24 20:10:16 2009
@@ -31,6 +31,7 @@
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat;
import org.apache.hadoop.util.GenericOptionsParser;
+import org.mortbay.log.Log;
/**
* Export an HBase table.
@@ -80,8 +81,17 @@
Job job = new Job(conf, NAME + "_" + tableName);
job.setJarByClass(Exporter.class);
// TODO: Allow passing filter and subset of rows/columns.
- TableMapReduceUtil.initTableMapperJob(tableName, new Scan(),
- Exporter.class, null, null, job);
+ Scan s = new Scan();
+ // Optional arguments.
+ int versions = args.length > 2? Integer.parseInt(args[2]): 1;
+ s.setMaxVersions(versions);
+ long startTime = args.length > 3? Long.parseLong(args[3]): 0L;
+ long endTime = args.length > 4? Long.parseLong(args[4]): Long.MAX_VALUE;
+ s.setTimeRange(startTime, endTime);
+ Log.info("verisons=" + versions + ", starttime=" + startTime +
+ ", endtime=" + endTime);
+ TableMapReduceUtil.initTableMapperJob(tableName, s, Exporter.class, null,
+ null, job);
// No reducers. Just write straight to output files.
job.setNumReduceTasks(0);
job.setOutputFormatClass(SequenceFileOutputFormat.class);
@@ -98,7 +108,8 @@
if (errorMsg != null && errorMsg.length() > 0) {
System.err.println("ERROR: " + errorMsg);
}
- System.err.println("Usage: Export <tablename> <outputdir>");
+ System.err.println("Usage: Export <tablename> <outputdir> [<versions> " +
+ "[<starttime> [<endtime>]]]");
}
/**
@@ -115,6 +126,6 @@
System.exit(-1);
}
Job job = createSubmittableJob(conf, otherArgs);
- System.exit(job.waitForCompletion(true) ? 0 : 1);
+ System.exit(job.waitForCompletion(true)? 0 : 1);
}
-}
\ No newline at end of file
+}
Modified:
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/mapreduce/TableInputFormatBase.java
URL:
http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/mapreduce/TableInputFormatBase.java?rev=883845&r1=883844&r2=883845&view=diff
==============================================================================
---
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/mapreduce/TableInputFormatBase.java
(original)
+++
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/mapreduce/TableInputFormatBase.java
Tue Nov 24 20:10:16 2009
@@ -102,7 +102,7 @@
public void restart(byte[] firstRow) throws IOException {
Scan newScan = new Scan(scan);
newScan.setStartRow(firstRow);
- this.scanner = this.htable.getScanner(newScan);
+ this.scanner = this.htable.getScanner(newScan);
}
/**
Modified:
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java
URL:
http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java?rev=883845&r1=883844&r2=883845&view=diff
==============================================================================
---
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java
(original)
+++
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java
Tue Nov 24 20:10:16 2009
@@ -1079,7 +1079,6 @@
SequenceFile.Reader in = null;
int count = 0;
try {
- long len = fs.getFileStatus(logfiles[i].getPath()).getLen();
in = HLog.getReader(fs, logfiles[i].getPath(), conf);
try {
HLogKey key = newKey(conf);
Modified:
hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/regionserver/TestStoreScanner.java
URL:
http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/regionserver/TestStoreScanner.java?rev=883845&r1=883844&r2=883845&view=diff
==============================================================================
---
hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/regionserver/TestStoreScanner.java
(original)
+++
hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/regionserver/TestStoreScanner.java
Tue Nov 24 20:10:16 2009
@@ -34,8 +34,8 @@
import org.apache.hadoop.hbase.util.Bytes;
public class TestStoreScanner extends TestCase {
-
- final byte [] CF = Bytes.toBytes("cf");
+ private final String CF_STR = "cf";
+ final byte [] CF = Bytes.toBytes(CF_STR);
/**
* Test utility for building a NavigableSet for scanners.
@@ -50,6 +50,60 @@
}
return cols;
}
+
+ public void testScanTimeRange() throws IOException {
+ String r1 = "R1";
+ // returns only 1 of these 2 even though same timestamp
+ KeyValue [] kvs = new KeyValue[] {
+ KeyValueTestUtil.create(r1, CF_STR, "a", 1, KeyValue.Type.Put,
"dont-care"),
+ KeyValueTestUtil.create(r1, CF_STR, "a", 2, KeyValue.Type.Put,
"dont-care"),
+ KeyValueTestUtil.create(r1, CF_STR, "a", 3, KeyValue.Type.Put,
"dont-care"),
+ KeyValueTestUtil.create(r1, CF_STR, "a", 4, KeyValue.Type.Put,
"dont-care"),
+ KeyValueTestUtil.create(r1, CF_STR, "a", 5, KeyValue.Type.Put,
"dont-care"),
+ };
+ KeyValueScanner [] scanners = new KeyValueScanner[] {
+ new KeyValueScanFixture(KeyValue.COMPARATOR, kvs)
+ };
+ Scan scanSpec = new Scan(Bytes.toBytes(r1));
+ scanSpec.setTimeRange(0, 6);
+ scanSpec.setMaxVersions();
+ StoreScanner scan =
+ new StoreScanner(scanSpec, CF, Long.MAX_VALUE,
+ KeyValue.COMPARATOR, getCols("a"), scanners);
+ List<KeyValue> results = new ArrayList<KeyValue>();
+ assertEquals(true, scan.next(results));
+ assertEquals(5, results.size());
+ assertEquals(kvs[kvs.length - 1], results.get(0));
+ // Scan limited TimeRange
+ scanSpec = new Scan(Bytes.toBytes(r1));
+ scanSpec.setTimeRange(1, 3);
+ scanSpec.setMaxVersions();
+ scan = new StoreScanner(scanSpec, CF, Long.MAX_VALUE,
+ KeyValue.COMPARATOR, getCols("a"), scanners);
+ results = new ArrayList<KeyValue>();
+ assertEquals(true, scan.next(results));
+ assertEquals(2, results.size());
+ // Another range.
+ scanSpec = new Scan(Bytes.toBytes(r1));
+ scanSpec.setTimeRange(5, 10);
+ scanSpec.setMaxVersions();
+ scan = new StoreScanner(scanSpec, CF, Long.MAX_VALUE,
+ KeyValue.COMPARATOR, getCols("a"), scanners);
+ results = new ArrayList<KeyValue>();
+ assertEquals(true, scan.next(results));
+ assertEquals(1, results.size());
+ // See how TimeRange and Versions interact.
+ // Another range.
+ scanSpec = new Scan(Bytes.toBytes(r1));
+ scanSpec.setTimeRange(0, 10);
+ scanSpec.setMaxVersions(3);
+ scan = new StoreScanner(scanSpec, CF, Long.MAX_VALUE,
+ KeyValue.COMPARATOR, getCols("a"), scanners);
+ results = new ArrayList<KeyValue>();
+ assertEquals(true, scan.next(results));
+ assertEquals(3, results.size());
+
+ }
public void testScanSameTimestamp() throws IOException {
// returns only 1 of these 2 even though same timestamp
@@ -58,8 +112,7 @@
KeyValueTestUtil.create("R1", "cf", "a", 1, KeyValue.Type.Put,
"dont-care"),
};
KeyValueScanner [] scanners = new KeyValueScanner[] {
- new KeyValueScanFixture(KeyValue.COMPARATOR,
- kvs)
+ new KeyValueScanFixture(KeyValue.COMPARATOR, kvs)
};
Scan scanSpec = new Scan(Bytes.toBytes("R1"));
