Author: jimk
Date: Sun Jun 3 01:52:06 2007
New Revision: 543862
URL: http://svn.apache.org/viewvc?view=rev&rev=543862
Log:
HADOOP-1391. Split test case for merge into two so it does not time out in
Hudson
Added:
lucene/hadoop/trunk/src/contrib/hbase/src/test/org/apache/hadoop/hbase/AbstractMergeTestBase.java
lucene/hadoop/trunk/src/contrib/hbase/src/test/org/apache/hadoop/hbase/TestMergeMeta.java
lucene/hadoop/trunk/src/contrib/hbase/src/test/org/apache/hadoop/hbase/TestMergeTable.java
Removed:
lucene/hadoop/trunk/src/contrib/hbase/src/test/org/apache/hadoop/hbase/TestMerge.java
Added:
lucene/hadoop/trunk/src/contrib/hbase/src/test/org/apache/hadoop/hbase/AbstractMergeTestBase.java
URL:
http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/contrib/hbase/src/test/org/apache/hadoop/hbase/AbstractMergeTestBase.java?view=auto&rev=543862
==============================================================================
---
lucene/hadoop/trunk/src/contrib/hbase/src/test/org/apache/hadoop/hbase/AbstractMergeTestBase.java
(added)
+++
lucene/hadoop/trunk/src/contrib/hbase/src/test/org/apache/hadoop/hbase/AbstractMergeTestBase.java
Sun Jun 3 01:52:06 2007
@@ -0,0 +1,146 @@
+/**
+ * Copyright 2007 The Apache Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase;
+
+import java.io.IOException;
+import java.io.UnsupportedEncodingException;
+import java.util.Random;
+
+import org.apache.hadoop.dfs.MiniDFSCluster;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.BytesWritable;
+import org.apache.hadoop.io.Text;
+
+/** Abstract base class for merge tests */
+public abstract class AbstractMergeTestBase extends HBaseTestCase {
+ protected static final Text COLUMN_NAME = new Text("contents:");
+ protected Random rand;
+ protected HTableDescriptor desc;
+ protected BytesWritable value;
+
+ protected MiniDFSCluster dfsCluster;
+ protected FileSystem fs;
+ protected Path dir;
+
+ public void setUp() throws Exception {
+ super.setUp();
+ rand = new Random();
+ desc = new HTableDescriptor("test");
+ desc.addFamily(new HColumnDescriptor(COLUMN_NAME.toString()));
+
+ // We will use the same value for the rows as that is not really important
here
+
+ String partialValue = String.valueOf(System.currentTimeMillis());
+ StringBuilder val = new StringBuilder();
+ while(val.length() < 1024) {
+ val.append(partialValue);
+ }
+ try {
+ value = new
BytesWritable(val.toString().getBytes(HConstants.UTF8_ENCODING));
+
+ } catch(UnsupportedEncodingException e) {
+ fail();
+ }
+
+ try {
+ dfsCluster = new MiniDFSCluster(conf, 2, true, (String[])null);
+ fs = dfsCluster.getFileSystem();
+ dir = new Path("/hbase");
+ fs.mkdirs(dir);
+
+ } catch(Throwable t) {
+ t.printStackTrace();
+ fail();
+ }
+
+ // We create three data regions: The first is too large to merge since it
+ // will be > 64 MB in size. The second two will be smaller and will be
+ // selected for merging.
+
+ // To ensure that the first region is larger than 64MB we need to write at
+ // least 65536 rows. We will make certain by writing 70000
+
+ try {
+ Text row_70001 = new Text("row_70001");
+ Text row_80001 = new Text("row_80001");
+
+ HRegion[] regions = {
+ createAregion(null, row_70001, 1, 70000),
+ createAregion(row_70001, row_80001, 70001, 10000),
+ createAregion(row_80001, null, 80001, 10000)
+ };
+
+ // Now create the root and meta regions and insert the data regions
+ // created above into the meta
+
+ HRegion root = HRegion.createNewHRegion(fs, dir, conf,
+ HGlobals.rootTableDesc, 0L, null, null);
+ HRegion meta = HRegion.createNewHRegion(fs, dir, conf,
+ HGlobals.metaTableDesc, 1L, null, null);
+
+ HRegion.addRegionToMeta(root, meta);
+
+ for(int i = 0; i < regions.length; i++) {
+ HRegion.addRegionToMeta(meta, regions[i]);
+ }
+
+ root.close();
+ root.getLog().close();
+ fs.delete(new Path(root.getRegionDir(), HConstants.HREGION_LOGDIR_NAME));
+ meta.close();
+ meta.getLog().close();
+ fs.delete(new Path(meta.getRegionDir(), HConstants.HREGION_LOGDIR_NAME));
+
+ } catch(Throwable t) {
+ t.printStackTrace();
+ fail();
+ }
+ }
+
+ public void tearDown() throws Exception {
+ super.tearDown();
+ dfsCluster.shutdown();
+ }
+
+ private HRegion createAregion(Text startKey, Text endKey, int firstRow, int
nrows)
+ throws IOException {
+ HRegion region = HRegion.createNewHRegion(fs, dir, conf, desc,
+ rand.nextLong(), startKey, endKey);
+
+ System.out.println("created region " + region.getRegionName());
+
+ for(int i = firstRow; i < firstRow + nrows; i++) {
+ long lockid = region.startUpdate(new Text("row_"
+ + String.format("%1$05d", i)));
+
+ region.put(lockid, COLUMN_NAME, value);
+ region.commit(lockid);
+ if(i % 10000 == 0) {
+ System.out.println("Flushing write #" + i);
+ region.flushcache(false);
+ }
+ }
+ System.out.println("Rolling log...");
+ region.log.rollWriter();
+ region.compactStores();
+ region.close();
+ region.getLog().close();
+ fs.delete(new Path(region.getRegionDir(), HConstants.HREGION_LOGDIR_NAME));
+ region.getRegionInfo().offLine = true;
+ return region;
+ }
+}
Added:
lucene/hadoop/trunk/src/contrib/hbase/src/test/org/apache/hadoop/hbase/TestMergeMeta.java
URL:
http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/contrib/hbase/src/test/org/apache/hadoop/hbase/TestMergeMeta.java?view=auto&rev=543862
==============================================================================
---
lucene/hadoop/trunk/src/contrib/hbase/src/test/org/apache/hadoop/hbase/TestMergeMeta.java
(added)
+++
lucene/hadoop/trunk/src/contrib/hbase/src/test/org/apache/hadoop/hbase/TestMergeMeta.java
Sun Jun 3 01:52:06 2007
@@ -0,0 +1,30 @@
+/**
+ * Copyright 2007 The Apache Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase;
+
+/** Tests region merging */
+public class TestMergeMeta extends AbstractMergeTestBase {
+
+ public void testMergeMeta() {
+ try {
+ HMerge.merge(conf, fs, HConstants.META_TABLE_NAME);
+
+ } catch(Throwable t) {
+ t.printStackTrace();
+ fail();
+ }
+ }
+}
Added:
lucene/hadoop/trunk/src/contrib/hbase/src/test/org/apache/hadoop/hbase/TestMergeTable.java
URL:
http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/contrib/hbase/src/test/org/apache/hadoop/hbase/TestMergeTable.java?view=auto&rev=543862
==============================================================================
---
lucene/hadoop/trunk/src/contrib/hbase/src/test/org/apache/hadoop/hbase/TestMergeTable.java
(added)
+++
lucene/hadoop/trunk/src/contrib/hbase/src/test/org/apache/hadoop/hbase/TestMergeTable.java
Sun Jun 3 01:52:06 2007
@@ -0,0 +1,35 @@
+/**
+ * Copyright 2007 The Apache Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase;
+
+public class TestMergeTable extends AbstractMergeTestBase {
+
+ public void testMergeTable() {
+ try {
+ MiniHBaseCluster hCluster = new MiniHBaseCluster(conf, 1, dfsCluster);
+ try {
+ HMerge.merge(conf, fs, desc.getName());
+
+ } finally {
+ hCluster.shutdown();
+ }
+
+ } catch(Throwable t) {
+ t.printStackTrace();
+ fail();
+ }
+ }
+}
