Updated Branches: refs/heads/master d93ed7645 -> 3ff140357
Fixed https://issues.apache.org/jira/browse/BLUR-13 . Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/3ff14035 Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/3ff14035 Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/3ff14035 Branch: refs/heads/master Commit: 3ff1403572df4e49f0c9376b38f7ebeeefc76cb4 Parents: d93ed76 Author: Aaron McCurry <[email protected]> Authored: Mon Sep 24 17:35:04 2012 -0400 Committer: Aaron McCurry <[email protected]> Committed: Mon Sep 24 17:35:04 2012 -0400 ---------------------------------------------------------------------- .../main/java/org/apache/blur/utils/BlurUtil.java | 25 ++++- .../java/org/apache/blur/utils/BlurUtilsTest.java | 88 ++++++++++++++- 2 files changed, 107 insertions(+), 6 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/3ff14035/src/blur-core/src/main/java/org/apache/blur/utils/BlurUtil.java ---------------------------------------------------------------------- diff --git a/src/blur-core/src/main/java/org/apache/blur/utils/BlurUtil.java b/src/blur-core/src/main/java/org/apache/blur/utils/BlurUtil.java index 7a8096e..e0b4b9d 100644 --- a/src/blur-core/src/main/java/org/apache/blur/utils/BlurUtil.java +++ b/src/blur-core/src/main/java/org/apache/blur/utils/BlurUtil.java @@ -89,7 +89,6 @@ import org.apache.zookeeper.Watcher; import org.apache.zookeeper.ZooDefs.Ids; import org.apache.zookeeper.ZooKeeper; - public class BlurUtil { private static final Object[] EMPTY_OBJECT_ARRAY = new Object[] {}; @@ -641,10 +640,28 @@ public class BlurUtil { } public static void validateShardCount(int shardCount, FileSystem fileSystem, Path tablePath) throws IOException { + // Check that all the directories that should be are in fact there. + for (int i = 0; i < shardCount; i++) { + Path path = new Path(tablePath, BlurUtil.getShardName(BlurConstants.SHARD_PREFIX, i)); + if (!fileSystem.exists(path)) { + LOG.error("Path [{0}] for shard [{1}] does not exist.", path, i); + throw new RuntimeException("Path [" + path + "] for shard [" + i + "] does not exist."); + } + } + FileStatus[] listStatus = fileSystem.listStatus(tablePath); - if (listStatus.length != shardCount) { - LOG.error("Number of directories in table path [" + tablePath + "] does not match definition of [" + shardCount + "] shard count."); - throw new RuntimeException("Number of directories in table path [" + tablePath + "] does not match definition of [" + shardCount + "] shard count."); + for (FileStatus fs : listStatus) { + Path path = fs.getPath(); + String name = path.getName(); + if (name.startsWith(SHARD_PREFIX)) { + int index = name.indexOf('-'); + String shardIndexStr = name.substring(index+1); + int shardIndex = Integer.parseInt(shardIndexStr); + if (shardIndex >= shardCount) { + LOG.error("Number of directories in table path [" + path + "] exceeds definition of [" + shardCount + "] shard count."); + throw new RuntimeException("Number of directories in table path [" + path + "] exceeds definition of [" + shardCount + "] shard count."); + } + } } } http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/3ff14035/src/blur-core/src/test/java/org/apache/blur/utils/BlurUtilsTest.java ---------------------------------------------------------------------- diff --git a/src/blur-core/src/test/java/org/apache/blur/utils/BlurUtilsTest.java b/src/blur-core/src/test/java/org/apache/blur/utils/BlurUtilsTest.java index 36a8dde..9cba2a6 100644 --- a/src/blur-core/src/test/java/org/apache/blur/utils/BlurUtilsTest.java +++ b/src/blur-core/src/test/java/org/apache/blur/utils/BlurUtilsTest.java @@ -21,7 +21,9 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; +import java.io.File; import java.io.IOException; import java.util.concurrent.TimeUnit; @@ -29,7 +31,9 @@ import org.apache.blur.lucene.LuceneConstant; import org.apache.blur.thrift.generated.Record; import org.apache.blur.thrift.generated.RecordMutation; import org.apache.blur.thrift.generated.RowMutation; -import org.apache.blur.utils.BlurUtil; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; import org.apache.lucene.analysis.KeywordAnalyzer; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; @@ -43,7 +47,6 @@ import org.apache.lucene.store.LockObtainFailedException; import org.apache.lucene.store.RAMDirectory; import org.junit.Test; - public class BlurUtilsTest { @Test @@ -132,6 +135,87 @@ public class BlurUtilsTest { assertNull("should not find record-99", BlurUtil.findRecordMutation(row, r2)); } + @Test + public void testValidateShardCount() throws IOException { + File file = new File("./tmp/ValidateShardCount-test"); + rm(file); + Path path = new Path(file.toURI()); + Configuration conf = new Configuration(); + FileSystem fileSystem = path.getFileSystem(conf); + fileSystem.mkdirs(path); + int shardCount = 10; + createShardDirs(shardCount, fileSystem, path); + BlurUtil.validateShardCount(shardCount, fileSystem, path); + } + + @Test + public void testValidateShardCountExtraDir() throws IOException { + File file = new File("./tmp/ValidateShardCount-test"); + rm(file); + Path path = new Path(file.toURI()); + Configuration conf = new Configuration(); + FileSystem fileSystem = path.getFileSystem(conf); + fileSystem.mkdirs(path); + int shardCount = 10; + createShardDirs(shardCount, fileSystem, path); + fileSystem.mkdirs(new Path(path, "logs")); + BlurUtil.validateShardCount(shardCount, fileSystem, path); + } + + @Test + public void testValidateShardCountTooFew() throws IOException { + File file = new File("./tmp/ValidateShardCount-test"); + rm(file); + Path path = new Path(file.toURI()); + Configuration conf = new Configuration(); + FileSystem fileSystem = path.getFileSystem(conf); + fileSystem.mkdirs(path); + int shardCount = 10; + createShardDirs(shardCount - 1, fileSystem, path); + try { + BlurUtil.validateShardCount(shardCount, fileSystem, path); + fail(); + } catch (Exception e) { + // Should throw exception + } + } + + @Test + public void testValidateShardCountTooMany() throws IOException { + File file = new File("./tmp/ValidateShardCount-test"); + rm(file); + Path path = new Path(file.toURI()); + Configuration conf = new Configuration(); + FileSystem fileSystem = path.getFileSystem(conf); + fileSystem.mkdirs(path); + int shardCount = 10; + createShardDirs(shardCount + 1, fileSystem, path); + try { + BlurUtil.validateShardCount(shardCount, fileSystem, path); + fail(); + } catch (Exception e) { + // Should throw exception + } + } + + private void rm(File file) { + if (!file.exists()) { + return; + } + if (file.isDirectory()) { + for (File f : file.listFiles()) { + rm(f); + } + } + file.delete(); + } + + private void createShardDirs(int shardCount, FileSystem fileSystem, Path path) throws IOException { + for (int i = 0; i < shardCount; i++) { + fileSystem.mkdirs(new Path(path, BlurUtil.getShardName(BlurConstants.SHARD_PREFIX, i))); + } + } + private IndexReader getReader() throws CorruptIndexException, LockObtainFailedException, IOException { RAMDirectory directory = new RAMDirectory(); IndexWriterConfig conf = new IndexWriterConfig(LuceneConstant.LUCENE_VERSION, new KeywordAnalyzer());
