Some more refactoring and adding more tests for the CacheDirectory.
Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/890be501 Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/890be501 Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/890be501 Branch: refs/heads/apache-blur-0.2 Commit: 890be5016797d91d1d7e685c584307924e3c7def Parents: 28be85a Author: Aaron McCurry <[email protected]> Authored: Fri Sep 20 14:08:52 2013 -0400 Committer: Aaron McCurry <[email protected]> Committed: Fri Sep 20 14:08:52 2013 -0400 ---------------------------------------------------------------------- .../cachevalue/UnsafeCacheValue.java | 2 +- .../apache/blur/store/BaseDirectoryTest.java | 273 -------------- .../blur/store/BaseDirectoryTestSuite.java | 375 +++++++++++++++++++ .../blur/store/CacheDirectoryTestSuite.java | 72 ++++ .../apache/blur/store/HdfsDirectoryTest.java | 41 -- .../blur/store/HdfsDirectoryTestSuite.java | 41 ++ .../blur/store/SoftlinkHdfsDirectoryTest.java | 43 --- .../store/SoftlinkHdfsDirectoryTestSuite.java | 43 +++ .../blur/store/blockcache/BlockCacheTest.java | 4 +- 9 files changed, 535 insertions(+), 359 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/890be501/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/cachevalue/UnsafeCacheValue.java ---------------------------------------------------------------------- diff --git a/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/cachevalue/UnsafeCacheValue.java b/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/cachevalue/UnsafeCacheValue.java index 46af604..d8e1cad 100644 --- a/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/cachevalue/UnsafeCacheValue.java +++ b/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/cachevalue/UnsafeCacheValue.java @@ -24,7 +24,7 @@ import sun.misc.Unsafe; @SuppressWarnings("serial") public class UnsafeCacheValue extends BaseCacheValue { - private static final int MINIMUM_SIZE = 1024; +// private static final int MINIMUM_SIZE = 1024; private static final Unsafe _unsafe; http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/890be501/blur-store/src/test/java/org/apache/blur/store/BaseDirectoryTest.java ---------------------------------------------------------------------- diff --git a/blur-store/src/test/java/org/apache/blur/store/BaseDirectoryTest.java b/blur-store/src/test/java/org/apache/blur/store/BaseDirectoryTest.java deleted file mode 100644 index 666dcfe..0000000 --- a/blur-store/src/test/java/org/apache/blur/store/BaseDirectoryTest.java +++ /dev/null @@ -1,273 +0,0 @@ -package org.apache.blur.store; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; - -import java.io.File; -import java.io.IOException; -import java.util.HashSet; -import java.util.Random; -import java.util.Set; - -import org.apache.blur.lucene.LuceneVersionConstant; -import org.apache.blur.store.buffer.BufferStore; -import org.apache.lucene.analysis.core.KeywordAnalyzer; -import org.apache.lucene.document.Document; -import org.apache.lucene.document.Field.Store; -import org.apache.lucene.document.IntField; -import org.apache.lucene.index.DirectoryReader; -import org.apache.lucene.index.IndexWriter; -import org.apache.lucene.index.IndexWriterConfig; -import org.apache.lucene.search.IndexSearcher; -import org.apache.lucene.search.NumericRangeQuery; -import org.apache.lucene.search.TopDocs; -import org.apache.lucene.store.Directory; -import org.apache.lucene.store.IOContext; -import org.apache.lucene.store.IndexInput; -import org.apache.lucene.store.IndexOutput; -import org.apache.lucene.store.RAMDirectory; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; - -public abstract class BaseDirectoryTest { - protected static final File TMPDIR = new File(System.getProperty("blur.tmp.dir", "/tmp")); - - protected static final int MAX_NUMBER_OF_WRITES = 10000; - protected static final int MIN_FILE_SIZE = 100; - protected static final int MAX_FILE_SIZE = 100000; - protected static final int MIN_BUFFER_SIZE = 1; - protected static final int MAX_BUFFER_SIZE = 5000; - protected static final int MAX_NUMBER_OF_READS = 10000; - protected Directory directory; - protected File file; - protected long seed; - protected Random random; - - @Before - public void setUp() throws IOException { - BufferStore.init(128, 128); - file = new File(TMPDIR, "hdfsdirectorytest"); - rm(file); - seed = new Random().nextLong(); - random = new Random(seed); - setupDirectory(); - } - - @After - public void tearDown() { - print(file, ""); - } - - private void print(File f, String buf) { - if (f.isDirectory()) { - System.out.println(buf + "\\" + f.getName()); - for (File fl : f.listFiles()) { - if (fl.getName().startsWith(".")) { - continue; - } - print(fl, buf + " "); - } - } else { - System.out.println(buf + f.getName() + " " + f.length()); - } - } - - protected abstract void setupDirectory() throws IOException; - - @Test - public void testWritingAndReadingAFile() throws IOException { - - IndexOutput output = directory.createOutput("testing.test", IOContext.DEFAULT); - output.writeInt(12345); - output.flush(); - output.close(); - - IndexInput input = directory.openInput("testing.test", IOContext.DEFAULT); - assertEquals(12345, input.readInt()); - input.close(); - - String[] listAll = directory.listAll(); - assertEquals(1, listAll.length); - assertEquals("testing.test", listAll[0]); - - assertEquals(4, directory.fileLength("testing.test")); - - IndexInput input1 = directory.openInput("testing.test", IOContext.DEFAULT); - - IndexInput input2 = (IndexInput) input1.clone(); - assertEquals(12345, input2.readInt()); - input2.close(); - - assertEquals(12345, input1.readInt()); - input1.close(); - - assertFalse(directory.fileExists("testing.test.other")); - assertTrue(directory.fileExists("testing.test")); - directory.deleteFile("testing.test"); - assertFalse(directory.fileExists("testing.test")); - } - - @Test - public void testEOF() throws IOException { - Directory fsDir = new RAMDirectory(); - String name = "test.eof"; - createFile(name, fsDir, directory); - long fsLength = fsDir.fileLength(name); - long hdfsLength = directory.fileLength(name); - assertEquals(fsLength, hdfsLength); - testEof(name, fsDir, fsLength); - testEof(name, directory, hdfsLength); - } - - private void testEof(String name, Directory directory, long length) throws IOException { - IndexInput input = directory.openInput(name, IOContext.DEFAULT); - input.seek(length); - try { - input.readByte(); - fail("should throw eof"); - } catch (IOException e) { - } - } - - @Test - public void testWrites() throws IOException { - int i = 0; - try { - Set<String> names = new HashSet<String>(); - for (; i < 10; i++) { - Directory fsDir = new RAMDirectory(); - String name = getName(); - System.out.println("Working on pass [" + i + "] seed [" + seed + "] contains [" + names.contains(name) + "]"); - names.add(name); - createFile(name, fsDir, directory); - assertInputsEquals(name, fsDir, directory); - fsDir.close(); - } - } catch (Exception e) { - e.printStackTrace(); - fail("Test failed with seed [" + seed + "] on pass [" + i + "]"); - } - } - - @Test - public void testCreateIndex() throws IOException { - IndexWriterConfig conf = new IndexWriterConfig(LuceneVersionConstant.LUCENE_VERSION, new KeywordAnalyzer()); - IndexWriter writer = new IndexWriter(directory, conf); - int numDocs = 1000; - DirectoryReader reader = null; - for (int i = 0; i < 100; i++) { - if (reader == null) { - reader = DirectoryReader.open(writer, true); - } else { - DirectoryReader old = reader; - reader = DirectoryReader.openIfChanged(old, writer, true); - if (reader == null) { - reader = old; - } else { - old.close(); - } - } - assertEquals(i * numDocs, reader.numDocs()); - IndexSearcher searcher = new IndexSearcher(reader); - NumericRangeQuery<Integer> query = NumericRangeQuery.newIntRange("id", 42, 42, true, true); - TopDocs topDocs = searcher.search(query, 10); - assertEquals(i, topDocs.totalHits); - addDocuments(writer, numDocs); - } - writer.close(false); - reader.close(); - } - - private void addDocuments(IndexWriter writer, int numDocs) throws IOException { - for (int i = 0; i < numDocs; i++) { - writer.addDocument(getDoc(i)); - } - } - - private Document getDoc(int i) { - Document document = new Document(); - document.add(new IntField("id", i, Store.YES)); - return document; - } - - private void assertInputsEquals(String name, Directory fsDir, Directory hdfs) throws IOException { - int reads = random.nextInt(MAX_NUMBER_OF_READS); - IndexInput fsInput = fsDir.openInput(name, IOContext.DEFAULT); - IndexInput hdfsInput = hdfs.openInput(name, IOContext.DEFAULT); - assertEquals(fsInput.length(), hdfsInput.length()); - int fileLength = (int) fsInput.length(); - for (int i = 0; i < reads; i++) { - byte[] fsBuf = new byte[random.nextInt(Math.min(MAX_BUFFER_SIZE - MIN_BUFFER_SIZE, fileLength)) + MIN_BUFFER_SIZE]; - byte[] hdfsBuf = new byte[fsBuf.length]; - int offset = random.nextInt(fsBuf.length); - int length = random.nextInt(fsBuf.length - offset); - int pos = random.nextInt(fileLength - length); - fsInput.seek(pos); - fsInput.readBytes(fsBuf, offset, length); - hdfsInput.seek(pos); - hdfsInput.readBytes(hdfsBuf, offset, length); - for (int f = offset; f < length; f++) { - if (fsBuf[f] != hdfsBuf[f]) { - fail(); - } - } - } - fsInput.close(); - hdfsInput.close(); - } - - private void createFile(String name, Directory fsDir, Directory hdfs) throws IOException { - int writes = random.nextInt(MAX_NUMBER_OF_WRITES); - int fileLength = random.nextInt(MAX_FILE_SIZE - MIN_FILE_SIZE) + MIN_FILE_SIZE; - IndexOutput fsOutput = fsDir.createOutput(name, IOContext.DEFAULT); - fsOutput.setLength(fileLength); - IndexOutput hdfsOutput = hdfs.createOutput(name, IOContext.DEFAULT); - hdfsOutput.setLength(fileLength); - for (int i = 0; i < writes; i++) { - byte[] buf = new byte[random.nextInt(Math.min(MAX_BUFFER_SIZE - MIN_BUFFER_SIZE, fileLength)) + MIN_BUFFER_SIZE]; - random.nextBytes(buf); - int offset = random.nextInt(buf.length); - int length = random.nextInt(buf.length - offset); - fsOutput.writeBytes(buf, offset, length); - hdfsOutput.writeBytes(buf, offset, length); - } - fsOutput.close(); - hdfsOutput.close(); - } - - private String getName() { - return Long.toString(Math.abs(random.nextLong())); - } - - public static void rm(File file) { - if (!file.exists()) { - return; - } - if (file.isDirectory()) { - for (File f : file.listFiles()) { - rm(f); - } - } - file.delete(); - } - -} http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/890be501/blur-store/src/test/java/org/apache/blur/store/BaseDirectoryTestSuite.java ---------------------------------------------------------------------- diff --git a/blur-store/src/test/java/org/apache/blur/store/BaseDirectoryTestSuite.java b/blur-store/src/test/java/org/apache/blur/store/BaseDirectoryTestSuite.java new file mode 100644 index 0000000..48364ca --- /dev/null +++ b/blur-store/src/test/java/org/apache/blur/store/BaseDirectoryTestSuite.java @@ -0,0 +1,375 @@ +package org.apache.blur.store; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import java.io.File; +import java.io.IOException; +import java.util.Collection; +import java.util.HashSet; +import java.util.Random; +import java.util.Set; + +import org.apache.blur.lucene.LuceneVersionConstant; +import org.apache.blur.store.blockcache.LastModified; +import org.apache.blur.store.buffer.BufferStore; +import org.apache.lucene.analysis.core.KeywordAnalyzer; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field.Store; +import org.apache.lucene.document.IntField; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.NumericRangeQuery; +import org.apache.lucene.search.TopDocs; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.FSDirectory; +import org.apache.lucene.store.IOContext; +import org.apache.lucene.store.IndexInput; +import org.apache.lucene.store.IndexOutput; +import org.apache.lucene.store.Lock; +import org.apache.lucene.store.LockFactory; +import org.apache.lucene.store.RAMDirectory; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +public abstract class BaseDirectoryTestSuite { + protected static final File TMPDIR = new File(System.getProperty("blur.tmp.dir", "/tmp")); + + protected static final int MAX_NUMBER_OF_WRITES = 10000; + protected static final int MIN_FILE_SIZE = 100; + protected static final int MAX_FILE_SIZE = 100000; + protected static final int MIN_BUFFER_SIZE = 1; + protected static final int MAX_BUFFER_SIZE = 5000; + protected static final int MAX_NUMBER_OF_READS = 10000; + protected Directory directory; + protected File file; + protected long seed; + protected Random random; + + @Before + public void setUp() throws IOException { + BufferStore.init(128, 128); + file = new File(TMPDIR, "hdfsdirectorytest"); + rm(file); + seed = new Random().nextLong(); + random = new Random(seed); + setupDirectory(); + } + + @After + public void tearDown() { + print(file, ""); + } + + private void print(File f, String buf) { + if (f.isDirectory()) { + System.out.println(buf + "\\" + f.getName()); + for (File fl : f.listFiles()) { + if (fl.getName().startsWith(".")) { + continue; + } + print(fl, buf + " "); + } + } else { + System.out.println(buf + f.getName() + " " + f.length()); + } + } + + protected abstract void setupDirectory() throws IOException; + + @Test + public void testWritingAndReadingAFile() throws IOException { + + IndexOutput output = directory.createOutput("testing.test", IOContext.DEFAULT); + output.writeInt(12345); + output.flush(); + output.close(); + + IndexInput input = directory.openInput("testing.test", IOContext.DEFAULT); + assertEquals(12345, input.readInt()); + input.close(); + + String[] listAll = directory.listAll(); + assertEquals(1, listAll.length); + assertEquals("testing.test", listAll[0]); + + assertEquals(4, directory.fileLength("testing.test")); + + IndexInput input1 = directory.openInput("testing.test", IOContext.DEFAULT); + + IndexInput input2 = (IndexInput) input1.clone(); + assertEquals(12345, input2.readInt()); + input2.close(); + + assertEquals(12345, input1.readInt()); + input1.close(); + + assertFalse(directory.fileExists("testing.test.other")); + assertTrue(directory.fileExists("testing.test")); + directory.deleteFile("testing.test"); + assertFalse(directory.fileExists("testing.test")); + } + + @Test + public void testEOF() throws IOException { + Directory fsDir = new RAMDirectory(); + String name = "test.eof"; + createFile(name, fsDir, directory); + long fsLength = fsDir.fileLength(name); + long hdfsLength = directory.fileLength(name); + assertEquals(fsLength, hdfsLength); + testEof(name, fsDir, fsLength); + testEof(name, directory, hdfsLength); + } + + private void testEof(String name, Directory directory, long length) throws IOException { + IndexInput input = directory.openInput(name, IOContext.DEFAULT); + try { + input.seek(length); + input.readByte(); + fail("should throw eof"); + } catch (IOException e) { + } + } + + @Test + public void testWrites() throws IOException { + int i = 0; + try { + Set<String> names = new HashSet<String>(); + for (; i < 10; i++) { + Directory fsDir = new RAMDirectory(); + String name = getName(); + System.out.println("Working on pass [" + i + "] seed [" + seed + "] contains [" + names.contains(name) + "]"); + names.add(name); + createFile(name, fsDir, directory); + assertInputsEquals(name, fsDir, directory); + fsDir.close(); + } + } catch (Exception e) { + e.printStackTrace(); + fail("Test failed with seed [" + seed + "] on pass [" + i + "]"); + } + } + + @Test + public void testCreateIndex() throws IOException { + IndexWriterConfig conf = new IndexWriterConfig(LuceneVersionConstant.LUCENE_VERSION, new KeywordAnalyzer()); + IndexWriter writer = new IndexWriter(directory, conf); + int numDocs = 1000; + DirectoryReader reader = null; + for (int i = 0; i < 100; i++) { + if (reader == null) { + reader = DirectoryReader.open(writer, true); + } else { + DirectoryReader old = reader; + reader = DirectoryReader.openIfChanged(old, writer, true); + if (reader == null) { + reader = old; + } else { + old.close(); + } + } + assertEquals(i * numDocs, reader.numDocs()); + IndexSearcher searcher = new IndexSearcher(reader); + NumericRangeQuery<Integer> query = NumericRangeQuery.newIntRange("id", 42, 42, true, true); + TopDocs topDocs = searcher.search(query, 10); + assertEquals(i, topDocs.totalHits); + addDocuments(writer, numDocs); + } + writer.close(false); + reader.close(); + } + + private void addDocuments(IndexWriter writer, int numDocs) throws IOException { + for (int i = 0; i < numDocs; i++) { + writer.addDocument(getDoc(i)); + } + } + + private Document getDoc(int i) { + Document document = new Document(); + document.add(new IntField("id", i, Store.YES)); + return document; + } + + private void assertInputsEquals(String name, Directory fsDir, Directory hdfs) throws IOException { + int reads = random.nextInt(MAX_NUMBER_OF_READS); + IndexInput fsInput = fsDir.openInput(name, IOContext.DEFAULT); + IndexInput hdfsInput = hdfs.openInput(name, IOContext.DEFAULT); + assertEquals(fsInput.length(), hdfsInput.length()); + int fileLength = (int) fsInput.length(); + for (int i = 0; i < reads; i++) { + byte[] fsBuf = new byte[random.nextInt(Math.min(MAX_BUFFER_SIZE - MIN_BUFFER_SIZE, fileLength)) + MIN_BUFFER_SIZE]; + byte[] hdfsBuf = new byte[fsBuf.length]; + int offset = random.nextInt(fsBuf.length); + int length = random.nextInt(fsBuf.length - offset); + int pos = random.nextInt(fileLength - length); + fsInput.seek(pos); + fsInput.readBytes(fsBuf, offset, length); + hdfsInput.seek(pos); + hdfsInput.readBytes(hdfsBuf, offset, length); + for (int f = offset; f < length; f++) { + if (fsBuf[f] != hdfsBuf[f]) { + fail(); + } + } + } + fsInput.close(); + hdfsInput.close(); + } + + private void createFile(String name, Directory fsDir, Directory hdfs) throws IOException { + int writes = random.nextInt(MAX_NUMBER_OF_WRITES); + int fileLength = random.nextInt(MAX_FILE_SIZE - MIN_FILE_SIZE) + MIN_FILE_SIZE; + IndexOutput fsOutput = fsDir.createOutput(name, IOContext.DEFAULT); + fsOutput.setLength(fileLength); + IndexOutput hdfsOutput = hdfs.createOutput(name, IOContext.DEFAULT); + hdfsOutput.setLength(fileLength); + for (int i = 0; i < writes; i++) { + byte[] buf = new byte[random.nextInt(Math.min(MAX_BUFFER_SIZE - MIN_BUFFER_SIZE, fileLength)) + MIN_BUFFER_SIZE]; + random.nextBytes(buf); + int offset = random.nextInt(buf.length); + int length = random.nextInt(buf.length - offset); + fsOutput.writeBytes(buf, offset, length); + hdfsOutput.writeBytes(buf, offset, length); + } + fsOutput.close(); + hdfsOutput.close(); + } + + private String getName() { + return Long.toString(Math.abs(random.nextLong())); + } + + public static void rm(File file) { + if (!file.exists()) { + return; + } + if (file.isDirectory()) { + for (File f : file.listFiles()) { + rm(f); + } + } + file.delete(); + } + + public static Directory wrapLastModified(Directory dir) { + return new DirectoryLastModified(dir); + } + + public static class DirectoryLastModified extends Directory implements LastModified { + + private Directory _directory; + + public DirectoryLastModified(Directory dir) { + _directory = dir; + } + + @Override + public long getFileModified(String name) throws IOException { + if (_directory instanceof FSDirectory) { + File fileDir = ((FSDirectory) _directory).getDirectory(); + return new File(fileDir, name).lastModified(); + } + throw new RuntimeException("not impl"); + } + + @Override + public String[] listAll() throws IOException { + return _directory.listAll(); + } + + @Override + public void deleteFile(String name) throws IOException { + _directory.deleteFile(name); + } + + @Override + public long fileLength(String name) throws IOException { + return _directory.fileLength(name); + } + + @Override + public IndexOutput createOutput(String name, IOContext context) throws IOException { + return _directory.createOutput(name, context); + } + + @Override + public void sync(Collection<String> names) throws IOException { + _directory.sync(names); + } + + @Override + public IndexInput openInput(String name, IOContext context) throws IOException { + return _directory.openInput(name, context); + } + + @Override + public Lock makeLock(String name) { + return _directory.makeLock(name); + } + + @Override + public void clearLock(String name) throws IOException { + _directory.clearLock(name); + } + + @Override + public void setLockFactory(LockFactory lockFactory) throws IOException { + _directory.setLockFactory(lockFactory); + } + + @Override + public LockFactory getLockFactory() { + return _directory.getLockFactory(); + } + + @Override + public String getLockID() { + return _directory.getLockID(); + } + + @Override + public void copy(Directory to, String src, String dest, IOContext context) throws IOException { + _directory.copy(to, src, dest, context); + } + + @Override + public IndexInputSlicer createSlicer(String name, IOContext context) throws IOException { + return _directory.createSlicer(name, context); + } + + @Override + public boolean fileExists(String name) throws IOException { + return _directory.fileExists(name); + } + + @Override + public void close() throws IOException { + _directory.close(); + } + } + +} http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/890be501/blur-store/src/test/java/org/apache/blur/store/CacheDirectoryTestSuite.java ---------------------------------------------------------------------- diff --git a/blur-store/src/test/java/org/apache/blur/store/CacheDirectoryTestSuite.java b/blur-store/src/test/java/org/apache/blur/store/CacheDirectoryTestSuite.java new file mode 100644 index 0000000..c2baa1e --- /dev/null +++ b/blur-store/src/test/java/org/apache/blur/store/CacheDirectoryTestSuite.java @@ -0,0 +1,72 @@ +package org.apache.blur.store; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import java.io.File; +import java.io.IOException; + +import org.apache.blur.store.blockcache_v2.BaseCache; +import org.apache.blur.store.blockcache_v2.BaseCache.STORE; +import org.apache.blur.store.blockcache_v2.Cache; +import org.apache.blur.store.blockcache_v2.CacheDirectory; +import org.apache.blur.store.blockcache_v2.FileNameBlockSize; +import org.apache.blur.store.blockcache_v2.FileNameFilter; +import org.apache.blur.store.buffer.BufferStore; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.FSDirectory; +import org.junit.Test; + +public class CacheDirectoryTestSuite extends BaseDirectoryTestSuite { + + @Override + protected void setupDirectory() throws IOException { + int totalNumberOfBytes = 1000000; + int fileBufferSize = 129; + final int blockSize = 137; + FileNameBlockSize fileNameBlockSize = new FileNameBlockSize() { + @Override + public int getBlockSize(String directoryName, String fileName) { + return blockSize; + } + }; + FileNameFilter writeFilter = new FileNameFilter() { + @Override + public boolean accept(String directoryName, String fileName) { + return true; + } + }; + FileNameFilter readFilter = new FileNameFilter() { + @Override + public boolean accept(String directoryName, String fileName) { + // @TODO this needs to be enabled and issues resolved... + return false; + } + }; + + Cache cache = new BaseCache(totalNumberOfBytes, fileBufferSize, fileNameBlockSize, readFilter, writeFilter, + STORE.ON_HEAP); + Directory dir = FSDirectory.open(new File(file, "cache")); + + BufferStore.init(128, 128); + directory = new CacheDirectory("test", wrapLastModified(dir), cache); + } + + @Test + public void runsTheTests() { + } + +} http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/890be501/blur-store/src/test/java/org/apache/blur/store/HdfsDirectoryTest.java ---------------------------------------------------------------------- diff --git a/blur-store/src/test/java/org/apache/blur/store/HdfsDirectoryTest.java b/blur-store/src/test/java/org/apache/blur/store/HdfsDirectoryTest.java deleted file mode 100644 index 26501c8..0000000 --- a/blur-store/src/test/java/org/apache/blur/store/HdfsDirectoryTest.java +++ /dev/null @@ -1,41 +0,0 @@ -package org.apache.blur.store; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -import java.io.File; -import java.io.IOException; -import java.net.URI; - -import org.apache.blur.store.hdfs.HdfsDirectory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.Path; -import org.junit.Test; - -public class HdfsDirectoryTest extends BaseDirectoryTest { - - @Override - protected void setupDirectory() throws IOException { - URI uri = new File(file, "hdfs").toURI(); - Path hdfsDirPath = new Path(uri.toString()); - Configuration conf = new Configuration(); - directory = new HdfsDirectory(conf, hdfsDirPath); - } - - @Test - public void runsTheTests() {} - -} http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/890be501/blur-store/src/test/java/org/apache/blur/store/HdfsDirectoryTestSuite.java ---------------------------------------------------------------------- diff --git a/blur-store/src/test/java/org/apache/blur/store/HdfsDirectoryTestSuite.java b/blur-store/src/test/java/org/apache/blur/store/HdfsDirectoryTestSuite.java new file mode 100644 index 0000000..7079c5c --- /dev/null +++ b/blur-store/src/test/java/org/apache/blur/store/HdfsDirectoryTestSuite.java @@ -0,0 +1,41 @@ +package org.apache.blur.store; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import java.io.File; +import java.io.IOException; +import java.net.URI; + +import org.apache.blur.store.hdfs.HdfsDirectory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.junit.Test; + +public class HdfsDirectoryTestSuite extends BaseDirectoryTestSuite { + + @Override + protected void setupDirectory() throws IOException { + URI uri = new File(file, "hdfs").toURI(); + Path hdfsDirPath = new Path(uri.toString()); + Configuration conf = new Configuration(); + directory = new HdfsDirectory(conf, hdfsDirPath); + } + + @Test + public void runsTheTests() {} + +} http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/890be501/blur-store/src/test/java/org/apache/blur/store/SoftlinkHdfsDirectoryTest.java ---------------------------------------------------------------------- diff --git a/blur-store/src/test/java/org/apache/blur/store/SoftlinkHdfsDirectoryTest.java b/blur-store/src/test/java/org/apache/blur/store/SoftlinkHdfsDirectoryTest.java deleted file mode 100644 index 0ac2735..0000000 --- a/blur-store/src/test/java/org/apache/blur/store/SoftlinkHdfsDirectoryTest.java +++ /dev/null @@ -1,43 +0,0 @@ -package org.apache.blur.store; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -import java.io.File; -import java.io.IOException; -import java.net.URI; - -import org.apache.blur.store.hdfs.SoftlinkHdfsDirectory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.Path; -import org.junit.Test; - -public class SoftlinkHdfsDirectoryTest extends BaseDirectoryTest { - - @Override - protected void setupDirectory() throws IOException { - URI uri = new File(file, "hdfs").toURI(); - Path hdfsDirPath = new Path(uri.toString()); - Path store = new Path(hdfsDirPath, "store"); - Path link = new Path(hdfsDirPath, "link"); - Configuration conf = new Configuration(); - directory = new SoftlinkHdfsDirectory(conf, store, link); - } - - @Test - public void runsTheTests() {} - -} http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/890be501/blur-store/src/test/java/org/apache/blur/store/SoftlinkHdfsDirectoryTestSuite.java ---------------------------------------------------------------------- diff --git a/blur-store/src/test/java/org/apache/blur/store/SoftlinkHdfsDirectoryTestSuite.java b/blur-store/src/test/java/org/apache/blur/store/SoftlinkHdfsDirectoryTestSuite.java new file mode 100644 index 0000000..d6648cf --- /dev/null +++ b/blur-store/src/test/java/org/apache/blur/store/SoftlinkHdfsDirectoryTestSuite.java @@ -0,0 +1,43 @@ +package org.apache.blur.store; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import java.io.File; +import java.io.IOException; +import java.net.URI; + +import org.apache.blur.store.hdfs.SoftlinkHdfsDirectory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.junit.Test; + +public class SoftlinkHdfsDirectoryTestSuite extends BaseDirectoryTestSuite { + + @Override + protected void setupDirectory() throws IOException { + URI uri = new File(file, "hdfs").toURI(); + Path hdfsDirPath = new Path(uri.toString()); + Path store = new Path(hdfsDirPath, "store"); + Path link = new Path(hdfsDirPath, "link"); + Configuration conf = new Configuration(); + directory = new SoftlinkHdfsDirectory(conf, store, link); + } + + @Test + public void runsTheTests() {} + +} http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/890be501/blur-store/src/test/java/org/apache/blur/store/blockcache/BlockCacheTest.java ---------------------------------------------------------------------- diff --git a/blur-store/src/test/java/org/apache/blur/store/blockcache/BlockCacheTest.java b/blur-store/src/test/java/org/apache/blur/store/blockcache/BlockCacheTest.java index b1d9d12..0699189 100644 --- a/blur-store/src/test/java/org/apache/blur/store/blockcache/BlockCacheTest.java +++ b/blur-store/src/test/java/org/apache/blur/store/blockcache/BlockCacheTest.java @@ -116,9 +116,10 @@ public class BlockCacheTest { /** * Verify checking of buffer size limits against the cached block size. + * @throws IOException */ @Test - public void testLongBuffer() { + public void testLongBuffer() throws IOException { Random random = new Random(); int blockSize = BlockCache._8K; int slabSize = blockSize * 1024; @@ -138,6 +139,7 @@ public class BlockCacheTest { assertTrue(blockCache.store(blockCacheKey, 1, testData, 0, blockSize - 1)); assertTrue(blockCache.store(blockCacheKey, 1, testData, blockSize, blockSize - 1)); assertTrue(blockCache.store(blockCacheKey, 1, testData, blockSize * 2, blockSize - 1)); + blockCache.close(); } private static byte[] testData(Random random, int size, byte[] buf) {
