Repository: incubator-blur Updated Branches: refs/heads/apache-blur-0.2 b726909c4 -> e71d44ab7
Let hdfs field manager ignore files he don't understand Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/e71d44ab Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/e71d44ab Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/e71d44ab Branch: refs/heads/apache-blur-0.2 Commit: e71d44ab738120d12dc22f4efb5dee65bb7de031 Parents: b726909 Author: twilliams <[email protected]> Authored: Tue Feb 25 18:33:15 2014 -0500 Committer: twilliams <[email protected]> Committed: Tue Feb 25 18:33:15 2014 -0500 ---------------------------------------------------------------------- .../apache/blur/analysis/HdfsFieldManager.java | 11 +++---- .../blur/analysis/HdfsFieldManagerTest.java | 31 ++++++++++++++++++-- 2 files changed, 34 insertions(+), 8 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/e71d44ab/blur-query/src/main/java/org/apache/blur/analysis/HdfsFieldManager.java ---------------------------------------------------------------------- diff --git a/blur-query/src/main/java/org/apache/blur/analysis/HdfsFieldManager.java b/blur-query/src/main/java/org/apache/blur/analysis/HdfsFieldManager.java index e2abfe3..4c311cd 100644 --- a/blur-query/src/main/java/org/apache/blur/analysis/HdfsFieldManager.java +++ b/blur-query/src/main/java/org/apache/blur/analysis/HdfsFieldManager.java @@ -60,6 +60,7 @@ public class HdfsFieldManager extends BaseFieldManager { private static final String FAMILY = "_family_"; private static final String COLUMN_NAME = "_columnName_"; private static final String SUB_COLUMN_NAME = "_subColumnName_"; + private static final String TYPE_FILE_EXT = ".type"; private static Lock _lock = new Lock() { private final java.util.concurrent.locks.Lock _javalock = new ReentrantReadWriteLock().writeLock(); @@ -104,10 +105,10 @@ public class HdfsFieldManager extends BaseFieldManager { FileStatus[] listStatus = _fileSystem.listStatus(_storagePath, new PathFilter() { @Override public boolean accept(Path path) { - if (path.getName().endsWith(".tmp")) { - return false; + if (path.getName().endsWith(TYPE_FILE_EXT)) { + return true; } - return true; + return false; } }); if (listStatus == null) { @@ -116,7 +117,7 @@ public class HdfsFieldManager extends BaseFieldManager { List<String> fieldNames = new ArrayList<String>(); for (FileStatus fileStatus : listStatus) { if (!fileStatus.isDir()) { - fieldNames.add(fileStatus.getPath().getName()); + fieldNames.add(fileStatus.getPath().getName().replace(TYPE_FILE_EXT, "")); } } return fieldNames; @@ -188,7 +189,7 @@ public class HdfsFieldManager extends BaseFieldManager { } private Path getFieldPath(String fieldName) { - return new Path(_storagePath, fieldName); + return new Path(_storagePath, fieldName + TYPE_FILE_EXT); } private String getComments() { http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/e71d44ab/blur-query/src/test/java/org/apache/blur/analysis/HdfsFieldManagerTest.java ---------------------------------------------------------------------- diff --git a/blur-query/src/test/java/org/apache/blur/analysis/HdfsFieldManagerTest.java b/blur-query/src/test/java/org/apache/blur/analysis/HdfsFieldManagerTest.java index 26169a5..f14fb8c 100644 --- a/blur-query/src/test/java/org/apache/blur/analysis/HdfsFieldManagerTest.java +++ b/blur-query/src/test/java/org/apache/blur/analysis/HdfsFieldManagerTest.java @@ -1,13 +1,18 @@ package org.apache.blur.analysis; -import static org.junit.Assert.*; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; import java.io.IOException; import java.util.HashMap; import java.util.Map; import java.util.Map.Entry; +import org.apache.commons.io.IOUtils; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.lucene.analysis.core.KeywordAnalyzer; @@ -30,17 +35,37 @@ import org.junit.Test; * the License. */ public class HdfsFieldManagerTest extends BaseFieldManagerTest { + private static final String DFS_FIELD_MANAGER_PATH = "./target/tmp/HdfsFieldManagerTest/meta"; + @Override protected BaseFieldManager newFieldManager(boolean create) throws IOException { Configuration config = new Configuration(); - Path path = new Path("./target/tmp/HdfsFieldManagerTest/meta"); + Path path = new Path(DFS_FIELD_MANAGER_PATH); FileSystem fileSystem = path.getFileSystem(config); if (create) { fileSystem.delete(path, true); } return new HdfsFieldManager(_fieldLessField, new KeywordAnalyzer(), path, config); } - + + @Test + public void fieldManagerShouldIgnoreUnknownFiles() throws IOException { + BaseFieldManager fieldManager = newFieldManager(true); + fieldManager.addColumnDefinition("fam1", "col1", null, true, "text", false, null); + FieldTypeDefinition fieldTypeDefinition1 = fieldManager.getFieldTypeDefinition("fam1.col1"); + + assertNotNull(fieldTypeDefinition1); + Path path = new Path(DFS_FIELD_MANAGER_PATH, "mydoc.txt"); + FSDataOutputStream outputStream = path.getFileSystem(new Configuration()).create(path, false); + IOUtils.write("Some text..", outputStream); + outputStream.close(); + + fieldManager.loadFromStorage(); + + FieldTypeDefinition fieldTypeDefinition2 = fieldManager.getFieldTypeDefinition("fam1.col1"); + assertNotNull(fieldTypeDefinition2); + } + @SuppressWarnings("unchecked") @Test public void testStoreMetaData() throws IOException {
