Make sure we're only dorking with the .type extensions
Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/707a4bdd Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/707a4bdd Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/707a4bdd Branch: refs/heads/apache-blur-0.2 Commit: 707a4bdd5ab148ac50656847de1fae65d0b8889b Parents: 31cc873 Author: twilliams <[email protected]> Authored: Tue Feb 25 21:05:36 2014 -0500 Committer: twilliams <[email protected]> Committed: Tue Feb 25 21:05:36 2014 -0500 ---------------------------------------------------------------------- .../apache/blur/analysis/HdfsFieldManager.java | 4 +- .../blur/analysis/HdfsFieldManagerTest.java | 64 ++++++++++++++------ 2 files changed, 48 insertions(+), 20 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/707a4bdd/blur-query/src/main/java/org/apache/blur/analysis/HdfsFieldManager.java ---------------------------------------------------------------------- diff --git a/blur-query/src/main/java/org/apache/blur/analysis/HdfsFieldManager.java b/blur-query/src/main/java/org/apache/blur/analysis/HdfsFieldManager.java index 8988e20..72a9577 100644 --- a/blur-query/src/main/java/org/apache/blur/analysis/HdfsFieldManager.java +++ b/blur-query/src/main/java/org/apache/blur/analysis/HdfsFieldManager.java @@ -98,7 +98,9 @@ public class HdfsFieldManager extends BaseFieldManager { List<String> fieldNames = new ArrayList<String>(); for (FileStatus fileStatus : listStatus) { if (!fileStatus.isDir()) { - fieldNames.add(fileStatus.getPath().getName().replace(TYPE_FILE_EXT, "")); + String fileName = fileStatus.getPath().getName(); + + fieldNames.add(fileName.substring(0, fileName.lastIndexOf(TYPE_FILE_EXT))); } } return fieldNames; http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/707a4bdd/blur-query/src/test/java/org/apache/blur/analysis/HdfsFieldManagerTest.java ---------------------------------------------------------------------- diff --git a/blur-query/src/test/java/org/apache/blur/analysis/HdfsFieldManagerTest.java b/blur-query/src/test/java/org/apache/blur/analysis/HdfsFieldManagerTest.java index f14fb8c..b1df816 100644 --- a/blur-query/src/test/java/org/apache/blur/analysis/HdfsFieldManagerTest.java +++ b/blur-query/src/test/java/org/apache/blur/analysis/HdfsFieldManagerTest.java @@ -1,6 +1,7 @@ package org.apache.blur.analysis; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; @@ -36,7 +37,7 @@ import org.junit.Test; */ public class HdfsFieldManagerTest extends BaseFieldManagerTest { private static final String DFS_FIELD_MANAGER_PATH = "./target/tmp/HdfsFieldManagerTest/meta"; - + @Override protected BaseFieldManager newFieldManager(boolean create) throws IOException { Configuration config = new Configuration(); @@ -47,25 +48,50 @@ public class HdfsFieldManagerTest extends BaseFieldManagerTest { } return new HdfsFieldManager(_fieldLessField, new KeywordAnalyzer(), path, config); } - - @Test - public void fieldManagerShouldIgnoreUnknownFiles() throws IOException { - BaseFieldManager fieldManager = newFieldManager(true); - fieldManager.addColumnDefinition("fam1", "col1", null, true, "text", false, null); - FieldTypeDefinition fieldTypeDefinition1 = fieldManager.getFieldTypeDefinition("fam1.col1"); - - assertNotNull(fieldTypeDefinition1); - Path path = new Path(DFS_FIELD_MANAGER_PATH, "mydoc.txt"); - FSDataOutputStream outputStream = path.getFileSystem(new Configuration()).create(path, false); - IOUtils.write("Some text..", outputStream); - outputStream.close(); - - fieldManager.loadFromStorage(); - - FieldTypeDefinition fieldTypeDefinition2 = fieldManager.getFieldTypeDefinition("fam1.col1"); - assertNotNull(fieldTypeDefinition2); + + @Test + public void fieldManagerShouldIgnoreUnknownFiles() throws IOException { + BaseFieldManager fieldManager = newFieldManager(true); + fieldManager.addColumnDefinition("unknowntest", "col1", null, true, "text", false, null); + FieldTypeDefinition fieldTypeDefinition1 = fieldManager.getFieldTypeDefinition("unknowntest.col1"); + + assertNotNull(fieldTypeDefinition1); + Path path = new Path(DFS_FIELD_MANAGER_PATH, "mydoc.txt"); + FSDataOutputStream outputStream = path.getFileSystem(new Configuration()).create(path, false); + IOUtils.write("Some text..", outputStream); + outputStream.close(); + + BaseFieldManager fieldManager2 = newFieldManager(false); + fieldManager2.loadFromStorage(); + FieldTypeDefinition fieldTypeDefinition2 = fieldManager2.getFieldTypeDefinition("unknowntest.col1"); + assertNotNull(fieldTypeDefinition2); + } + + @Test + public void columnsWithNamesSimilarToOurSuffixShouldntCauseProblems() throws IOException { + BaseFieldManager fieldManager = newFieldManager(true); + fieldManager.addColumnDefinition("similartest", "typeFoo", null, true, "text", false, null); + FieldTypeDefinition fieldTypeDefinition1 = fieldManager.getFieldTypeDefinition("similartest.typeFoo"); + + assertNotNull(fieldTypeDefinition1); + assertEquals("typeFoo", fieldTypeDefinition1.getColumnName()); + + assertTrue(fieldManager.getFieldNames().contains("similartest.typeFoo")); + BaseFieldManager fieldManager2 = newFieldManager(false); + fieldManager2.loadFromStorage(); + + assertTrue("Should have the real field.", fieldManager2.getFieldNames().contains("similartest.typeFoo")); + FieldTypeDefinition fieldTypeDefinition2 = fieldManager2.getFieldTypeDefinition("similartest.typeFoo"); + assertNotNull(fieldTypeDefinition2); + assertEquals("typeFoo", fieldTypeDefinition2.getColumnName()); + assertEquals("similartest", fieldTypeDefinition2.getFamily()); + assertEquals("text", fieldTypeDefinition2.getFieldType()); + + fieldTypeDefinition2 = fieldManager2.getFieldTypeDefinition("similartestFoo"); + + assertNull("This one shouldn't exist.", fieldTypeDefinition2); } - + @SuppressWarnings("unchecked") @Test public void testStoreMetaData() throws IOException {
