@@ -116,7 +117,7 @@ public class HdfsFieldManager extends BaseFieldManager {
List<String> fieldNames = new ArrayList<String>();
for (FileStatus fileStatus : listStatus) {
if (!fileStatus.isDir()) {
- fieldNames.add(fileStatus.getPath().getName());
+ fieldNames.add(fileStatus.getPath().getName().replace(TYPE_FILE_EXT,
""));
}
}
Do you think that this will cause problems for fields that are <some
family>.type<any suffix> ? Maybe we should check that the file ends with
the TYPE_FILE_EXT and substring up to the beginning of the suffix?
Aaron
On Tue, Feb 25, 2014 at 6:33 PM, <[email protected]> wrote:
> Repository: incubator-blur
> Updated Branches:
> refs/heads/apache-blur-0.2 b726909c4 -> e71d44ab7
>
>
> Let hdfs field manager ignore files he don't understand
>
>
> Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo
> Commit:
> http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/e71d44ab
> Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/e71d44ab
> Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/e71d44ab
>
> Branch: refs/heads/apache-blur-0.2
> Commit: e71d44ab738120d12dc22f4efb5dee65bb7de031
> Parents: b726909
> Author: twilliams <[email protected]>
> Authored: Tue Feb 25 18:33:15 2014 -0500
> Committer: twilliams <[email protected]>
> Committed: Tue Feb 25 18:33:15 2014 -0500
>
> ----------------------------------------------------------------------
> .../apache/blur/analysis/HdfsFieldManager.java | 11 +++----
> .../blur/analysis/HdfsFieldManagerTest.java | 31 ++++++++++++++++++--
> 2 files changed, 34 insertions(+), 8 deletions(-)
> ----------------------------------------------------------------------
>
>
>
> http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/e71d44ab/blur-query/src/main/java/org/apache/blur/analysis/HdfsFieldManager.java
> ----------------------------------------------------------------------
> diff --git
> a/blur-query/src/main/java/org/apache/blur/analysis/HdfsFieldManager.java
> b/blur-query/src/main/java/org/apache/blur/analysis/HdfsFieldManager.java
> index e2abfe3..4c311cd 100644
> ---
> a/blur-query/src/main/java/org/apache/blur/analysis/HdfsFieldManager.java
> +++
> b/blur-query/src/main/java/org/apache/blur/analysis/HdfsFieldManager.java
> @@ -60,6 +60,7 @@ public class HdfsFieldManager extends BaseFieldManager {
> private static final String FAMILY = "_family_";
> private static final String COLUMN_NAME = "_columnName_";
> private static final String SUB_COLUMN_NAME = "_subColumnName_";
> + private static final String TYPE_FILE_EXT = ".type";
>
> private static Lock _lock = new Lock() {
> private final java.util.concurrent.locks.Lock _javalock = new
> ReentrantReadWriteLock().writeLock();
> @@ -104,10 +105,10 @@ public class HdfsFieldManager extends
> BaseFieldManager {
> FileStatus[] listStatus = _fileSystem.listStatus(_storagePath, new
> PathFilter() {
> @Override
> public boolean accept(Path path) {
> - if (path.getName().endsWith(".tmp")) {
> - return false;
> + if (path.getName().endsWith(TYPE_FILE_EXT)) {
> + return true;
> }
> - return true;
> + return false;
> }
> });
> if (listStatus == null) {
> @@ -116,7 +117,7 @@ public class HdfsFieldManager extends BaseFieldManager
> {
> List<String> fieldNames = new ArrayList<String>();
> for (FileStatus fileStatus : listStatus) {
> if (!fileStatus.isDir()) {
> - fieldNames.add(fileStatus.getPath().getName());
> +
> fieldNames.add(fileStatus.getPath().getName().replace(TYPE_FILE_EXT, ""));
> }
> }
> return fieldNames;
> @@ -188,7 +189,7 @@ public class HdfsFieldManager extends BaseFieldManager
> {
> }
>
> private Path getFieldPath(String fieldName) {
> - return new Path(_storagePath, fieldName);
> + return new Path(_storagePath, fieldName + TYPE_FILE_EXT);
> }
>
> private String getComments() {
>
>
> http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/e71d44ab/blur-query/src/test/java/org/apache/blur/analysis/HdfsFieldManagerTest.java
> ----------------------------------------------------------------------
> diff --git
> a/blur-query/src/test/java/org/apache/blur/analysis/HdfsFieldManagerTest.java
> b/blur-query/src/test/java/org/apache/blur/analysis/HdfsFieldManagerTest.java
> index 26169a5..f14fb8c 100644
> ---
> a/blur-query/src/test/java/org/apache/blur/analysis/HdfsFieldManagerTest.java
> +++
> b/blur-query/src/test/java/org/apache/blur/analysis/HdfsFieldManagerTest.java
> @@ -1,13 +1,18 @@
> package org.apache.blur.analysis;
>
> -import static org.junit.Assert.*;
> +import static org.junit.Assert.assertFalse;
> +import static org.junit.Assert.assertNotNull;
> +import static org.junit.Assert.assertNull;
> +import static org.junit.Assert.assertTrue;
>
> import java.io.IOException;
> import java.util.HashMap;
> import java.util.Map;
> import java.util.Map.Entry;
>
> +import org.apache.commons.io.IOUtils;
> import org.apache.hadoop.conf.Configuration;
> +import org.apache.hadoop.fs.FSDataOutputStream;
> import org.apache.hadoop.fs.FileSystem;
> import org.apache.hadoop.fs.Path;
> import org.apache.lucene.analysis.core.KeywordAnalyzer;
> @@ -30,17 +35,37 @@ import org.junit.Test;
> * the License.
> */
> public class HdfsFieldManagerTest extends BaseFieldManagerTest {
> + private static final String DFS_FIELD_MANAGER_PATH =
> "./target/tmp/HdfsFieldManagerTest/meta";
> +
> @Override
> protected BaseFieldManager newFieldManager(boolean create) throws
> IOException {
> Configuration config = new Configuration();
> - Path path = new Path("./target/tmp/HdfsFieldManagerTest/meta");
> + Path path = new Path(DFS_FIELD_MANAGER_PATH);
> FileSystem fileSystem = path.getFileSystem(config);
> if (create) {
> fileSystem.delete(path, true);
> }
> return new HdfsFieldManager(_fieldLessField, new KeywordAnalyzer(),
> path, config);
> }
> -
> +
> + @Test
> + public void fieldManagerShouldIgnoreUnknownFiles() throws IOException
> {
> + BaseFieldManager fieldManager = newFieldManager(true);
> + fieldManager.addColumnDefinition("fam1", "col1", null, true,
> "text", false, null);
> + FieldTypeDefinition fieldTypeDefinition1 =
> fieldManager.getFieldTypeDefinition("fam1.col1");
> +
> + assertNotNull(fieldTypeDefinition1);
> + Path path = new Path(DFS_FIELD_MANAGER_PATH, "mydoc.txt");
> + FSDataOutputStream outputStream = path.getFileSystem(new
> Configuration()).create(path, false);
> + IOUtils.write("Some text..", outputStream);
> + outputStream.close();
> +
> + fieldManager.loadFromStorage();
> +
> + FieldTypeDefinition fieldTypeDefinition2 =
> fieldManager.getFieldTypeDefinition("fam1.col1");
> + assertNotNull(fieldTypeDefinition2);
> + }
> +
> @SuppressWarnings("unchecked")
> @Test
> public void testStoreMetaData() throws IOException {
>
>
