Updated Branches: refs/heads/apache-blur-0.2 06121ffd5 -> 1266c8169
Fixed BLUR-299 and BLUR-300 Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/1266c816 Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/1266c816 Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/1266c816 Branch: refs/heads/apache-blur-0.2 Commit: 1266c81699a8e4025bce2e45148189a15f5f6b86 Parents: 06121ff Author: Aaron McCurry <[email protected]> Authored: Wed Nov 20 10:50:57 2013 -0500 Committer: Aaron McCurry <[email protected]> Committed: Wed Nov 20 10:50:57 2013 -0500 ---------------------------------------------------------------------- .../org/apache/blur/server/TableContext.java | 30 +++++++++++------- .../java/org/apache/blur/thrift/TableAdmin.java | 4 ++- .../test/java/org/apache/blur/MiniCluster.java | 1 + .../src/test/java/org/apache/blur/TestType.java | 28 +++++++++++++++++ .../org/apache/blur/thrift/BlurClusterTest.java | 33 +++++++++++++++++--- .../apache/blur/analysis/HdfsFieldManager.java | 11 ++++++- 6 files changed, 88 insertions(+), 19 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/1266c816/blur-core/src/main/java/org/apache/blur/server/TableContext.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/server/TableContext.java b/blur-core/src/main/java/org/apache/blur/server/TableContext.java index 1e22ce6..d361625 100644 --- a/blur-core/src/main/java/org/apache/blur/server/TableContext.java +++ b/blur-core/src/main/java/org/apache/blur/server/TableContext.java @@ -83,7 +83,6 @@ public class TableContext { cache.clear(); } - @SuppressWarnings("unchecked") public static TableContext create(TableDescriptor tableDescriptor) { if (tableDescriptor == null) { throw new NullPointerException("TableDescriptor can not be null."); @@ -134,23 +133,40 @@ public class TableContext { try { HdfsFieldManager hdfsFieldManager = new HdfsFieldManager(SUPER, new NoStopWordStandardAnalyzer(), storagePath, configuration, strict, defaultMissingFieldType, defaultMissingFieldLessIndexing, defaultMissingFieldProps); + loadCustomTypes(tableContext, blurConfiguration, hdfsFieldManager); hdfsFieldManager.load(); tableContext.fieldManager = hdfsFieldManager; } catch (IOException e) { throw new RuntimeException(e); } + Class<?> c1 = configuration.getClass(BLUR_SHARD_INDEX_DELETION_POLICY_MAXAGE, + KeepOnlyLastCommitDeletionPolicy.class); + tableContext.indexDeletionPolicy = (IndexDeletionPolicy) configure(ReflectionUtils.newInstance(c1, configuration), + tableContext); + Class<?> c2 = configuration.getClass(BLUR_SHARD_INDEX_SIMILARITY, DefaultSimilarity.class); + tableContext.similarity = (Similarity) configure(ReflectionUtils.newInstance(c2, configuration), tableContext); + + cache.put(name, tableContext); + return tableContext; + } + + @SuppressWarnings("unchecked") + private static void loadCustomTypes(TableContext tableContext, BlurConfiguration blurConfiguration, + FieldManager fieldManager) { Set<Entry<String, String>> entrySet = blurConfiguration.getProperties().entrySet(); + TableDescriptor descriptor = tableContext.descriptor; for (Entry<String, String> entry : entrySet) { String key = entry.getKey(); if (key.startsWith(BLUR_FIELDTYPE)) { String className = entry.getValue(); + descriptor.putToTableProperties(key, className); LOG.info("Attempting to load new type [{0}]", className); Class<? extends FieldTypeDefinition> clazz; try { clazz = (Class<? extends FieldTypeDefinition>) Class.forName(className); FieldTypeDefinition fieldTypeDefinition = clazz.newInstance(); - tableContext.fieldManager.registerType(clazz); + fieldManager.registerType(clazz); LOG.info("Sucessfully loaded new type [{0}] with name [{1}]", className, fieldTypeDefinition.getName()); } catch (ClassNotFoundException e) { LOG.error("The field type definition class [{0}] was not found. Check the classpath.", e, className); @@ -161,16 +177,6 @@ public class TableContext { } } } - - Class<?> c1 = configuration.getClass(BLUR_SHARD_INDEX_DELETION_POLICY_MAXAGE, - KeepOnlyLastCommitDeletionPolicy.class); - tableContext.indexDeletionPolicy = (IndexDeletionPolicy) configure(ReflectionUtils.newInstance(c1, configuration), - tableContext); - Class<?> c2 = configuration.getClass(BLUR_SHARD_INDEX_SIMILARITY, DefaultSimilarity.class); - tableContext.similarity = (Similarity) configure(ReflectionUtils.newInstance(c2, configuration), tableContext); - - cache.put(name, tableContext); - return tableContext; } private static Map<String, String> emptyIfNull(Map<String, String> defaultMissingFieldProps) { http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/1266c816/blur-core/src/main/java/org/apache/blur/thrift/TableAdmin.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/thrift/TableAdmin.java b/blur-core/src/main/java/org/apache/blur/thrift/TableAdmin.java index cac2818..9bd68a3 100644 --- a/blur-core/src/main/java/org/apache/blur/thrift/TableAdmin.java +++ b/blur-core/src/main/java/org/apache/blur/thrift/TableAdmin.java @@ -370,7 +370,9 @@ public abstract class TableAdmin implements Iface { if (cluster == null) { throw new BException("Table [" + table + "] not found."); } - return _clusterStatus.getTableDescriptor(true, cluster, table); + TableDescriptor tableDescriptor = _clusterStatus.getTableDescriptor(true, cluster, table); + TableContext tableContext = TableContext.create(tableDescriptor); + return tableContext.getDescriptor(); } catch (Exception e) { LOG.error("Unknown error while trying to describe a table [" + table + "].", e); throw new BException("Unknown error while trying to describe a table [" + table + "].", e); http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/1266c816/blur-core/src/test/java/org/apache/blur/MiniCluster.java ---------------------------------------------------------------------- diff --git a/blur-core/src/test/java/org/apache/blur/MiniCluster.java b/blur-core/src/test/java/org/apache/blur/MiniCluster.java index d8a40fb..bd8af0c 100644 --- a/blur-core/src/test/java/org/apache/blur/MiniCluster.java +++ b/blur-core/src/test/java/org/apache/blur/MiniCluster.java @@ -223,6 +223,7 @@ public class MiniCluster { configuration.setLong(BLUR_SHARD_SAFEMODEDELAY, 5000); configuration.setInt(BLUR_GUI_CONTROLLER_PORT, -1); configuration.setInt(BLUR_GUI_SHARD_PORT, -1); + configuration.set("blur.fieldtype.customtype1", TestType.class.getName()); return configuration; } http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/1266c816/blur-core/src/test/java/org/apache/blur/TestType.java ---------------------------------------------------------------------- diff --git a/blur-core/src/test/java/org/apache/blur/TestType.java b/blur-core/src/test/java/org/apache/blur/TestType.java new file mode 100644 index 0000000..9575102 --- /dev/null +++ b/blur-core/src/test/java/org/apache/blur/TestType.java @@ -0,0 +1,28 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.blur; + +import org.apache.blur.analysis.type.StringFieldTypeDefinition; + +public class TestType extends StringFieldTypeDefinition { + + @Override + public String getName() { + return "test_type"; + } + +} http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/1266c816/blur-core/src/test/java/org/apache/blur/thrift/BlurClusterTest.java ---------------------------------------------------------------------- diff --git a/blur-core/src/test/java/org/apache/blur/thrift/BlurClusterTest.java b/blur-core/src/test/java/org/apache/blur/thrift/BlurClusterTest.java index eb96e78..ed8ae00 100644 --- a/blur-core/src/test/java/org/apache/blur/thrift/BlurClusterTest.java +++ b/blur-core/src/test/java/org/apache/blur/thrift/BlurClusterTest.java @@ -21,6 +21,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import java.io.File; @@ -31,12 +32,14 @@ import java.lang.management.MemoryUsage; import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.Map; import java.util.UUID; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; import org.apache.blur.MiniCluster; +import org.apache.blur.TestType; import org.apache.blur.manager.IndexManager; import org.apache.blur.thirdparty.thrift_0_9_0.TException; import org.apache.blur.thrift.generated.Blur; @@ -116,6 +119,26 @@ public class BlurClusterTest { testTestShardFailover(); testTermsList(); testCreateDisableAndRemoveTable(); + testCreateTableWithCustomType(); + } + + private void testCreateTableWithCustomType() throws IOException, BlurException, TException { + Blur.Iface client = getClient(); + TableDescriptor tableDescriptor = new TableDescriptor(); + tableDescriptor.setName("test_type"); + tableDescriptor.setShardCount(1); + tableDescriptor.setTableUri(miniCluster.getFileSystemUri().toString() + "/blur/test_type"); + client.createTable(tableDescriptor); + List<String> tableList = client.tableList(); + assertTrue(tableList.contains("test_type")); + + client.disableTable("test_type"); + + client.enableTable("test_type"); + + TableDescriptor describe = client.describe("test_type"); + Map<String, String> tableProperties = describe.getTableProperties(); + assertEquals(TestType.class.getName(), tableProperties.get("blur.fieldtype.customtype1")); } public void testCreateTable() throws BlurException, TException, IOException { @@ -167,7 +190,7 @@ public class BlurClusterTest { Schema schema = client.schema("test"); assertFalse(schema.getFamilies().isEmpty()); } - + private void testQueryWithSelector() throws BlurException, TException { Iface client = getClient(); BlurQuery blurQueryRow = new BlurQuery(); @@ -177,15 +200,15 @@ public class BlurClusterTest { blurQueryRow.setUseCacheIfPresent(false); blurQueryRow.setCacheResult(false); blurQueryRow.setSelector(new Selector()); - + BlurResults resultsRow = client.query("test", blurQueryRow); -// assertRowResults(resultsRow); + // assertRowResults(resultsRow); assertEquals(100, resultsRow.getTotalResults()); - + for (BlurResult blurResult : resultsRow.getResults()) { System.out.println(blurResult); } - + System.out.println(); } http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/1266c816/blur-query/src/main/java/org/apache/blur/analysis/HdfsFieldManager.java ---------------------------------------------------------------------- diff --git a/blur-query/src/main/java/org/apache/blur/analysis/HdfsFieldManager.java b/blur-query/src/main/java/org/apache/blur/analysis/HdfsFieldManager.java index a293c91..daf43ee 100644 --- a/blur-query/src/main/java/org/apache/blur/analysis/HdfsFieldManager.java +++ b/blur-query/src/main/java/org/apache/blur/analysis/HdfsFieldManager.java @@ -35,6 +35,7 @@ import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.PathFilter; import org.apache.lucene.analysis.Analyzer; public class HdfsFieldManager extends BaseFieldManager { @@ -95,7 +96,15 @@ public class HdfsFieldManager extends BaseFieldManager { if (!_fileSystem.exists(_storagePath)) { return EMPTY_LIST; } - FileStatus[] listStatus = _fileSystem.listStatus(_storagePath); + FileStatus[] listStatus = _fileSystem.listStatus(_storagePath, new PathFilter() { + @Override + public boolean accept(Path path) { + if (path.getName().endsWith(".tmp")) { + return false; + } + return true; + } + }); if (listStatus == null) { return EMPTY_LIST; }
