Updated Branches:
refs/heads/master 77666a0ba -> 0e91346dc
Fixed BLUR-299 and BLUR-300
Conflicts:
blur-core/src/test/java/org/apache/blur/thrift/BlurClusterTest.java
Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/0e91346d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/0e91346d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/0e91346d
Branch: refs/heads/master
Commit: 0e91346dcb55ebe81edffd768d22e6cd6fe7c567
Parents: 77666a0
Author: Aaron McCurry <[email protected]>
Authored: Wed Nov 20 10:50:57 2013 -0500
Committer: Aaron McCurry <[email protected]>
Committed: Wed Nov 20 10:53:06 2013 -0500
----------------------------------------------------------------------
.../org/apache/blur/server/TableContext.java | 30 ++++++++++++--------
.../java/org/apache/blur/thrift/TableAdmin.java | 4 ++-
.../test/java/org/apache/blur/MiniCluster.java | 1 +
.../src/test/java/org/apache/blur/TestType.java | 28 ++++++++++++++++++
.../org/apache/blur/thrift/BlurClusterTest.java | 22 ++++++++++++++
.../apache/blur/analysis/HdfsFieldManager.java | 11 ++++++-
6 files changed, 82 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/0e91346d/blur-core/src/main/java/org/apache/blur/server/TableContext.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/server/TableContext.java
b/blur-core/src/main/java/org/apache/blur/server/TableContext.java
index 1e22ce6..d361625 100644
--- a/blur-core/src/main/java/org/apache/blur/server/TableContext.java
+++ b/blur-core/src/main/java/org/apache/blur/server/TableContext.java
@@ -83,7 +83,6 @@ public class TableContext {
cache.clear();
}
- @SuppressWarnings("unchecked")
public static TableContext create(TableDescriptor tableDescriptor) {
if (tableDescriptor == null) {
throw new NullPointerException("TableDescriptor can not be null.");
@@ -134,23 +133,40 @@ public class TableContext {
try {
HdfsFieldManager hdfsFieldManager = new HdfsFieldManager(SUPER, new
NoStopWordStandardAnalyzer(), storagePath,
configuration, strict, defaultMissingFieldType,
defaultMissingFieldLessIndexing, defaultMissingFieldProps);
+ loadCustomTypes(tableContext, blurConfiguration, hdfsFieldManager);
hdfsFieldManager.load();
tableContext.fieldManager = hdfsFieldManager;
} catch (IOException e) {
throw new RuntimeException(e);
}
+ Class<?> c1 =
configuration.getClass(BLUR_SHARD_INDEX_DELETION_POLICY_MAXAGE,
+ KeepOnlyLastCommitDeletionPolicy.class);
+ tableContext.indexDeletionPolicy = (IndexDeletionPolicy)
configure(ReflectionUtils.newInstance(c1, configuration),
+ tableContext);
+ Class<?> c2 = configuration.getClass(BLUR_SHARD_INDEX_SIMILARITY,
DefaultSimilarity.class);
+ tableContext.similarity = (Similarity)
configure(ReflectionUtils.newInstance(c2, configuration), tableContext);
+
+ cache.put(name, tableContext);
+ return tableContext;
+ }
+
+ @SuppressWarnings("unchecked")
+ private static void loadCustomTypes(TableContext tableContext,
BlurConfiguration blurConfiguration,
+ FieldManager fieldManager) {
Set<Entry<String, String>> entrySet =
blurConfiguration.getProperties().entrySet();
+ TableDescriptor descriptor = tableContext.descriptor;
for (Entry<String, String> entry : entrySet) {
String key = entry.getKey();
if (key.startsWith(BLUR_FIELDTYPE)) {
String className = entry.getValue();
+ descriptor.putToTableProperties(key, className);
LOG.info("Attempting to load new type [{0}]", className);
Class<? extends FieldTypeDefinition> clazz;
try {
clazz = (Class<? extends FieldTypeDefinition>)
Class.forName(className);
FieldTypeDefinition fieldTypeDefinition = clazz.newInstance();
- tableContext.fieldManager.registerType(clazz);
+ fieldManager.registerType(clazz);
LOG.info("Sucessfully loaded new type [{0}] with name [{1}]",
className, fieldTypeDefinition.getName());
} catch (ClassNotFoundException e) {
LOG.error("The field type definition class [{0}] was not found.
Check the classpath.", e, className);
@@ -161,16 +177,6 @@ public class TableContext {
}
}
}
-
- Class<?> c1 =
configuration.getClass(BLUR_SHARD_INDEX_DELETION_POLICY_MAXAGE,
- KeepOnlyLastCommitDeletionPolicy.class);
- tableContext.indexDeletionPolicy = (IndexDeletionPolicy)
configure(ReflectionUtils.newInstance(c1, configuration),
- tableContext);
- Class<?> c2 = configuration.getClass(BLUR_SHARD_INDEX_SIMILARITY,
DefaultSimilarity.class);
- tableContext.similarity = (Similarity)
configure(ReflectionUtils.newInstance(c2, configuration), tableContext);
-
- cache.put(name, tableContext);
- return tableContext;
}
private static Map<String, String> emptyIfNull(Map<String, String>
defaultMissingFieldProps) {
http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/0e91346d/blur-core/src/main/java/org/apache/blur/thrift/TableAdmin.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/thrift/TableAdmin.java
b/blur-core/src/main/java/org/apache/blur/thrift/TableAdmin.java
index cac2818..9bd68a3 100644
--- a/blur-core/src/main/java/org/apache/blur/thrift/TableAdmin.java
+++ b/blur-core/src/main/java/org/apache/blur/thrift/TableAdmin.java
@@ -370,7 +370,9 @@ public abstract class TableAdmin implements Iface {
if (cluster == null) {
throw new BException("Table [" + table + "] not found.");
}
- return _clusterStatus.getTableDescriptor(true, cluster, table);
+ TableDescriptor tableDescriptor =
_clusterStatus.getTableDescriptor(true, cluster, table);
+ TableContext tableContext = TableContext.create(tableDescriptor);
+ return tableContext.getDescriptor();
} catch (Exception e) {
LOG.error("Unknown error while trying to describe a table [" + table +
"].", e);
throw new BException("Unknown error while trying to describe a table ["
+ table + "].", e);
http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/0e91346d/blur-core/src/test/java/org/apache/blur/MiniCluster.java
----------------------------------------------------------------------
diff --git a/blur-core/src/test/java/org/apache/blur/MiniCluster.java
b/blur-core/src/test/java/org/apache/blur/MiniCluster.java
index d8a40fb..bd8af0c 100644
--- a/blur-core/src/test/java/org/apache/blur/MiniCluster.java
+++ b/blur-core/src/test/java/org/apache/blur/MiniCluster.java
@@ -223,6 +223,7 @@ public class MiniCluster {
configuration.setLong(BLUR_SHARD_SAFEMODEDELAY, 5000);
configuration.setInt(BLUR_GUI_CONTROLLER_PORT, -1);
configuration.setInt(BLUR_GUI_SHARD_PORT, -1);
+ configuration.set("blur.fieldtype.customtype1", TestType.class.getName());
return configuration;
}
http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/0e91346d/blur-core/src/test/java/org/apache/blur/TestType.java
----------------------------------------------------------------------
diff --git a/blur-core/src/test/java/org/apache/blur/TestType.java
b/blur-core/src/test/java/org/apache/blur/TestType.java
new file mode 100644
index 0000000..9575102
--- /dev/null
+++ b/blur-core/src/test/java/org/apache/blur/TestType.java
@@ -0,0 +1,28 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.blur;
+
+import org.apache.blur.analysis.type.StringFieldTypeDefinition;
+
+public class TestType extends StringFieldTypeDefinition {
+
+ @Override
+ public String getName() {
+ return "test_type";
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/0e91346d/blur-core/src/test/java/org/apache/blur/thrift/BlurClusterTest.java
----------------------------------------------------------------------
diff --git
a/blur-core/src/test/java/org/apache/blur/thrift/BlurClusterTest.java
b/blur-core/src/test/java/org/apache/blur/thrift/BlurClusterTest.java
index bfa1250..f29ef14 100644
--- a/blur-core/src/test/java/org/apache/blur/thrift/BlurClusterTest.java
+++ b/blur-core/src/test/java/org/apache/blur/thrift/BlurClusterTest.java
@@ -32,12 +32,14 @@ import java.lang.management.MemoryUsage;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
+import java.util.Map;
import java.util.UUID;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicReference;
import org.apache.blur.MiniCluster;
+import org.apache.blur.TestType;
import org.apache.blur.manager.IndexManager;
import org.apache.blur.thirdparty.thrift_0_9_0.TException;
import org.apache.blur.thrift.generated.Blur;
@@ -117,6 +119,26 @@ public class BlurClusterTest {
testTestShardFailover();
testTermsList();
testCreateDisableAndRemoveTable();
+ testCreateTableWithCustomType();
+ }
+
+ private void testCreateTableWithCustomType() throws IOException,
BlurException, TException {
+ Blur.Iface client = getClient();
+ TableDescriptor tableDescriptor = new TableDescriptor();
+ tableDescriptor.setName("test_type");
+ tableDescriptor.setShardCount(1);
+ tableDescriptor.setTableUri(miniCluster.getFileSystemUri().toString() +
"/blur/test_type");
+ client.createTable(tableDescriptor);
+ List<String> tableList = client.tableList();
+ assertTrue(tableList.contains("test_type"));
+
+ client.disableTable("test_type");
+
+ client.enableTable("test_type");
+
+ TableDescriptor describe = client.describe("test_type");
+ Map<String, String> tableProperties = describe.getTableProperties();
+ assertEquals(TestType.class.getName(),
tableProperties.get("blur.fieldtype.customtype1"));
}
public void testCreateTable() throws BlurException, TException, IOException {
http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/0e91346d/blur-query/src/main/java/org/apache/blur/analysis/HdfsFieldManager.java
----------------------------------------------------------------------
diff --git
a/blur-query/src/main/java/org/apache/blur/analysis/HdfsFieldManager.java
b/blur-query/src/main/java/org/apache/blur/analysis/HdfsFieldManager.java
index a293c91..daf43ee 100644
--- a/blur-query/src/main/java/org/apache/blur/analysis/HdfsFieldManager.java
+++ b/blur-query/src/main/java/org/apache/blur/analysis/HdfsFieldManager.java
@@ -35,6 +35,7 @@ import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PathFilter;
import org.apache.lucene.analysis.Analyzer;
public class HdfsFieldManager extends BaseFieldManager {
@@ -95,7 +96,15 @@ public class HdfsFieldManager extends BaseFieldManager {
if (!_fileSystem.exists(_storagePath)) {
return EMPTY_LIST;
}
- FileStatus[] listStatus = _fileSystem.listStatus(_storagePath);
+ FileStatus[] listStatus = _fileSystem.listStatus(_storagePath, new
PathFilter() {
+ @Override
+ public boolean accept(Path path) {
+ if (path.getName().endsWith(".tmp")) {
+ return false;
+ }
+ return true;
+ }
+ });
if (listStatus == null) {
return EMPTY_LIST;
}
