Fixed BLUR-250 and partial fix for BLUR-251
Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/c08b0404 Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/c08b0404 Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/c08b0404 Branch: refs/heads/master Commit: c08b0404f609ea73e9e6bc814c926372a81d86c8 Parents: 0ac7006 Author: Aaron McCurry <[email protected]> Authored: Fri Oct 4 09:20:20 2013 -0400 Committer: Aaron McCurry <[email protected]> Committed: Fri Oct 4 09:21:46 2013 -0400 ---------------------------------------------------------------------- .../org/apache/blur/server/TableContext.java | 75 ++++++++++++++++- .../blur/thrift/ThriftBlurShardServer.java | 3 + .../apache/blur/server/TableContextTest.java | 86 ++++++++++++++++++++ .../java/org/apache/blur/server/TestType.java | 12 +++ .../org/apache/blur/utils/BlurConstants.java | 2 + 5 files changed, 174 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/c08b0404/blur-core/src/main/java/org/apache/blur/server/TableContext.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/server/TableContext.java b/blur-core/src/main/java/org/apache/blur/server/TableContext.java index 24fe308..5b85e2e 100644 --- a/blur-core/src/main/java/org/apache/blur/server/TableContext.java +++ b/blur-core/src/main/java/org/apache/blur/server/TableContext.java @@ -16,6 +16,7 @@ package org.apache.blur.server; * See the License for the specific language governing permissions and * limitations under the License. */ +import static org.apache.blur.utils.BlurConstants.BLUR_FIELDTYPE; import static org.apache.blur.utils.BlurConstants.BLUR_SAHRD_INDEX_SIMILARITY; import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_INDEX_DELETION_POLICY_MAXAGE; import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_TIME_BETWEEN_COMMITS; @@ -26,10 +27,13 @@ import java.io.IOException; import java.util.HashMap; import java.util.Map; import java.util.Map.Entry; +import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; +import org.apache.blur.BlurConfiguration; import org.apache.blur.analysis.FieldManager; +import org.apache.blur.analysis.FieldTypeDefinition; import org.apache.blur.analysis.HdfsFieldManager; import org.apache.blur.analysis.NoStopWordStandardAnalyzer; import org.apache.blur.log.Log; @@ -52,6 +56,10 @@ public class TableContext { private static final String LOGS = "logs"; private static final String TYPES = "types"; + private static ConcurrentHashMap<String, TableContext> cache = new ConcurrentHashMap<String, TableContext>(); + private static Configuration systemConfiguration; + private static BlurConfiguration systemBlurConfiguration; + private Path tablePath; private Path walTablePath; private String defaultFieldName; @@ -65,8 +73,7 @@ public class TableContext { private ScoreType defaultScoreType; private Term defaultPrimeDocTerm; private FieldManager fieldManager; - - private static ConcurrentHashMap<String, TableContext> cache = new ConcurrentHashMap<String, TableContext>(); + private BlurConfiguration blurConfiguration; protected TableContext() { @@ -76,6 +83,7 @@ public class TableContext { cache.clear(); } + @SuppressWarnings("unchecked") public static TableContext create(TableDescriptor tableDescriptor) { if (tableDescriptor == null) { throw new NullPointerException("TableDescriptor can not be null."); @@ -84,22 +92,29 @@ public class TableContext { if (name == null) { throw new NullPointerException("Table name in the TableDescriptor can not be null."); } + String tableUri = tableDescriptor.getTableUri(); + if (tableUri == null) { + throw new NullPointerException("Table uri in the TableDescriptor can not be null."); + } TableContext tableContext = cache.get(name); if (tableContext != null) { return tableContext; } LOG.info("Creating table context for table [{0}]", name); - Configuration configuration = new Configuration(); + Configuration configuration = getSystemConfiguration(); + BlurConfiguration blurConfiguration = getSystemBlurConfiguration(); Map<String, String> tableProperties = tableDescriptor.getTableProperties(); if (tableProperties != null) { for (Entry<String, String> prop : tableProperties.entrySet()) { configuration.set(prop.getKey(), prop.getValue()); + blurConfiguration.set(prop.getKey(), prop.getValue()); } } tableContext = new TableContext(); tableContext.configuration = configuration; - tableContext.tablePath = new Path(tableDescriptor.getTableUri()); + tableContext.blurConfiguration = blurConfiguration; + tableContext.tablePath = new Path(tableUri); tableContext.walTablePath = new Path(tableContext.tablePath, LOGS); tableContext.defaultFieldName = SUPER; @@ -125,6 +140,28 @@ public class TableContext { throw new RuntimeException(e); } + Set<Entry<String, String>> entrySet = blurConfiguration.getProperties().entrySet(); + for (Entry<String, String> entry : entrySet) { + String key = entry.getKey(); + if (key.startsWith(BLUR_FIELDTYPE)) { + String className = entry.getValue(); + LOG.info("Attempting to load new type [{0}]", className); + Class<? extends FieldTypeDefinition> clazz; + try { + clazz = (Class<? extends FieldTypeDefinition>) Class.forName(className); + FieldTypeDefinition fieldTypeDefinition = clazz.newInstance(); + tableContext.fieldManager.registerType(clazz); + LOG.info("Sucessfully loaded new type [{0}] with name [{1}]", className, fieldTypeDefinition.getName()); + } catch (ClassNotFoundException e) { + LOG.error("The field type definition class [{0}] was not found. Check the classpath.", e, className); + } catch (InstantiationException e) { + LOG.error("Could not create the field type definition [{0}].", e, className); + } catch (IllegalAccessException e) { + LOG.error("Unknown exception while trying to load field type definition [{0}].", e, className); + } + } + } + Class<?> c1 = configuration.getClass(BLUR_SHARD_INDEX_DELETION_POLICY_MAXAGE, KeepOnlyLastCommitDeletionPolicy.class); tableContext.indexDeletionPolicy = (IndexDeletionPolicy) configure(ReflectionUtils.newInstance(c1, configuration), @@ -205,4 +242,34 @@ public class TableContext { public long getTimeBetweenWALSyncsNanos() { return TimeUnit.MILLISECONDS.toNanos(10); } + + public BlurConfiguration getBlurConfiguration() { + return blurConfiguration; + } + + public static synchronized Configuration getSystemConfiguration() { + if (systemConfiguration == null) { + systemConfiguration = new Configuration(); + } + return systemConfiguration; + } + + public static void setSystemConfiguration(Configuration systemConfiguration) { + TableContext.systemConfiguration = systemConfiguration; + } + + public static synchronized BlurConfiguration getSystemBlurConfiguration() { + if (systemBlurConfiguration == null) { + try { + systemBlurConfiguration = new BlurConfiguration(); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + return systemBlurConfiguration; + } + + public static void setSystemBlurConfiguration(BlurConfiguration systemBlurConfiguration) { + TableContext.systemBlurConfiguration = systemBlurConfiguration; + } } http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/c08b0404/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java b/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java index 4a89cff..4c7d133 100644 --- a/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java +++ b/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java @@ -69,6 +69,7 @@ import org.apache.blur.manager.writer.BlurIndexRefresher; import org.apache.blur.metrics.JSONReporter; import org.apache.blur.metrics.ReporterSetup; import org.apache.blur.server.ShardServerEventHandler; +import org.apache.blur.server.TableContext; import org.apache.blur.store.BlockCacheDirectoryFactory; import org.apache.blur.store.BlockCacheDirectoryFactoryV1; import org.apache.blur.store.BlockCacheDirectoryFactoryV2; @@ -117,6 +118,8 @@ public class ThriftBlurShardServer extends ThriftServer { public static ThriftServer createServer(int serverIndex, BlurConfiguration configuration, boolean randomPort) throws Exception { Configuration config = new Configuration(); + TableContext.setSystemBlurConfiguration(configuration); + TableContext.setSystemConfiguration(config); String bindAddress = configuration.get(BLUR_SHARD_BIND_ADDRESS); int bindPort = configuration.getInt(BLUR_SHARD_BIND_PORT, -1); http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/c08b0404/blur-core/src/test/java/org/apache/blur/server/TableContextTest.java ---------------------------------------------------------------------- diff --git a/blur-core/src/test/java/org/apache/blur/server/TableContextTest.java b/blur-core/src/test/java/org/apache/blur/server/TableContextTest.java new file mode 100644 index 0000000..45e3c91 --- /dev/null +++ b/blur-core/src/test/java/org/apache/blur/server/TableContextTest.java @@ -0,0 +1,86 @@ +package org.apache.blur.server; + +import static org.junit.Assert.*; + +import java.io.File; +import java.io.IOException; + +import org.apache.blur.BlurConfiguration; +import org.apache.blur.analysis.FieldManager; +import org.apache.blur.thrift.generated.TableDescriptor; +import org.apache.blur.utils.BlurConstants; +import org.junit.Before; +import org.junit.Test; + +public class TableContextTest { + + private static final File TMPDIR = new File("./target/tmp"); + private File base; + private String name = "testtable-TableContextTest"; + private File file; + + @Before + public void setup() { + TableContext.clear(); + base = new File(TMPDIR, "TableContextTest"); + rm(base); + file = new File(base, name); + file.mkdirs(); + } + + @Test + public void testLoadingNewTypeWhenNotDefined() throws IOException { + TableDescriptor tableDescriptor = new TableDescriptor(); + tableDescriptor.setName(name); + tableDescriptor.setTableUri(file.toURI().toString()); + TableContext context = TableContext.create(tableDescriptor); + + FieldManager fieldManager = context.getFieldManager(); + try { + fieldManager.addColumnDefinition("fam", "col", null, false, "test", null); + fail("should fail because new type is not loaded."); + } catch (IllegalArgumentException e) { + + } + } + + @Test + public void testLoadingNewTypeWhenDefinedFromTableDescriptor() throws IOException { + TableDescriptor tableDescriptor = new TableDescriptor(); + tableDescriptor.setName(name); + tableDescriptor.setTableUri(file.toURI().toString()); + tableDescriptor.putToTableProperties(BlurConstants.BLUR_FIELDTYPE + "test", TestType.class.getName()); + + TableContext context = TableContext.create(tableDescriptor); + FieldManager fieldManager = context.getFieldManager(); + + fieldManager.addColumnDefinition("fam", "col", null, false, "test", null); + } + + @Test + public void testLoadingNewTypeWhenDefinedFromBlurConfigure() throws IOException { + BlurConfiguration blurConfiguration = new BlurConfiguration(); + blurConfiguration.set(BlurConstants.BLUR_FIELDTYPE + "test", TestType.class.getName()); + + TableContext.setSystemBlurConfiguration(blurConfiguration); + + TableDescriptor tableDescriptor = new TableDescriptor(); + tableDescriptor.setName(name); + tableDescriptor.setTableUri(file.toURI().toString()); + + TableContext context = TableContext.create(tableDescriptor); + FieldManager fieldManager = context.getFieldManager(); + + fieldManager.addColumnDefinition("fam", "col", null, false, "test", null); + } + + private void rm(File file) { + if (file.isDirectory()) { + for (File f : file.listFiles()) { + rm(f); + } + } + file.delete(); + } + +} http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/c08b0404/blur-core/src/test/java/org/apache/blur/server/TestType.java ---------------------------------------------------------------------- diff --git a/blur-core/src/test/java/org/apache/blur/server/TestType.java b/blur-core/src/test/java/org/apache/blur/server/TestType.java new file mode 100644 index 0000000..6c2e6ab --- /dev/null +++ b/blur-core/src/test/java/org/apache/blur/server/TestType.java @@ -0,0 +1,12 @@ +package org.apache.blur.server; + +import org.apache.blur.analysis.type.TextFieldTypeDefinition; + +public class TestType extends TextFieldTypeDefinition { + + @Override + public String getName() { + return "test"; + } + +} http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/c08b0404/blur-util/src/main/java/org/apache/blur/utils/BlurConstants.java ---------------------------------------------------------------------- diff --git a/blur-util/src/main/java/org/apache/blur/utils/BlurConstants.java b/blur-util/src/main/java/org/apache/blur/utils/BlurConstants.java index 1439aa1..ecc1b41 100644 --- a/blur-util/src/main/java/org/apache/blur/utils/BlurConstants.java +++ b/blur-util/src/main/java/org/apache/blur/utils/BlurConstants.java @@ -72,6 +72,8 @@ public class BlurConstants { public static final String BLUR_SHARD_INDEX_DELETION_POLICY_MAXAGE = "blur.shard.index.deletion.policy.maxage"; public static final String BLUR_ZOOKEEPER_SYSTEM_TIME_TOLERANCE = "blur.zookeeper.system.time.tolerance"; public static final String BLUR_SAHRD_INDEX_SIMILARITY = "blur.sahrd.index.similarity"; + + public static final String BLUR_FIELDTYPE = "blur.fieldtype."; public static final String BLUR_SHARD_TIME_BETWEEN_COMMITS = "blur.shard.time.between.commits"; public static final String BLUR_SHARD_TIME_BETWEEN_REFRESHS = "blur.shard.time.between.refreshs";
