BLUR-ID:49 Added Javascript method call to intialize Analyzer Signed-off-by: Aaron McCurry <[email protected]>
Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/b14bde71 Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/b14bde71 Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/b14bde71 Branch: refs/heads/0.2-dev Commit: b14bde717147062d88cdacf4700b38759aa74fa5 Parents: 203b8bd Author: Gagan <[email protected]> Authored: Thu Feb 7 14:43:51 2013 +0530 Committer: Aaron McCurry <[email protected]> Committed: Mon Feb 18 21:08:43 2013 -0500 ---------------------------------------------------------------------- .../org/apache/blur/analysis/BlurAnalyzer.java | 53 +++++++++++++-- .../org/apache/blur/analysis/BlurAnalyzerTest.java | 37 ++++++++++- .../src/main/scripts/interface/Blur.thrift | 7 ++- 3 files changed, 89 insertions(+), 8 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/b14bde71/src/blur-core/src/main/java/org/apache/blur/analysis/BlurAnalyzer.java ---------------------------------------------------------------------- diff --git a/src/blur-core/src/main/java/org/apache/blur/analysis/BlurAnalyzer.java b/src/blur-core/src/main/java/org/apache/blur/analysis/BlurAnalyzer.java index fa5dd2d..de63673 100644 --- a/src/blur-core/src/main/java/org/apache/blur/analysis/BlurAnalyzer.java +++ b/src/blur-core/src/main/java/org/apache/blur/analysis/BlurAnalyzer.java @@ -18,20 +18,30 @@ package org.apache.blur.analysis; */ import static org.apache.blur.lucene.LuceneVersionConstant.LUCENE_VERSION; +import java.nio.ByteBuffer; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; +import javax.script.Invocable; +import javax.script.ScriptEngine; +import javax.script.ScriptEngineManager; +import javax.script.ScriptException; + +import org.apache.blur.log.Log; +import org.apache.blur.log.LogFactory; +import org.apache.blur.server.BlurServer; import org.apache.blur.thrift.generated.AnalyzerSubType; import org.apache.blur.thrift.generated.AnalyzerType; +import org.apache.blur.thrift.generated.ClassDefinition; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.AnalyzerWrapper; import org.apache.lucene.analysis.standard.StandardAnalyzer; public final class BlurAnalyzer extends AnalyzerWrapper { - + private static final Log LOG = LogFactory.getLog(BlurAnalyzer.class); private Map<String, Set<String>> _subIndexNameLookups = new HashMap<String, Set<String>>(); private Analyzer _defaultAnalyzer; private Map<String, Analyzer> _analyzers = new HashMap<String, Analyzer>(); @@ -52,6 +62,23 @@ public final class BlurAnalyzer extends AnalyzerWrapper { _defaultAnalyzer = new StandardAnalyzer(LUCENE_VERSION); convert(analyzer); } + + public Analyzer createAnalyzerFromJavascriptInitFunction(ClassDefinition classDefinition) { + String initFunction = classDefinition.getInitFunction(); + Map<String,ByteBuffer> arguments = classDefinition.getArguments(); + ScriptEngineManager manager = new ScriptEngineManager(); + ScriptEngine engine = manager.getEngineByName("JavaScript"); + try { + engine.eval(initFunction); + Invocable inv = (Invocable) engine; + return (Analyzer)inv.invokeFunction("initAnalyzer", arguments ); + } catch (ScriptException e) { + throw new RuntimeException(e); + } catch (NoSuchMethodException e) { + LOG.error("Javascript function initAnalyzer not found."); + throw new RuntimeException(e); + } + } private void convert(org.apache.blur.thrift.generated.Analyzer analyzer) { if (analyzer == null) { @@ -67,10 +94,17 @@ public final class BlurAnalyzer extends AnalyzerWrapper { org.apache.blur.thrift.generated.ClassDefinition analyzerClassDefinition = analyzerType.getClassDefinition(); if (analyzerClassDefinition != null) { String analyzerClassName = analyzerClassDefinition.getClassName(); + String initFunction = analyzerClassDefinition.getInitFunction(); Object analyzerClassObject; - - analyzerClassObject = analyzerClassName == null ? null : Class.forName(analyzerClassName).newInstance(); - + if (initFunction != null){ + analyzerClassObject = createAnalyzerFromJavascriptInitFunction(analyzerClassDefinition); + if (analyzerClassName != null){ + LOG.warn("Analyzer class :: "+analyzerClassName+" ignored. Using javascript init function to created Analyzer."); + } + }else{ + analyzerClassObject = analyzerClassName == null ? null : Class.forName(analyzerClassName).newInstance(); + } + if (analyzerClassObject != null) { if (analyzerClassObject instanceof Analyzer) { _analyzers.put(indexName, (Analyzer) analyzerClassObject); @@ -84,7 +118,16 @@ public final class BlurAnalyzer extends AnalyzerWrapper { for (AnalyzerSubType analyzerSubType : analyzerSubTypes) { String subIndexName = analyzerSubType.getSubFieldName(); String subAnalyzerClassName = analyzerSubType.getClassDefinition().getClassName(); - Object subAnalyzerClassObject = subAnalyzerClassName == null ? null : Class.forName(subAnalyzerClassName).newInstance(); + String sunInitFunction = analyzerSubType.getClassDefinition().getInitFunction(); + Object subAnalyzerClassObject; + if (sunInitFunction != null){ + subAnalyzerClassObject = createAnalyzerFromJavascriptInitFunction(analyzerClassDefinition); + if (subAnalyzerClassName != null){ + LOG.warn("Sub Analyzer class :: "+subAnalyzerClassName+" ignored. Using javascript init function to created Analyzer."); + } + }else{ + subAnalyzerClassObject = subAnalyzerClassName == null ? null : Class.forName(subAnalyzerClassName).newInstance(); + } if (subAnalyzerClassObject != null) { if (subAnalyzerClassObject instanceof Analyzer) { _analyzers.put(indexName, (Analyzer) subAnalyzerClassObject); http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/b14bde71/src/blur-core/src/test/java/org/apache/blur/analysis/BlurAnalyzerTest.java ---------------------------------------------------------------------- diff --git a/src/blur-core/src/test/java/org/apache/blur/analysis/BlurAnalyzerTest.java b/src/blur-core/src/test/java/org/apache/blur/analysis/BlurAnalyzerTest.java index e7eecbf..e4c5fe2 100644 --- a/src/blur-core/src/test/java/org/apache/blur/analysis/BlurAnalyzerTest.java +++ b/src/blur-core/src/test/java/org/apache/blur/analysis/BlurAnalyzerTest.java @@ -18,14 +18,47 @@ package org.apache.blur.analysis; */ import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; +import javax.script.ScriptException; + +import org.apache.blur.thrift.generated.Analyzer; +import org.apache.blur.thrift.generated.AnalyzerSubType; +import org.apache.blur.thrift.generated.AnalyzerType; +import org.apache.blur.thrift.generated.ClassDefinition; +import org.apache.lucene.analysis.standard.StandardAnalyzer; +import org.junit.Assert; import org.junit.Test; public class BlurAnalyzerTest { @Test - public void testNeedsToBeWritten() throws IOException { - + public void testBlurAnalyzerForClassDefinition() throws IOException, ScriptException, NoSuchMethodException { + + Map<String,ByteBuffer> arguments = new HashMap<String, ByteBuffer>(); + arguments.put("StopWord1",null); + arguments.put("StopWord2",null); + + String initFunction = "function initAnalyzer(arguments) { args = new java.util.HashMap(arguments); stopWordsSet = new org.apache.lucene.analysis.util.CharArraySet(org.apache.lucene.util.Version.LUCENE_40, args.keySet(), true);return new org.apache.lucene.analysis.standard.StandardAnalyzer(org.apache.lucene.util.Version.LUCENE_40,stopWordsSet);}"; + ClassDefinition classDefinition = new ClassDefinition(); + classDefinition.setArguments(arguments); + classDefinition.setInitFunction(initFunction); + + AnalyzerType analyzerType = new AnalyzerType(); + analyzerType.setFieldName("test"); + analyzerType.setClassDefinition(classDefinition); + + Analyzer analyzer = new Analyzer(); + analyzer.setAnalyzerTypes(Arrays.asList(analyzerType)); + + BlurAnalyzer blurAnalyzer = new BlurAnalyzer(analyzer); + StandardAnalyzer createdAnalyzer = (StandardAnalyzer)blurAnalyzer.getWrappedAnalyzer("test"); + blurAnalyzer.close(); + createdAnalyzer.getStopwordSet(); + Assert.assertEquals(2,createdAnalyzer.getStopwordSet().size()); } } http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/b14bde71/src/distribution/src/main/scripts/interface/Blur.thrift ---------------------------------------------------------------------- diff --git a/src/distribution/src/main/scripts/interface/Blur.thrift b/src/distribution/src/main/scripts/interface/Blur.thrift index 8fda442..74a12ab 100644 --- a/src/distribution/src/main/scripts/interface/Blur.thrift +++ b/src/distribution/src/main/scripts/interface/Blur.thrift @@ -194,7 +194,12 @@ struct TableSchema { struct ClassDefinition { 1:string className, - 2:map<string,binary> arguments + 2:map<string,binary> arguments, + /** + * user can define its own way to initialize analyzer by providing javascript function. + * This function will always be used if defined. + */ + 3:string initFunction } /**
