Index: java/engine/org/apache/derby/catalog/Java5SystemProcedures.java =================================================================== --- java/engine/org/apache/derby/catalog/Java5SystemProcedures.java (revision 1531427) +++ java/engine/org/apache/derby/catalog/Java5SystemProcedures.java (working copy) @@ -64,6 +64,7 @@ { { "databaseMetaData", "org.apache.derby.impl.tools.optional.DBMDWrapper" }, { "foreignViews", "org.apache.derby.impl.tools.optional.ForeignDBViews" }, + { "luceneSupport", "org.apache.derby.impl.tools.optional.LuceneSupport" }, { "optimizerTracing", "org.apache.derby.impl.sql.compile.OptimizerTracer" }, { "optimizerTracingViews", "org.apache.derby.impl.sql.compile.OptTraceViewer" }, }; Index: java/tools/org/apache/derby/impl/tools/build.xml =================================================================== --- java/tools/org/apache/derby/impl/tools/build.xml (revision 1531427) +++ java/tools/org/apache/derby/impl/tools/build.xml (working copy) @@ -90,8 +90,7 @@ - - + + + + + + + + + + + + + + + + Index: java/tools/org/apache/derby/impl/tools/optional/LuceneQueryVTI.java =================================================================== --- java/tools/org/apache/derby/impl/tools/optional/LuceneQueryVTI.java (revision 0) +++ java/tools/org/apache/derby/impl/tools/optional/LuceneQueryVTI.java (working copy) @@ -0,0 +1,142 @@ +package org.apache.derby.impl.tools.optional; + +import java.io.File; +import java.io.IOException; +import java.security.AccessController; +import java.security.PrivilegedActionException; +import java.security.PrivilegedExceptionAction; +import java.sql.SQLException; + +import org.apache.derby.vti.Restriction; +import org.apache.derby.vti.Restriction.ColumnQualifier; +import org.apache.derby.vti.StringColumnVTI; +import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.analysis.standard.StandardAnalyzer; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.queryparser.classic.ParseException; +import org.apache.lucene.queryparser.classic.QueryParser; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.ScoreDoc; +import org.apache.lucene.search.TopDocs; +import org.apache.lucene.search.TopScoreDocCollector; +import org.apache.lucene.store.FSDirectory; +import org.apache.lucene.util.Version; + +public class LuceneQueryVTI extends StringColumnVTI { + + String derbyQuery; + ScoreDoc[] hits; + IndexReader ir; + IndexSearcher searcher; + double rankCutoff = 0; + int hitindex = -1; + + public LuceneQueryVTI(String q, String docDomain, String column) throws IOException, ParseException { + + super(new String[]{"documentDomain", "columnname", "docId", "rank", "keycolumn", "keyvalue"}); + + derbyQuery = q; + + String schema = docDomain.substring(0,docDomain.indexOf(".")); + String table = docDomain.substring(docDomain.indexOf(".")+1, docDomain.length()); + + StringBuffer indexhome = new StringBuffer(); + indexhome.append(getDerbySystemHome()); + indexhome.append(File.separator); + indexhome.append("lucene"); + indexhome.append(File.separator); + indexhome.append(schema); + indexhome.append("_"); + indexhome.append(table); + indexhome.append("_"); + indexhome.append(column); + ir = DirectoryReader.open(FSDirectory.open(new File(indexhome.toString()))); + searcher = new IndexSearcher(ir); + Analyzer a = new StandardAnalyzer(Version.LUCENE_45); + QueryParser qp = new QueryParser(Version.LUCENE_45, "contents", a); + Query luceneQuery = qp.parse(derbyQuery); + TopScoreDocCollector tsdc = TopScoreDocCollector.create(1000, true); + if (rankCutoff != 0) { + tsdc = TopScoreDocCollector.create(1000, new ScoreDoc(0, (float) rankCutoff), true); + } + searcher.search(luceneQuery, tsdc); + TopDocs topdocs = tsdc.topDocs(); + hits = topdocs.scoreDocs; + } + + public void initScan(String[] columns, Restriction restriction) { + ColumnQualifier cq = (ColumnQualifier) restriction; + rankCutoff = Double.parseDouble((String)cq.getConstantOperand()); + } + + // 1 == documentDomain (schema.table) + // 2 == column + // 3 == lucene docId + // 4 == lucene rank + // 5 == keycolumn + // 6 == keyvalue + public String getRawColumn(int columnid) throws SQLException { + try { + if (columnid == 1) { + return searcher.doc(hits[hitindex].doc).get("schema") + "." + searcher.doc(hits[hitindex].doc).get("table"); + } else if (columnid == 2) { + return searcher.doc(hits[hitindex].doc).get("column"); + } else if (columnid == 3) { + return Integer.toString(hits[hitindex].doc); + } else if (columnid == 4) { + return Float.toString(hits[hitindex].score); + } else if (columnid == 5) { + return searcher.doc(hits[hitindex].doc).get("keycolumn"); + } else if (columnid == 6) { + return searcher.doc(hits[hitindex].doc).get("keyvalue"); + } else throw new SQLException("Column index out of bounds."); + } catch (IOException e) { + throw new SQLException("IOException reading Lucene index."); + } + } + + public boolean next() { + hitindex++; + if (hitindex < hits.length) { + return true; + } + return false; + } + + public void close() { + derbyQuery = null; + hits = null; + hitindex = 0; + } + + private static String getDerbySystemHome() throws IOException { + try { + return AccessController + .doPrivileged(new PrivilegedExceptionAction() { + public String run() throws IOException { + String dir = System.getProperty("derby.system.home"); + if (dir == null) { + return System.getProperty("user.dir"); + } else { + return dir; + } + } + }); + } catch (PrivilegedActionException pae) { + throw (IOException) pae.getCause(); + } catch (SecurityException se) { + throw (IOException) se.getCause(); + } + } + + public void finalize() { + try { + ir.close(); + } catch (IOException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + } +} Index: java/tools/org/apache/derby/impl/tools/optional/LuceneSupport.java =================================================================== --- java/tools/org/apache/derby/impl/tools/optional/LuceneSupport.java (revision 0) +++ java/tools/org/apache/derby/impl/tools/optional/LuceneSupport.java (working copy) @@ -0,0 +1,263 @@ +package org.apache.derby.impl.tools.optional; + +import java.io.File; +import java.io.IOException; +import java.security.AccessController; +import java.security.PrivilegedActionException; +import java.security.PrivilegedExceptionAction; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; + +import org.apache.derby.iapi.sql.dictionary.OptionalTool; +import org.apache.derby.vti.Restriction.ColumnQualifier; +import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.analysis.standard.StandardAnalyzer; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.Field.Store; +import org.apache.lucene.document.StringField; +import org.apache.lucene.document.TextField; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.index.Term; +import org.apache.lucene.queryparser.classic.ParseException; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.FSDirectory; +import org.apache.lucene.util.Version; + +public class LuceneSupport implements OptionalTool { + + // 0-arg constructor as an OptionalTool + public LuceneSupport() { + } + + public void loadTool(String... configurationParameters) throws SQLException { + createQueryFunction(); + createIndexProcedure(); + createUpdateProcedure(); + } + + public void unloadTool(String... configurationParameters) + throws SQLException { + dropQueryFunction(); + dropIndexProcedure(); + dropUpdateProcedure(); + } + + public static LuceneQueryVTI luceneQuery(String queryText, String docDomain, String column, double rankCutoff) throws ParseException, IOException{ + LuceneQueryVTI lqvti = new LuceneQueryVTI(queryText, docDomain, column); + if (rankCutoff > 0) { + ColumnQualifier r = new ColumnQualifier("rank",ColumnQualifier.ORDER_OP_LESSOREQUALS,Double.toString(rankCutoff)); + lqvti.initScan(new String[]{column}, r); + } + return lqvti; + } + + public static void luceneUpdateDocument(String schema, String table, String textcol, String keycol, String keyvalue) throws SQLException, IOException { + + IndexWriter iw = openLuceneIndex(schema, table, textcol); + + // select all from this column, add to lucene index + StringBuffer query = new StringBuffer("select "); + query.append(keycol); + query.append(","); + query.append(textcol); + query.append(" from "); + query.append(schema); + query.append("."); + query.append(table); + query.append(" where "); + query.append(keycol); + query.append("="); + query.append(keyvalue); + ResultSet rs = executeQuery(getDefaultConnection(), query.toString()); + + while (rs.next()) { + Document doc = new Document(); + doc.add(new StringField("schema", schema, Field.Store.YES)); + doc.add(new StringField("table", table, Field.Store.YES)); + doc.add(new StringField("column", textcol, Field.Store.YES)); + doc.add(new StringField("keycolumn", keycol, Field.Store.YES)); + + if (rs.getString(1) != null) { + // concatenate table/column/keyvalue together to make a uuid-ish keyvalue less + // likely to be hit as a term during updateDocument? + doc.add(new StringField("keyvalue", rs.getString(1), Store.YES)); + } + if (rs.getString(2) != null) { + doc.add(new TextField("contents", rs.getString(2), Store.NO)); + } + iw.updateDocument(new Term("keyvalue", keyvalue), doc); + } + rs.close(); + + iw.close(); + } + + private static Connection getDefaultConnection() throws SQLException { + return DriverManager.getConnection("jdbc:default:connection"); + } + + public static void luceneIndex(String schema, String table, String textcol, + String keycol) throws SQLException, IOException { + + IndexWriter iw = openLuceneIndex(schema, table, textcol); + + // select all from this column, add to lucene index + StringBuffer query = new StringBuffer("select "); + query.append(keycol); + query.append(","); + query.append(textcol); + query.append(" from "); + query.append(schema); + query.append("."); + query.append(table); + ResultSet rs = executeQuery(getDefaultConnection(), query.toString()); + + while (rs.next()) { + Document doc = new Document(); + doc.add(new StringField("schema", schema, Field.Store.YES)); + doc.add(new StringField("table", table, Field.Store.YES)); + doc.add(new StringField("column", textcol, Field.Store.YES)); + doc.add(new StringField("keycolumn", keycol, Field.Store.YES)); + + if (rs.getString(1) != null) { + // concatenate table/column/keyvalue together to make a uuid-ish keyvalue less + // likely to be hit as a term during updateDocument? + doc.add(new StringField("keyvalue", rs.getString(1), Store.YES)); + } + if (rs.getString(2) != null) { + doc.add(new TextField("contents", rs.getString(2), Store.NO)); + } + iw.addDocument(doc); + } + rs.close(); + + iw.close(); + } + + private void createQueryFunction() throws SQLException { + StringBuffer luceneFunction = new StringBuffer(); + luceneFunction.append("create function luceneQuery"); + luceneFunction.append("(queryText varchar( 32672 ),"); + luceneFunction.append("documentDomain varchar( 32672 ),"); + luceneFunction.append("textcolumn varchar( 32672 ),"); + luceneFunction.append("rankCutoff double)"); + luceneFunction.append("returns table"); + luceneFunction.append("("); + luceneFunction.append("documentDomain varchar( 32672 ),"); + luceneFunction.append("columnname varchar( 32672 ),"); + luceneFunction.append("documentID int,"); + luceneFunction.append("rank double,"); + luceneFunction.append("keycolumn varchar( 32672 ),"); + luceneFunction.append("keyvalue varchar( 32672 )"); + luceneFunction.append(")"); + luceneFunction.append("language java "); + luceneFunction.append("parameter style DERBY_JDBC_RESULT_SET "); + luceneFunction.append("contains sql "); + luceneFunction.append("external name 'org.apache.derby.impl.tools.optional.LuceneSupport.luceneQuery'"); + + executeDDL(getDefaultConnection(), luceneFunction.toString()); + } + + private void createIndexProcedure() throws SQLException { + StringBuffer luceneProcedure = new StringBuffer(); + luceneProcedure.append("create procedure LuceneSupport.luceneIndex("); + luceneProcedure.append("schemaname varchar(32672),"); + luceneProcedure.append("tablename varchar(32672),"); + luceneProcedure.append("keycolumn varchar(32672),"); + luceneProcedure.append("textcolumn varchar(32672))"); + luceneProcedure.append("parameter style java reads sql data language java external name "); + luceneProcedure.append("'org.apache.derby.impl.tools.optional.LuceneSupport.luceneIndex'"); + + executeDDL(getDefaultConnection(), luceneProcedure.toString()); + } + + private void createUpdateProcedure() throws SQLException { + StringBuffer luceneProcedure = new StringBuffer(); + luceneProcedure.append("create procedure LuceneSupport.luceneUpdateDocument("); + luceneProcedure.append("schemaname varchar(32672),"); + luceneProcedure.append("tablename varchar(32672),"); + luceneProcedure.append("keycolumn varchar(32672),"); + luceneProcedure.append("keyvalue varchar(32672),"); + luceneProcedure.append("textcolumn varchar(32672))"); + luceneProcedure.append("parameter style java reads sql data language java external name "); + luceneProcedure.append("'org.apache.derby.impl.tools.optional.LuceneSupport.luceneUpdateDocument'"); + + executeDDL(getDefaultConnection(), luceneProcedure.toString()); + } + + private void dropQueryFunction() throws SQLException { + String dropLuceneQuery = "drop function luceneQuery"; + + executeDDL(getDefaultConnection(), dropLuceneQuery); + } + + private void dropIndexProcedure() throws SQLException { + String dropIndexProcedure = "drop procedure LuceneSupport.luceneIndex"; + + executeDDL(getDefaultConnection(), dropIndexProcedure); + } + + private void dropUpdateProcedure() throws SQLException { + String dropUpdateProcedure = "drop procedure LuceneSupport.luceneUpdateDocument"; + + executeDDL(getDefaultConnection(), dropUpdateProcedure); + } + + private void executeDDL(Connection c, String text) throws SQLException { + PreparedStatement ddl = c.prepareStatement(text); + ddl.execute(); + ddl.close(); + } + + private static ResultSet executeQuery(Connection c, String text) throws SQLException { + PreparedStatement query = c.prepareStatement(text); + return query.executeQuery(); + } + + private static IndexWriter openLuceneIndex(String schema, String table, String textcol) throws IOException { + StringBuffer derbyHome = new StringBuffer(); + derbyHome.append(getDerbySystemHome()); + derbyHome.append(File.separator); + derbyHome.append("lucene"); + derbyHome.append(File.separator); + derbyHome.append(schema); + derbyHome.append("_"); + derbyHome.append(table); + derbyHome.append("_"); + derbyHome.append(textcol); + + Directory dir = FSDirectory.open(new File(derbyHome.toString())); + // allow this to be overridden in the configuration during load later. + Analyzer analyzer = new StandardAnalyzer(Version.LUCENE_45); + IndexWriterConfig iwc = new IndexWriterConfig(Version.LUCENE_45, + analyzer); + IndexWriter iw = new IndexWriter(dir, iwc); + + return iw; + } + + private static String getDerbySystemHome() throws IOException { + try { + return AccessController + .doPrivileged(new PrivilegedExceptionAction() { + public String run() throws IOException { + String dir = System.getProperty("derby.system.home"); + if (dir == null) { + return System.getProperty("user.dir"); + } else { + return dir; + } + } + }); + } catch (PrivilegedActionException pae) { + throw (IOException) pae.getCause(); + } catch (SecurityException se) { + throw (IOException) se.getCause(); + } + } +} Index: tools/ant/properties/extrapath.properties =================================================================== --- tools/ant/properties/extrapath.properties (revision 1531427) +++ tools/ant/properties/extrapath.properties (working copy) @@ -30,6 +30,9 @@ junit=${javatools.dir}/junit.jar emma=${javatools.dir}/emma.jar emma_ant=${javatools.dir}/emma_ant.jar +lucene_core=${javatools.dir}/lucene-core-4.5.0.jar +lucene_a_co=${javatools.dir}/lucene-analyzers-common-4.5.0.jar +lucene_qp=${javatools.dir}/lucene-queryparser-4.5.0.jar # # Base JavaDoc collection, used for both product and test JavaDocs. Index: tools/jar/tools.properties =================================================================== --- tools/jar/tools.properties (revision 1531427) +++ tools/jar/tools.properties (working copy) @@ -31,5 +31,5 @@ derby.module.planexporter=org.apache.derby.tools.PlanExporter derby.module.dbmdwrapper=org.apache.derby.impl.tools.optional.DBMDWrapper derby.module.fdbv=org.apache.derby.impl.tools.optional.ForeignDBViews +derby.module.lucenespt=org.apache.derby.impl.tools.optional.LuceneSupport - Index: tools/java/lucene-analyzers-common-4.5.0.jar =================================================================== Cannot display: file marked as a binary type. svn:mime-type = application/jar Index: tools/java/lucene-analyzers-common-4.5.0.jar =================================================================== --- tools/java/lucene-analyzers-common-4.5.0.jar (revision 0) +++ tools/java/lucene-analyzers-common-4.5.0.jar (working copy) Property changes on: tools/java/lucene-analyzers-common-4.5.0.jar ___________________________________________________________________ Added: svn:mime-type ## -0,0 +1 ## +application/jar \ No newline at end of property Index: tools/java/lucene-core-4.5.0.jar =================================================================== Cannot display: file marked as a binary type. svn:mime-type = application/jar Index: tools/java/lucene-core-4.5.0.jar =================================================================== --- tools/java/lucene-core-4.5.0.jar (revision 0) +++ tools/java/lucene-core-4.5.0.jar (working copy) Property changes on: tools/java/lucene-core-4.5.0.jar ___________________________________________________________________ Added: svn:mime-type ## -0,0 +1 ## +application/jar \ No newline at end of property Index: tools/java/lucene-queryparser-4.5.0.jar =================================================================== Cannot display: file marked as a binary type. svn:mime-type = application/jar Index: tools/java/lucene-queryparser-4.5.0.jar =================================================================== --- tools/java/lucene-queryparser-4.5.0.jar (revision 0) +++ tools/java/lucene-queryparser-4.5.0.jar (working copy) Property changes on: tools/java/lucene-queryparser-4.5.0.jar ___________________________________________________________________ Added: svn:mime-type ## -0,0 +1 ## +application/jar \ No newline at end of property