Repository: zeppelin Updated Branches: refs/heads/master 0f9a1a80c -> c195b5609
[ZEPPELIN-3593] Change LuceneSearch's directory to file system from memory ### What is this PR for? Reducing a usage of memory by moving Lucene search's directory to the file system ### What type of PR is it? [Improvement] ### Todos * [x] - Replace `RamDirectory` to `MMapDirectory` ### What is the Jira issue? * https://issues.apache.org/jira/browse/ZEPPELIN-3593 ### How should this be tested? It doesn't affect any current behaviors. Test search service ### Screenshots (if appropriate) ### Questions: * Does the licenses files need update? No * Is there breaking changes for older versions? No * Does this needs documentation? No Author: Jongyoul Lee <jongy...@gmail.com> Closes #3058 from jongyoul/ZEPPELIN-3593 and squashes the following commits: cdae583c4 [Jongyoul Lee] Add a logic to check if directoryPath is null or not 5a383c4ab [Jongyoul Lee] Add `RamDirectory` to use search service in case where disk is not available 3b76cc1c0 [Jongyoul Lee] Set a default temporary directory to `java.io.tmpdir` Clean up javadoc 1e415e00e [Jongyoul Lee] Add deletion logic for the temporary directory d26e64f88 [Jongyoul Lee] Replace `RamDirectory` to `MMapDirectory` provided by Lucene Project: http://git-wip-us.apache.org/repos/asf/zeppelin/repo Commit: http://git-wip-us.apache.org/repos/asf/zeppelin/commit/c195b560 Tree: http://git-wip-us.apache.org/repos/asf/zeppelin/tree/c195b560 Diff: http://git-wip-us.apache.org/repos/asf/zeppelin/diff/c195b560 Branch: refs/heads/master Commit: c195b5609bb4dbb9fd037381ea58d1650cee92bf Parents: 0f9a1a8 Author: Jongyoul Lee <jongy...@gmail.com> Authored: Wed Jul 11 13:15:04 2018 +0900 Committer: Jongyoul Lee <jongy...@apache.org> Committed: Fri Jul 13 11:28:40 2018 +0900 ---------------------------------------------------------------------- .../zeppelin/conf/ZeppelinConfiguration.java | 24 ++- .../apache/zeppelin/server/ZeppelinServer.java | 2 +- .../apache/zeppelin/search/LuceneSearch.java | 187 ++++++++++--------- .../zeppelin/search/LuceneSearchTest.java | 147 ++++++++------- 4 files changed, 195 insertions(+), 165 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/zeppelin/blob/c195b560/zeppelin-interpreter/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java ---------------------------------------------------------------------- diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java index 635adbf..0b13e59 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java @@ -17,6 +17,12 @@ package org.apache.zeppelin.conf; +import java.io.File; +import java.net.URL; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; import org.apache.commons.configuration.ConfigurationException; import org.apache.commons.configuration.XMLConfiguration; import org.apache.commons.configuration.tree.ConfigurationNode; @@ -25,13 +31,6 @@ import org.apache.zeppelin.util.Util; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.File; -import java.net.URL; -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - /** * Zeppelin configuration. * @@ -494,10 +493,10 @@ public class ZeppelinConfiguration extends XMLConfiguration { } public String getRelativeDir(String path) { - if (path != null && path.startsWith("/") || isWindowsPath(path)) { + if (path != null && path.startsWith(File.separator) || isWindowsPath(path)) { return path; } else { - return getString(ConfVars.ZEPPELIN_HOME) + "/" + path; + return getString(ConfVars.ZEPPELIN_HOME) + File.separator + path; } } @@ -623,6 +622,10 @@ public class ZeppelinConfiguration extends XMLConfiguration { return getString(ConfVars.ZEPPELIN_PROXY_PASSWORD); } + public String getZeppelinSearchTempPath() { + return getRelativeDir(ConfVars.ZEPPELIN_SEARCH_TEMP_PATH); + } + public Map<String, String> dumpConfigurations(ZeppelinConfiguration conf, ConfigurationKeyPredicate predicate) { Map<String, String> configurations = new HashMap<>(); @@ -823,7 +826,8 @@ public class ZeppelinConfiguration extends XMLConfiguration { ZEPPELIN_NOTEBOOK_CRON_FOLDERS("zeppelin.notebook.cron.folders", null), ZEPPELIN_PROXY_URL("zeppelin.proxy.url", null), ZEPPELIN_PROXY_USER("zeppelin.proxy.user", null), - ZEPPELIN_PROXY_PASSWORD("zeppelin.proxy.password", null); + ZEPPELIN_PROXY_PASSWORD("zeppelin.proxy.password", null), + ZEPPELIN_SEARCH_TEMP_PATH("zeppelin.search.temp.path", System.getProperty("java.io.tmpdir")); private String varName; @SuppressWarnings("rawtypes") http://git-wip-us.apache.org/repos/asf/zeppelin/blob/c195b560/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java ---------------------------------------------------------------------- diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java b/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java index 9e2c5f8..5c85c23 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java @@ -168,7 +168,7 @@ public class ZeppelinServer extends Application { notebookWsServer, notebookWsServer); this.replFactory = new InterpreterFactory(interpreterSettingManager); this.notebookRepo = new NotebookRepoSync(conf); - this.noteSearchService = new LuceneSearch(); + this.noteSearchService = new LuceneSearch(conf); this.notebookAuthorization = NotebookAuthorization.getInstance(); this.credentials = new Credentials( conf.credentialsPersist(), http://git-wip-us.apache.org/repos/asf/zeppelin/blob/c195b560/zeppelin-zengine/src/main/java/org/apache/zeppelin/search/LuceneSearch.java ---------------------------------------------------------------------- diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/search/LuceneSearch.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/search/LuceneSearch.java index b43c453..49719a6 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/search/LuceneSearch.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/search/LuceneSearch.java @@ -16,14 +16,20 @@ */ package org.apache.zeppelin.search; +import com.google.common.base.Joiner; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Lists; import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; import java.util.Collection; import java.util.Collections; import java.util.Date; import java.util.List; import java.util.Map; import java.util.concurrent.TimeUnit; - +import org.apache.commons.io.FileUtils; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.standard.StandardAnalyzer; @@ -39,7 +45,6 @@ import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.index.Term; import org.apache.lucene.queryparser.classic.MultiFieldQueryParser; import org.apache.lucene.queryparser.classic.ParseException; -import org.apache.lucene.queryparser.classic.QueryParser; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreDoc; @@ -51,43 +56,50 @@ import org.apache.lucene.search.highlight.SimpleHTMLFormatter; import org.apache.lucene.search.highlight.TextFragment; import org.apache.lucene.search.highlight.TokenSources; import org.apache.lucene.store.Directory; +import org.apache.lucene.store.MMapDirectory; import org.apache.lucene.store.RAMDirectory; +import org.apache.zeppelin.conf.ZeppelinConfiguration; import org.apache.zeppelin.notebook.Note; import org.apache.zeppelin.notebook.Paragraph; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.google.common.base.Joiner; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.Lists; - /** - * Search (both, indexing and query) the notebooks using Lucene. - * - * Query is thread-safe, as creates new IndexReader every time. - * Index is thread-safe, as re-uses single IndexWriter, which is thread-safe. + * Search (both, indexing and query) the notebooks using Lucene. Query is thread-safe, as creates + * new IndexReader every time. Index is thread-safe, as re-uses single IndexWriter, which is + * thread-safe. */ public class LuceneSearch implements SearchService { - private static final Logger LOG = LoggerFactory.getLogger(LuceneSearch.class); + private static final Logger logger = LoggerFactory.getLogger(LuceneSearch.class); private static final String SEARCH_FIELD_TEXT = "contents"; private static final String SEARCH_FIELD_TITLE = "header"; - static final String PARAGRAPH = "paragraph"; - static final String ID_FIELD = "id"; - - Directory ramDirectory; - Analyzer analyzer; - IndexWriterConfig iwc; - IndexWriter writer; - - public LuceneSearch() { - ramDirectory = new RAMDirectory(); - analyzer = new StandardAnalyzer(); - iwc = new IndexWriterConfig(analyzer); + private static final String PARAGRAPH = "paragraph"; + private static final String ID_FIELD = "id"; + + private Directory directory; + private Path directoryPath; + private Analyzer analyzer; + private IndexWriterConfig indexWriterConfig; + private IndexWriter indexWriter; + + public LuceneSearch(ZeppelinConfiguration zeppelinConfiguration) { + try { + this.directoryPath = + Files.createTempDirectory( + Paths.get(zeppelinConfiguration.getZeppelinSearchTempPath()), "zeppelin-search-"); + this.directory = new MMapDirectory(directoryPath); + } catch (IOException e) { + logger.error( + "Failed to create temporary directory for search service. Use memory instead", e); + this.directory = new RAMDirectory(); + } + this.analyzer = new StandardAnalyzer(); + this.indexWriterConfig = new IndexWriterConfig(analyzer); try { - writer = new IndexWriter(ramDirectory, iwc); + this.indexWriter = new IndexWriter(directory, indexWriterConfig); } catch (IOException e) { - LOG.error("Failed to create new IndexWriter", e); + logger.error("Failed to create new IndexWriter", e); } } @@ -96,51 +108,49 @@ public class LuceneSearch implements SearchService { */ @Override public List<Map<String, String>> query(String queryStr) { - if (null == ramDirectory) { + if (null == directory) { throw new IllegalStateException( "Something went wrong on instance creation time, index dir is null"); } List<Map<String, String>> result = Collections.emptyList(); - try (IndexReader indexReader = DirectoryReader.open(ramDirectory)) { + try (IndexReader indexReader = DirectoryReader.open(directory)) { IndexSearcher indexSearcher = new IndexSearcher(indexReader); Analyzer analyzer = new StandardAnalyzer(); - MultiFieldQueryParser parser = new MultiFieldQueryParser( - new String[] {SEARCH_FIELD_TEXT, SEARCH_FIELD_TITLE}, - analyzer); + MultiFieldQueryParser parser = + new MultiFieldQueryParser(new String[] {SEARCH_FIELD_TEXT, SEARCH_FIELD_TITLE}, analyzer); Query query = parser.parse(queryStr); - LOG.debug("Searching for: " + query.toString(SEARCH_FIELD_TEXT)); + logger.debug("Searching for: " + query.toString(SEARCH_FIELD_TEXT)); SimpleHTMLFormatter htmlFormatter = new SimpleHTMLFormatter(); Highlighter highlighter = new Highlighter(htmlFormatter, new QueryScorer(query)); result = doSearch(indexSearcher, query, analyzer, highlighter); - indexReader.close(); } catch (IOException e) { - LOG.error("Failed to open index dir {}, make sure indexing finished OK", ramDirectory, e); + logger.error("Failed to open index dir {}, make sure indexing finished OK", directory, e); } catch (ParseException e) { - LOG.error("Failed to parse query " + queryStr, e); + logger.error("Failed to parse query " + queryStr, e); } return result; } - private List<Map<String, String>> doSearch(IndexSearcher searcher, Query query, - Analyzer analyzer, Highlighter highlighter) { + private List<Map<String, String>> doSearch( + IndexSearcher searcher, Query query, Analyzer analyzer, Highlighter highlighter) { List<Map<String, String>> matchingParagraphs = Lists.newArrayList(); ScoreDoc[] hits; try { hits = searcher.search(query, 20).scoreDocs; for (int i = 0; i < hits.length; i++) { - LOG.debug("doc={} score={}", hits[i].doc, hits[i].score); + logger.debug("doc={} score={}", hits[i].doc, hits[i].score); int id = hits[i].doc; Document doc = searcher.doc(id); String path = doc.get(ID_FIELD); if (path != null) { - LOG.debug((i + 1) + ". " + path); + logger.debug((i + 1) + ". " + path); String title = doc.get("title"); if (title != null) { - LOG.debug(" Title: {}", doc.get("title")); + logger.debug(" Title: {}", doc.get("title")); } String text = doc.get(SEARCH_FIELD_TEXT); @@ -148,34 +158,38 @@ public class LuceneSearch implements SearchService { String fragment = ""; if (text != null) { - TokenStream tokenStream = TokenSources.getTokenStream(searcher.getIndexReader(), id, - SEARCH_FIELD_TEXT, analyzer); + TokenStream tokenStream = + TokenSources.getTokenStream( + searcher.getIndexReader(), id, SEARCH_FIELD_TEXT, analyzer); TextFragment[] frag = highlighter.getBestTextFragments(tokenStream, text, true, 3); - LOG.debug(" {} fragments found for query '{}'", frag.length, query); + logger.debug(" {} fragments found for query '{}'", frag.length, query); for (int j = 0; j < frag.length; j++) { if ((frag[j] != null) && (frag[j].getScore() > 0)) { - LOG.debug(" Fragment: {}", frag[j].toString()); + logger.debug(" Fragment: {}", frag[j].toString()); } } fragment = (frag != null && frag.length > 0) ? frag[0].toString() : ""; } if (header != null) { - TokenStream tokenTitle = TokenSources.getTokenStream(searcher.getIndexReader(), id, - SEARCH_FIELD_TITLE, analyzer); + TokenStream tokenTitle = + TokenSources.getTokenStream( + searcher.getIndexReader(), id, SEARCH_FIELD_TITLE, analyzer); TextFragment[] frgTitle = highlighter.getBestTextFragments(tokenTitle, header, true, 3); header = (frgTitle != null && frgTitle.length > 0) ? frgTitle[0].toString() : ""; } else { header = ""; } - matchingParagraphs.add(ImmutableMap.of("id", path, // <noteId>/paragraph/<paragraphId> - "name", title, "snippet", fragment, "text", text, "header", header)); + matchingParagraphs.add( + ImmutableMap.of( + "id", path, // <noteId>/paragraph/<paragraphId> + "name", title, "snippet", fragment, "text", text, "header", header)); } else { - LOG.info("{}. No {} for this document", i + 1, ID_FIELD); + logger.info("{}. No {} for this document", i + 1, ID_FIELD); } } } catch (IOException | InvalidTokenOffsetsException e) { - LOG.error("Exception on searching for {}", query, e); + logger.error("Exception on searching for {}", query, e); } return matchingParagraphs; } @@ -186,7 +200,7 @@ public class LuceneSearch implements SearchService { @Override public void updateIndexDoc(Note note) throws IOException { updateIndexNoteName(note); - for (Paragraph p: note.getParagraphs()) { + for (Paragraph p : note.getParagraphs()) { updateIndexParagraph(note, p); } } @@ -194,9 +208,9 @@ public class LuceneSearch implements SearchService { private void updateIndexNoteName(Note note) throws IOException { String noteName = note.getName(); String noteId = note.getId(); - LOG.debug("Indexing Notebook {}, '{}'", noteId, noteName); + logger.debug("Indexing Notebook {}, '{}'", noteId, noteName); if (null == noteName || noteName.isEmpty()) { - LOG.debug("Skipping empty notebook name"); + logger.debug("Skipping empty notebook name"); return; } updateDoc(noteId, noteName, null); @@ -204,15 +218,15 @@ public class LuceneSearch implements SearchService { private void updateIndexParagraph(Note note, Paragraph p) throws IOException { if (p.getText() == null) { - LOG.debug("Skipping empty paragraph"); + logger.debug("Skipping empty paragraph"); return; } updateDoc(note.getId(), note.getName(), p); } /** - * Updates index for the given note: either note.name or a paragraph If - * paragraph is <code>null</code> - updates only for the note.name + * Updates index for the given note: either note.name or a paragraph If paragraph is <code>null + * </code> - updates only for the note.name * * @param noteId * @param noteName @@ -223,16 +237,16 @@ public class LuceneSearch implements SearchService { String id = formatId(noteId, p); Document doc = newDocument(id, noteName, p); try { - writer.updateDocument(new Term(ID_FIELD, id), doc); - writer.commit(); + indexWriter.updateDocument(new Term(ID_FIELD, id), doc); + indexWriter.commit(); } catch (IOException e) { - LOG.error("Failed to updaet index of notebook {}", noteId, e); + logger.error("Failed to updaet index of notebook {}", noteId, e); } } /** - * If paragraph is not null, id is <noteId>/paragraphs/<paragraphId>, - * otherwise it's just <noteId>. + * If paragraph is not null, id is <noteId>/paragraphs/<paragraphId>, otherwise it's just + * <noteId>. */ static String formatId(String noteId, Paragraph p) { String id = noteId; @@ -253,8 +267,7 @@ public class LuceneSearch implements SearchService { } /** - * If paragraph is not null, indexes code in the paragraph, otherwise indexes - * the notebook name. + * If paragraph is not null, indexes code in the paragraph, otherwise indexes the notebook name. * * @param id id of the document, different for Note name and paragraph * @param noteName name of the note @@ -294,15 +307,17 @@ public class LuceneSearch implements SearchService { docsIndexed++; } } catch (IOException e) { - LOG.error("Failed to index all Notebooks", e); + logger.error("Failed to index all Notebooks", e); } finally { try { // save what's been indexed, even if not full collection - writer.commit(); + indexWriter.commit(); } catch (IOException e) { - LOG.error("Failed to save index", e); + logger.error("Failed to save index", e); } long end = System.nanoTime(); - LOG.info("Indexing {} notebooks took {}ms", docsIndexed, + logger.info( + "Indexing {} notebooks took {}ms", + docsIndexed, TimeUnit.NANOSECONDS.toMillis(end - start)); } } @@ -314,9 +329,9 @@ public class LuceneSearch implements SearchService { public void addIndexDoc(Note note) { try { addIndexDocAsync(note); - writer.commit(); + indexWriter.commit(); } catch (IOException e) { - LOG.error("Failed to add note {} to index", note, e); + logger.error("Failed to add note {} to index", note, e); } } @@ -327,13 +342,13 @@ public class LuceneSearch implements SearchService { * @throws IOException */ private void addIndexDocAsync(Note note) throws IOException { - indexNoteName(writer, note.getId(), note.getName()); + indexNoteName(indexWriter, note.getId(), note.getName()); for (Paragraph doc : note.getParagraphs()) { if (doc.getText() == null) { - LOG.debug("Skipping empty paragraph"); + logger.debug("Skipping empty paragraph"); continue; } - indexDoc(writer, note.getId(), note.getName(), doc); + indexDoc(indexWriter, note.getId(), note.getName(), doc); } } @@ -356,18 +371,18 @@ public class LuceneSearch implements SearchService { private void deleteDoc(Note note, Paragraph p) { if (null == note) { - LOG.error("Trying to delete note by reference to NULL"); + logger.error("Trying to delete note by reference to NULL"); return; } String fullNoteOrJustParagraph = formatDeleteId(note.getId(), p); - LOG.debug("Deleting note {}, out of: {}", note.getId(), writer.numDocs()); + logger.debug("Deleting note {}, out of: {}", note.getId(), indexWriter.numDocs()); try { - writer.deleteDocuments(new WildcardQuery(new Term(ID_FIELD, fullNoteOrJustParagraph))); - writer.commit(); + indexWriter.deleteDocuments(new WildcardQuery(new Term(ID_FIELD, fullNoteOrJustParagraph))); + indexWriter.commit(); } catch (IOException e) { - LOG.error("Failed to delete {} from index by '{}'", note, fullNoteOrJustParagraph, e); + logger.error("Failed to delete {} from index by '{}'", note, fullNoteOrJustParagraph, e); } - LOG.debug("Done, index contains {} docs now" + writer.numDocs()); + logger.debug("Done, index contains {} docs now" + indexWriter.numDocs()); } /* (non-Javadoc) @@ -376,9 +391,12 @@ public class LuceneSearch implements SearchService { @Override public void close() { try { - writer.close(); + indexWriter.close(); + if (null != directoryPath) { + FileUtils.deleteDirectory(directoryPath.toFile()); + } } catch (IOException e) { - LOG.error("Failed to .close() the notebook index", e); + logger.error("Failed to .close() the notebook index", e); } } @@ -388,24 +406,19 @@ public class LuceneSearch implements SearchService { * @throws IOException */ private void indexNoteName(IndexWriter w, String noteId, String noteName) throws IOException { - LOG.debug("Indexing Notebook {}, '{}'", noteId, noteName); + logger.debug("Indexing Notebook {}, '{}'", noteId, noteName); if (null == noteName || noteName.isEmpty()) { - LOG.debug("Skipping empty notebook name"); + logger.debug("Skipping empty notebook name"); return; } indexDoc(w, noteId, noteName, null); } - /** - * Indexes a single document: - * - code of the paragraph (if non-null) - * - or just a note name - */ + /** Indexes a single document: - code of the paragraph (if non-null) - or just a note name */ private void indexDoc(IndexWriter w, String noteId, String noteName, Paragraph p) throws IOException { String id = formatId(noteId, p); Document doc = newDocument(id, noteName, p); w.addDocument(doc); } - } http://git-wip-us.apache.org/repos/asf/zeppelin/blob/c195b560/zeppelin-zengine/src/test/java/org/apache/zeppelin/search/LuceneSearchTest.java ---------------------------------------------------------------------- diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/search/LuceneSearchTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/search/LuceneSearchTest.java index 0ac4697..aeead7e 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/search/LuceneSearchTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/search/LuceneSearchTest.java @@ -17,14 +17,15 @@ package org.apache.zeppelin.search; import static com.google.common.truth.Truth.assertThat; -import static org.mockito.Mockito.*; import static org.apache.zeppelin.search.LuceneSearch.formatId; +import static org.mockito.Mockito.mock; +import com.google.common.base.Splitter; import java.io.IOException; import java.util.Arrays; import java.util.List; import java.util.Map; - +import org.apache.zeppelin.conf.ZeppelinConfiguration; import org.apache.zeppelin.interpreter.InterpreterFactory; import org.apache.zeppelin.interpreter.InterpreterSettingManager; import org.apache.zeppelin.notebook.Note; @@ -36,14 +37,12 @@ import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; -import com.google.common.base.Splitter; - public class LuceneSearchTest { private static NotebookRepo notebookRepoMock; private static InterpreterFactory interpreterFactory; private static InterpreterSettingManager interpreterSettingManager; - + private SearchService noteSearchService; private AuthenticationInfo anonymous; @@ -53,13 +52,13 @@ public class LuceneSearchTest { interpreterFactory = mock(InterpreterFactory.class); interpreterSettingManager = mock(InterpreterSettingManager.class); -// when(replLoaderMock.getInterpreterSettings()) -// .thenReturn(ImmutableList.<InterpreterSetting>of()); + // when(replLoaderMock.getInterpreterSettings()) + // .thenReturn(ImmutableList.<InterpreterSetting>of()); } @Before public void startUp() { - noteSearchService = new LuceneSearch(); + noteSearchService = new LuceneSearch(ZeppelinConfiguration.create()); anonymous = new AuthenticationInfo("anonymous"); } @@ -68,42 +67,45 @@ public class LuceneSearchTest { noteSearchService.close(); } - @Test public void canIndexNotebook() { - //give + @Test + public void canIndexNotebook() { + // give Note note1 = newNoteWithParagraph("Notebook1", "test"); Note note2 = newNoteWithParagraph("Notebook2", "not test"); List<Note> notebook = Arrays.asList(note1, note2); - //when + // when noteSearchService.addIndexDocs(notebook); } - @Test public void canIndexAndQuery() { - //given + @Test + public void canIndexAndQuery() { + // given Note note1 = newNoteWithParagraph("Notebook1", "test"); Note note2 = newNoteWithParagraphs("Notebook2", "not test", "not test at all"); noteSearchService.addIndexDocs(Arrays.asList(note1, note2)); - //when + // when List<Map<String, String>> results = noteSearchService.query("all"); - //then + // then assertThat(results).isNotEmpty(); assertThat(results.size()).isEqualTo(1); assertThat(results.get(0)) - .containsEntry("id", formatId(note2.getId(), note2.getLastParagraph())); + .containsEntry("id", formatId(note2.getId(), note2.getLastParagraph())); } - @Test public void canIndexAndQueryByNotebookName() { - //given + @Test + public void canIndexAndQueryByNotebookName() { + // given Note note1 = newNoteWithParagraph("Notebook1", "test"); Note note2 = newNoteWithParagraphs("Notebook2", "not test", "not test at all"); noteSearchService.addIndexDocs(Arrays.asList(note1, note2)); - //when + // when List<Map<String, String>> results = noteSearchService.query("Notebook1"); - //then + // then assertThat(results).isNotEmpty(); assertThat(results.size()).isEqualTo(1); assertThat(results.get(0)).containsEntry("id", note1.getId()); @@ -111,15 +113,15 @@ public class LuceneSearchTest { @Test public void canIndexAndQueryByParagraphTitle() { - //given + // given Note note1 = newNoteWithParagraph("Notebook1", "test", "testingTitleSearch"); Note note2 = newNoteWithParagraph("Notebook2", "not test", "notTestingTitleSearch"); noteSearchService.addIndexDocs(Arrays.asList(note1, note2)); - //when + // when List<Map<String, String>> results = noteSearchService.query("testingTitleSearch"); - //then + // then assertThat(results).isNotEmpty(); assertThat(results.size()).isAtLeast(1); int TitleHits = 0; @@ -131,41 +133,44 @@ public class LuceneSearchTest { assertThat(TitleHits).isAtLeast(1); } - @Test public void indexKeyContract() throws IOException { - //give + @Test + public void indexKeyContract() throws IOException { + // give Note note1 = newNoteWithParagraph("Notebook1", "test"); - //when + // when noteSearchService.addIndexDoc(note1); - //then - String id = resultForQuery("test").get(0).get(LuceneSearch.ID_FIELD); + // then + String id = resultForQuery("test").get(0).get("id"); // LuceneSearch.ID_FIELD - assertThat(Splitter.on("/").split(id)) //key structure <noteId>/paragraph/<paragraphId> - .containsAllOf(note1.getId(), LuceneSearch.PARAGRAPH, note1.getLastParagraph().getId()); + assertThat(Splitter.on("/").split(id)) // key structure <noteId>/paragraph/<paragraphId> + .containsAllOf( + note1.getId(), "paragraph", note1.getLastParagraph().getId()); // LuceneSearch.PARAGRAPH } - @Test //(expected=IllegalStateException.class) + @Test // (expected=IllegalStateException.class) public void canNotSearchBeforeIndexing() { - //given NO noteSearchService.index() was called - //when + // given NO noteSearchService.index() was called + // when List<Map<String, String>> result = noteSearchService.query("anything"); - //then + // then assertThat(result).isEmpty(); - //assert logs were printed - //"ERROR org.apache.zeppelin.search.SearchService:97 - Failed to open index dir RAMDirectory" + // assert logs were printed + // "ERROR org.apache.zeppelin.search.SearchService:97 - Failed to open index dir RAMDirectory" } - @Test public void canIndexAndReIndex() throws IOException { - //given + @Test + public void canIndexAndReIndex() throws IOException { + // given Note note1 = newNoteWithParagraph("Notebook1", "test"); Note note2 = newNoteWithParagraphs("Notebook2", "not test", "not test at all"); noteSearchService.addIndexDocs(Arrays.asList(note1, note2)); - //when + // when Paragraph p2 = note2.getLastParagraph(); p2.setText("test indeed"); noteSearchService.updateIndexDoc(note2); - //then + // then List<Map<String, String>> results = noteSearchService.query("all"); assertThat(results).isEmpty(); @@ -173,24 +178,26 @@ public class LuceneSearchTest { assertThat(results).isNotEmpty(); } - @Test public void canDeleteNull() throws IOException { - //give + @Test + public void canDeleteNull() throws IOException { + // give // looks like a bug in web UI: it tries to delete a note twice (after it has just been deleted) - //when + // when noteSearchService.deleteIndexDocs(null); } - @Test public void canDeleteFromIndex() throws IOException { - //given + @Test + public void canDeleteFromIndex() throws IOException { + // given Note note1 = newNoteWithParagraph("Notebook1", "test"); Note note2 = newNoteWithParagraphs("Notebook2", "not test", "not test at all"); noteSearchService.addIndexDocs(Arrays.asList(note1, note2)); assertThat(resultForQuery("Notebook2")).isNotEmpty(); - //when + // when noteSearchService.deleteIndexDocs(note2); - //then + // then assertThat(noteSearchService.query("all")).isEmpty(); assertThat(resultForQuery("Notebook2")).isEmpty(); @@ -199,43 +206,46 @@ public class LuceneSearchTest { assertThat(results.size()).isEqualTo(1); } - @Test public void indexParagraphUpdatedOnNoteSave() throws IOException { - //given: total 2 notebooks, 3 paragraphs + @Test + public void indexParagraphUpdatedOnNoteSave() throws IOException { + // given: total 2 notebooks, 3 paragraphs Note note1 = newNoteWithParagraph("Notebook1", "test"); Note note2 = newNoteWithParagraphs("Notebook2", "not test", "not test at all"); noteSearchService.addIndexDocs(Arrays.asList(note1, note2)); assertThat(resultForQuery("test").size()).isEqualTo(3); - //when + // when Paragraph p1 = note1.getLastParagraph(); p1.setText("no no no"); note1.persist(anonymous); - //then + // then assertThat(resultForQuery("Notebook1").size()).isEqualTo(1); List<Map<String, String>> results = resultForQuery("test"); assertThat(results).isNotEmpty(); assertThat(results.size()).isEqualTo(2); - //does not include Notebook1's paragraph any more - for (Map<String, String> result: results) { - assertThat(result.get("id").startsWith(note1.getId())).isFalse();; + // does not include Notebook1's paragraph any more + for (Map<String, String> result : results) { + assertThat(result.get("id").startsWith(note1.getId())).isFalse(); + ; } } - @Test public void indexNoteNameUpdatedOnNoteSave() throws IOException { - //given: total 2 notebooks, 3 paragraphs + @Test + public void indexNoteNameUpdatedOnNoteSave() throws IOException { + // given: total 2 notebooks, 3 paragraphs Note note1 = newNoteWithParagraph("Notebook1", "test"); Note note2 = newNoteWithParagraphs("Notebook2", "not test", "not test at all"); noteSearchService.addIndexDocs(Arrays.asList(note1, note2)); assertThat(resultForQuery("test").size()).isEqualTo(3); - //when + // when note1.setName("NotebookN"); note1.persist(anonymous); - //then + // then assertThat(resultForQuery("Notebook1")).isEmpty(); assertThat(resultForQuery("NotebookN")).isNotEmpty(); assertThat(resultForQuery("NotebookN").size()).isEqualTo(1); @@ -246,8 +256,7 @@ public class LuceneSearchTest { } /** - * Creates a new Note \w given name, - * adds a new paragraph \w given text + * Creates a new Note \w given name, adds a new paragraph \w given text * * @param noteName name of the note * @param parText text of the paragraph @@ -259,16 +268,13 @@ public class LuceneSearchTest { return note1; } - private Note newNoteWithParagraph(String noteName, String parText,String title) { + private Note newNoteWithParagraph(String noteName, String parText, String title) { Note note = newNote(noteName); addParagraphWithTextAndTitle(note, parText, title); return note; } - /** - * Creates a new Note \w given name, - * adds N paragraphs \w given texts - */ + /** Creates a new Note \w given name, adds N paragraphs \w given texts */ private Note newNoteWithParagraphs(String noteName, String... parTexts) { Note note1 = newNote(noteName); for (String parText : parTexts) { @@ -291,9 +297,16 @@ public class LuceneSearchTest { } private Note newNote(String name) { - Note note = new Note(notebookRepoMock, interpreterFactory, interpreterSettingManager, null, noteSearchService, null, null); + Note note = + new Note( + notebookRepoMock, + interpreterFactory, + interpreterSettingManager, + null, + noteSearchService, + null, + null); note.setName(name); return note; } - }