WAVE-311 Makes Solr base url and executor configurable.
Project: http://git-wip-us.apache.org/repos/asf/incubator-wave/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-wave/commit/c180e4fe Tree: http://git-wip-us.apache.org/repos/asf/incubator-wave/tree/c180e4fe Diff: http://git-wip-us.apache.org/repos/asf/incubator-wave/diff/c180e4fe Branch: refs/heads/fulltextsearch Commit: c180e4fed18c9b5b73e464e010551afc4b020161 Parents: ef9a9a9 Author: Yuri Zelikov <[email protected]> Authored: Thu Aug 21 21:34:38 2014 +0300 Committer: Yuri Zelikov <[email protected]> Committed: Wed Aug 27 20:50:29 2014 +0300 ---------------------------------------------------------------------- server-config.xml | 7 +++-- server.config.example | 7 +++++ .../waveprotocol/box/server/CoreSettings.java | 11 +++++++ .../server/executor/ExecutorAnnotations.java | 4 +++ .../box/server/executor/ExecutorsModule.java | 13 +++++++- .../waveserver/SolrSearchProviderImpl.java | 31 ++++---------------- .../server/waveserver/SolrWaveIndexerImpl.java | 31 +++++++++++--------- 7 files changed, 61 insertions(+), 43 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-wave/blob/c180e4fe/server-config.xml ---------------------------------------------------------------------- diff --git a/server-config.xml b/server-config.xml index 3fbc576..652a26d 100644 --- a/server-config.xml +++ b/server-config.xml @@ -47,6 +47,7 @@ <property name="lookup_executor_thread_count" value="1" /> <property name="robot_connection_thread_count" value="10" /> <property name="robot_gateway_thread_count" value="10" /> + <property name="solr_thread_count" value="1" /> <property name="disable_registration" value="false" /> <property name="enable_ssl" value="false" /> <property name="ssl_keystore_path" value="wiab.ks" /> @@ -60,9 +61,7 @@ <property name="analytics_account" value="" /> <property name="profile_fetcher_type" value="gravatar" /> <property name="enable_profiling" value="false" /> - <property name="mongodb_host" value="127.0.0.1" /> - <property name="mongodb_port" value="27017" /> - <property name="mongodb_database" value="wiab" /> + <property name="solr_base_url" value="http://localhost:8983/solr"; /> <!-- Properties for server.federation.config --> <property name="xmpp_server_secret" value="opensesame" /> @@ -134,6 +133,7 @@ <token key="LOOKUP_EXECUTOR_THREAD_COUNT" value="${lookup_executor_thread_count}" /> <token key="ROBOT_CONNECTION_THREAD_COUNT" value="${robot_connection_thread_count}" /> <token key="ROBOT_GATEWAY_THREAD_COUNT" value="${robot_gateway_thread_count}" /> + <token key="SOLR_THREAD_COUNT" value="${solr_thread_count}" /> <token key="DISABLE_REGISTRATION" value="${disable_registration}" /> <token key="ENABLE_SSL" value="${enable_ssl}" /> <token key="SSL_KEYSTORE_PATH" value="${ssl_keystore_path}" /> @@ -147,6 +147,7 @@ <token key="THUMBNAIL_PATTERNS_DIRECTORY" value="${thumbnail_patterns_directory}" /> <token key="PROFILE_FETCHER_TYPE" value="${profile_fetcher_type}" /> <token key="ENABLE_PROFILING" value="${enable_profiling}" /> + <token key="SOLR_BASE_URL" value="${solr_base_url}" /> <token key="MONGODB_HOST" value="${mongodb_host}" /> <token key="MONGODB_PORT" value="${mongodb_port}" /> <token key="MONGODB_DATABASE" value="${mongodb_database}" /> http://git-wip-us.apache.org/repos/asf/incubator-wave/blob/c180e4fe/server.config.example ---------------------------------------------------------------------- diff --git a/server.config.example b/server.config.example index ae515ab..5ca2a60 100644 --- a/server.config.example +++ b/server.config.example @@ -140,6 +140,9 @@ storage_continuation_executor_thread_count = @STORAGE_CONTINUATION_EXECUTOR_THRE # while creating a list of all wavelets in the persistent storage. Default value: 1 lookup_executor_thread_count = @LOOKUP_EXECUTOR_THREAD_COUNT@ +# The number of threads for making search/update requests to Solr. Default value: 1 +solr_thread_count = @SOLR_THREAD_COUNT@ + # To enable federation, edit the server.federation.config file and include it here. # Or run ant -f server-config.xml server-federation-config # If not using the server-config.xml ant script - it is possible just to comment the line. @@ -204,6 +207,10 @@ thumbnail_patterns_directory = @THUMBNAIL_PATTERNS_DIRECTORY@ # Default value: gravatar. profile_fetcher_type = @PROFILE_FETCHER_TYPE@ +# The Solr endpoint url. +# Default value: http://localhost:8983/solr +solr_base_url = @SOLR_BASE_URL@ + # Mongodb connection options # Only used if some of the *_store_type properties are set to 'mongodb' http://git-wip-us.apache.org/repos/asf/incubator-wave/blob/c180e4fe/src/org/waveprotocol/box/server/CoreSettings.java ---------------------------------------------------------------------- diff --git a/src/org/waveprotocol/box/server/CoreSettings.java b/src/org/waveprotocol/box/server/CoreSettings.java index 7a43829..5fbd345 100644 --- a/src/org/waveprotocol/box/server/CoreSettings.java +++ b/src/org/waveprotocol/box/server/CoreSettings.java @@ -61,6 +61,7 @@ public class CoreSettings { public static final String LOOKUP_EXECUTOR_THREAD_COUNT = "lookup_executor_thread_count"; public static final String ROBOT_CONNECTION_THREAD_COUNT = "robot_connection_thread_count"; public static final String ROBOT_GATEWAY_THREAD_COUNT = "robot_gateway_thread_count"; + public static final String SOLR_THREAD_COUNT = "solr_thread_count"; public static final String DISABLE_REGISTRATION = "disable_registration"; public static final String ENABLE_SSL = "enable_ssl"; public static final String SSL_KEYSTORE_PATH = "ssl_keystore_path"; @@ -77,6 +78,7 @@ public class CoreSettings { public static final String MONGODB_HOST = "mongodb_host"; public static final String MONGODB_PORT = "mongodb_port"; public static final String MONGODB_DATABASE = "mongodb_database"; + public static final String SOLR_BASE_URL = "solr_base_url"; @Setting(name = WAVE_SERVER_DOMAIN) private static String waveServerDomain; @@ -226,6 +228,11 @@ public class CoreSettings { defaultValue = "10") private static int robotGatewayThreadCount; + @Setting(name = SOLR_THREAD_COUNT, + description = "The number of threads to make search/update requests to Solr.", + defaultValue = "1") + private static int solrThreadCount; + @Setting(name = DISABLE_REGISTRATION, description = "Prevents the register page from being available to anyone", defaultValue = "false") private static boolean disableRegistration; @@ -290,4 +297,8 @@ public class CoreSettings { @Setting(name = MONGODB_DATABASE, description = "The database name used in the MongoDB server", defaultValue = "wiab") private static String mongoDBdatabase; + + @Setting(name = SOLR_BASE_URL, description = "The Solr base url", + defaultValue = "http://localhost:8983/solr";) + private static String solrBaseUrl; } http://git-wip-us.apache.org/repos/asf/incubator-wave/blob/c180e4fe/src/org/waveprotocol/box/server/executor/ExecutorAnnotations.java ---------------------------------------------------------------------- diff --git a/src/org/waveprotocol/box/server/executor/ExecutorAnnotations.java b/src/org/waveprotocol/box/server/executor/ExecutorAnnotations.java index 751389a..a002c76 100644 --- a/src/org/waveprotocol/box/server/executor/ExecutorAnnotations.java +++ b/src/org/waveprotocol/box/server/executor/ExecutorAnnotations.java @@ -84,4 +84,8 @@ public interface ExecutorAnnotations { @BindingAnnotation public @interface XmppExecutor { } + + @BindingAnnotation + public @interface SolrExecutor { + } } http://git-wip-us.apache.org/repos/asf/incubator-wave/blob/c180e4fe/src/org/waveprotocol/box/server/executor/ExecutorsModule.java ---------------------------------------------------------------------- diff --git a/src/org/waveprotocol/box/server/executor/ExecutorsModule.java b/src/org/waveprotocol/box/server/executor/ExecutorsModule.java index abf8ad6..9441b43 100644 --- a/src/org/waveprotocol/box/server/executor/ExecutorsModule.java +++ b/src/org/waveprotocol/box/server/executor/ExecutorsModule.java @@ -27,15 +27,18 @@ import com.google.inject.Provider; import com.google.inject.Provides; import com.google.inject.Singleton; import com.google.inject.name.Named; + import java.util.concurrent.Executor; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ThreadFactory; + import org.waveprotocol.box.server.CoreSettings; import org.waveprotocol.box.server.executor.ExecutorAnnotations.ClientServerExecutor; import org.waveprotocol.box.server.executor.ExecutorAnnotations.ContactExecutor; import org.waveprotocol.box.server.executor.ExecutorAnnotations.DeltaPersistExecutor; +import org.waveprotocol.box.server.executor.ExecutorAnnotations.SolrExecutor; import org.waveprotocol.box.server.executor.ExecutorAnnotations.XmppExecutor; import org.waveprotocol.box.server.executor.ExecutorAnnotations.IndexExecutor; import org.waveprotocol.box.server.executor.ExecutorAnnotations.ListenerExecutor; @@ -145,7 +148,15 @@ public class ExecutorsModule extends AbstractModule { protected ScheduledExecutorService provideXmppExecutor(Provider<ScheduledRequestScopeExecutor> executorProvider) { return provideScheduledThreadPoolExecutor(executorProvider, 1, XmppExecutor.class.getSimpleName()); } - + + @Provides + @Singleton + @SolrExecutor + protected Executor provideSolrxecutor(Provider<RequestScopeExecutor> executorProvider, + @Named(CoreSettings.SOLR_THREAD_COUNT) int threadCount) { + return provideThreadPoolExecutor(executorProvider, threadCount, SolrExecutor.class.getSimpleName()); + } + private Executor provideThreadPoolExecutor(Provider<RequestScopeExecutor> executorProvider, int threadCount, String name) { if (threadCount == 0) { http://git-wip-us.apache.org/repos/asf/incubator-wave/blob/c180e4fe/src/org/waveprotocol/box/server/waveserver/SolrSearchProviderImpl.java ---------------------------------------------------------------------- diff --git a/src/org/waveprotocol/box/server/waveserver/SolrSearchProviderImpl.java b/src/org/waveprotocol/box/server/waveserver/SolrSearchProviderImpl.java index 6eb91cd..137681e 100644 --- a/src/org/waveprotocol/box/server/waveserver/SolrSearchProviderImpl.java +++ b/src/org/waveprotocol/box/server/waveserver/SolrSearchProviderImpl.java @@ -78,11 +78,8 @@ public class SolrSearchProviderImpl extends AbstractSearchProviderImpl { public static final String CREATOR = "creator_t"; public static final String TEXT = "text_t"; public static final String IN = "in_ss"; - - /* - * TODO (Frank R.) make it configurable - */ - public static final String SOLR_BASE_URL = "http://localhost:8983/solr";; + + private final String solrBaseUrl; /*- * http://wiki.apache.org/solr/CommonQueryParameters#q @@ -126,8 +123,10 @@ public class SolrSearchProviderImpl extends AbstractSearchProviderImpl { @Inject public SolrSearchProviderImpl(WaveDigester digester, WaveMap waveMap, - @Named(CoreSettings.WAVE_SERVER_DOMAIN) String waveDomain) { + @Named(CoreSettings.WAVE_SERVER_DOMAIN) String waveDomain, + @Named(CoreSettings.SOLR_BASE_URL) String solrUrl) { super(waveDomain, digester, waveMap); + solrBaseUrl = solrUrl; } @Override @@ -135,10 +134,6 @@ public class SolrSearchProviderImpl extends AbstractSearchProviderImpl { LOG.fine("Search query '" + query + "' from user: " + user + " [" + startAt + ", " + ((startAt + numResults) - 1) + "]"); - /*- - * see - * org.waveprotocol.box.server.waveserver.SimpleSearchProviderImpl.search(ParticipantId, String, int, int).isAllQuery - */ // Maybe should be changed in case other folders in addition to 'inbox' are // added. final boolean isAllQuery = isAllQuery(query); @@ -159,7 +154,7 @@ public class SolrSearchProviderImpl extends AbstractSearchProviderImpl { GetMethod getMethod = new GetMethod(); try { while (true) { - getMethod.setURI(new URI(SOLR_BASE_URL + "/select?wt=json" + "&start=" + start + "&rows=" + getMethod.setURI(new URI(solrBaseUrl + "/select?wt=json" + "&start=" + start + "&rows=" + rows + "&sort=" + LMT + "+desc" + "&q=" + Q + "&fq=" + fq, false)); HttpClient httpClient = new HttpClient(); @@ -203,8 +198,6 @@ public class SolrSearchProviderImpl extends AbstractSearchProviderImpl { } } - ensureWavesHaveUserDataWavelet(currentUserWavesView, user); - Function<ReadableWaveletData, Boolean> matchesFunction = new Function<ReadableWaveletData, Boolean>() { @@ -236,18 +229,6 @@ public class SolrSearchProviderImpl extends AbstractSearchProviderImpl { return digester.generateSearchResult(user, query, searchResult); } - private void ensureWavesHaveUserDataWavelet( - LinkedHashMultimap<WaveId, WaveletId> currentUserWavesView, ParticipantId user) { - WaveletId udw = - WaveletId.of(user.getDomain(), - IdUtil.join(IdConstants.USER_DATA_WAVELET_PREFIX, user.getAddress())); - Set<WaveId> waveIds = currentUserWavesView.keySet(); - for (WaveId waveId : waveIds) { - Set<WaveletId> waveletIds = currentUserWavesView.get(waveId); - waveletIds.add(udw); - } - } - public static boolean isAllQuery(String query) { return !IN_PATTERN.matcher(query).find(); } http://git-wip-us.apache.org/repos/asf/incubator-wave/blob/c180e4fe/src/org/waveprotocol/box/server/waveserver/SolrWaveIndexerImpl.java ---------------------------------------------------------------------- diff --git a/src/org/waveprotocol/box/server/waveserver/SolrWaveIndexerImpl.java b/src/org/waveprotocol/box/server/waveserver/SolrWaveIndexerImpl.java index e9b6876..2a770c8 100644 --- a/src/org/waveprotocol/box/server/waveserver/SolrWaveIndexerImpl.java +++ b/src/org/waveprotocol/box/server/waveserver/SolrWaveIndexerImpl.java @@ -29,6 +29,7 @@ import com.google.gson.JsonObject; import com.google.gson.JsonPrimitive; import com.google.inject.Inject; import com.google.inject.Singleton; +import com.google.inject.name.Named; import org.apache.commons.httpclient.HttpClient; import org.apache.commons.httpclient.URI; @@ -39,6 +40,9 @@ import org.apache.commons.httpclient.methods.StringRequestEntity; import org.apache.http.HttpStatus; import org.waveprotocol.box.common.DeltaSequence; import org.waveprotocol.box.common.Snippets; +import org.waveprotocol.box.server.CoreSettings; +import org.waveprotocol.box.server.executor.ExecutorAnnotations.SolrExecutor; +import org.waveprotocol.box.server.executor.ExecutorAnnotations.WaveletLoadExecutor; import org.waveprotocol.box.server.robots.util.ConversationUtil; import org.waveprotocol.wave.model.document.operation.DocInitialization; import org.waveprotocol.wave.model.id.IdUtil; @@ -62,28 +66,27 @@ import java.util.logging.Level; */ @Singleton public class SolrWaveIndexerImpl extends AbstractWaveIndexer implements WaveBus.Subscriber, -PerUserWaveViewBus.Listener { + PerUserWaveViewBus.Listener { private static final Log LOG = Log.get(SolrWaveIndexerImpl.class); - // TODO (Yuri Z.): Inject executor. - private static final Executor executor = Executors.newSingleThreadExecutor(); - + private final Executor executor; private final ReadableWaveletDataProvider waveletDataProvider; + private final String solrBaseUrl; @Inject public SolrWaveIndexerImpl(WaveMap waveMap, WaveletProvider waveletProvider, ReadableWaveletDataProvider waveletDataProvider, ConversationUtil conversationUtil, - WaveletNotificationDispatcher notificationDispatcher) { - + WaveletNotificationDispatcher notificationDispatcher, + @Named(CoreSettings.SOLR_BASE_URL) String solrUrl, + @SolrExecutor Executor solrExecutor) { super(waveMap, waveletProvider); - + + executor = solrExecutor; + solrBaseUrl = solrUrl; this.waveletDataProvider = waveletDataProvider; - notificationDispatcher.subscribe(this); - - return; } @Override @@ -142,7 +145,7 @@ PerUserWaveViewBus.Listener { private void updateIndex(ReadableWaveletData wavelet) throws IndexException { Preconditions.checkNotNull(wavelet); - if (!IdUtil.isConversationalId(wavelet.getWaveletId())) { + if (IdUtil.isConversationalId(wavelet.getWaveletId())) { JsonArray docsJson = buildJsonDoc(wavelet); postUpdateToSolr(wavelet, docsJson); } @@ -150,7 +153,7 @@ PerUserWaveViewBus.Listener { private void postUpdateToSolr(ReadableWaveletData wavelet, JsonArray docsJson) { PostMethod postMethod = - new PostMethod(SolrSearchProviderImpl.SOLR_BASE_URL + "/update/json?commit=true"); + new PostMethod(solrBaseUrl + "/update/json?commit=true"); try { RequestEntity requestEntity = new StringRequestEntity(docsJson.toString(), "application/json", "UTF-8"); @@ -277,14 +280,14 @@ PerUserWaveViewBus.Listener { GetMethod getMethod = new GetMethod(); try { getMethod - .setURI(new URI(SolrSearchProviderImpl.SOLR_BASE_URL + "/update?wt=json" + .setURI(new URI(solrBaseUrl + "/update?wt=json" + "&stream.body=<delete><query>" + SolrSearchProviderImpl.Q + "</query></delete>", false)); HttpClient httpClient = new HttpClient(); int statusCode = httpClient.executeMethod(getMethod); if (statusCode == HttpStatus.SC_OK) { - getMethod.setURI(new URI(SolrSearchProviderImpl.SOLR_BASE_URL + "/update?wt=json" + getMethod.setURI(new URI(solrBaseUrl + "/update?wt=json" + "&stream.body=<commit/>", false)); httpClient = new HttpClient();
