This is an automated email from the ASF dual-hosted git repository.
madhan pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/atlas.git
The following commit(s) were added to refs/heads/branch-2.0 by this push:
new cfca38d ATLAS-3654: enable support for solr in standalone mode
cfca38d is described below
commit cfca38d0ca39d6cc3abbc6d6f54ad1c9d3355045
Author: Damian Warszawski <[email protected]>
AuthorDate: Tue May 19 23:57:56 2020 +0200
ATLAS-3654: enable support for solr in standalone mode
Signed-off-by: Madhan Neethiraj <[email protected]>
(cherry picked from commit 6248e361fc01256ba263a6f4523c50541f6fc86b)
---
distro/src/bin/atlas_config.py | 13 +++-
.../src/documents/Setup/InstallationInstruction.md | 17 ++++-
.../graphdb/janus/AtlasJanusGraphIndexClient.java | 74 +++++++++++++++-------
.../janusgraph/diskstorage/solr/Solr6Index.java | 16 ++++-
4 files changed, 91 insertions(+), 29 deletions(-)
diff --git a/distro/src/bin/atlas_config.py b/distro/src/bin/atlas_config.py
index f09026f..2d61433 100755
--- a/distro/src/bin/atlas_config.py
+++ b/distro/src/bin/atlas_config.py
@@ -66,7 +66,9 @@ CONF_FILE="atlas-application.properties"
STORAGE_BACKEND_CONF="atlas.graph.storage.backend"
HBASE_STORAGE_LOCAL_CONF_ENTRY="atlas.graph.storage.hostname\s*=\s*localhost"
SOLR_INDEX_CONF_ENTRY="atlas.graph.index.search.backend\s*=\s*solr"
-SOLR_INDEX_LOCAL_CONF_ENTRY="atlas.graph.index.search.solr.zookeeper-url\s*=\s*localhost"
+SOLR_INDEX_MODE_CONF_ENTRY="atlas.graph.index.search.solr.mode"
+SOLR_INDEX_LOCAL_STANDALONE_CONF_ENTRY="atlas.graph.index.search.solr.http-urls\s*=(http|https)://localhost"
+SOLR_INDEX_LOCAL_CLOUD_CONF_ENTRY="atlas.graph.index.search.solr.zookeeper-url\s*=\s*localhost"
SOLR_INDEX_ZK_URL="atlas.graph.index.search.solr.zookeeper-url"
TOPICS_TO_CREATE="atlas.notification.topics"
ATLAS_HTTP_PORT="atlas.server.http.port"
@@ -453,7 +455,14 @@ def is_solr_local(confdir):
return False
confdir = os.path.join(confdir, CONF_FILE)
- return grep(confdir, SOLR_INDEX_CONF_ENTRY) is not None and grep(confdir,
SOLR_INDEX_LOCAL_CONF_ENTRY) is not None
+ return is_solr_local_cloud_mode(confdir) or
is_solr_local_standalone_mode(confdir)
+
+def is_solr_local_cloud_mode(confdir):
+ return grep(confdir, SOLR_INDEX_CONF_ENTRY) is not None and
getConfig(confdir, SOLR_INDEX_MODE_CONF_ENTRY) == 'cloud' and grep(confdir,
SOLR_INDEX_LOCAL_CLOUD_CONF_ENTRY) is not None
+
+
+def is_solr_local_standalone_mode(confdir):
+ return grep(confdir, SOLR_INDEX_CONF_ENTRY) is not None and
getConfig(confdir, SOLR_INDEX_MODE_CONF_ENTRY) == 'http' and grep(confdir,
SOLR_INDEX_LOCAL_STANDALONE_CONF_ENTRY) is not None
def is_elasticsearch_local():
if os.environ.get(MANAGE_LOCAL_ELASTICSEARCH, "False").lower() == 'false':
diff --git a/docs/src/documents/Setup/InstallationInstruction.md
b/docs/src/documents/Setup/InstallationInstruction.md
index d1b22d6..d22cb0c 100644
--- a/docs/src/documents/Setup/InstallationInstruction.md
+++ b/docs/src/documents/Setup/InstallationInstruction.md
@@ -150,7 +150,7 @@ By default, Apache Atlas uses JanusGraph as the graph
repository and is the only
* Start Apache Solr in cloud mode.
SolrCloud mode uses a ZooKeeper Service as a highly available, central
location for cluster management. For a small cluster, running with an existing
ZooKeeper quorum should be fine. For larger clusters, you would want to run
separate multiple ZooKeeper quorum with at least 3 servers.
- Note: Apache Atlas currently supports Apache Solr in "cloud" mode only.
"http" mode is not supported. For more information, refer Apache Solr
documentation - https://cwiki.apache.org/confluence/display/solr/SolrCloud
+ For more information, refer Apache Solr documentation -
https://cwiki.apache.org/confluence/display/solr/SolrCloud
* For e.g., to bring up an Apache Solr node listening on port 8983 on a
machine, you can use the command:
@@ -197,6 +197,21 @@ Pre-requisites for running Apache Solr in cloud mode
* SolrCloud has support for replication and sharding. It is highly
recommended to use SolrCloud with at least two Apache Solr nodes running on
different servers with replication enabled.
If using SolrCloud, then you also need ZooKeeper installed and configured
with 3 or 5 ZooKeeper nodes
+ * Start Apache Solr in http mode - alternative setup to Solr in cloud mode.
+
+ Solr Standalone is used for a single instance, and it keeps configuration
information on the file system. It does not require zookeeper and provides high
performance for medium size index.
+ Can be consider as a good option for fast prototyping as well as valid
configuration for development environments. In some cases it demonstrates a
better performance than solr cloud mode in production grade setup of Atlas.
+
+ * Change ATLAS configuration to point to Standalone Apache Solr instance
setup. Please make sure the following configurations are set to the below
values in ATLAS_HOME/conf/atlas-application.properties
+
+<SyntaxHighlighter wrapLines={true} language="powershell" style={theme.dark}>
+{`atlas.graph.index.search.backend=solr
+atlas.graph.index.search.solr.mode=http
+atlas.graph.index.search.solr.http-urls=<a single or list of URLs for the Solr
instances must be provided.> eg: localhost:2181,10.1.6.5:2181`}
+</SyntaxHighlighter>
+
+ Note: Solr standalone can be run in embedded mode using
`embedded-hbase-solr` profile.
+
*Configuring Elasticsearch as the indexing backend for the Graph Repository
(Tech Preview)*
By default, Apache Atlas uses [JanusGraph](https://janusgraph.org/) as the
graph repository and is the only graph repository implementation available
currently. For configuring [JanusGraph](https://janusgraph.org/) to work with
Elasticsearch, please follow the instructions below
diff --git
a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphIndexClient.java
b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphIndexClient.java
index ba65f3d..29bd2a4 100644
---
a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphIndexClient.java
+++
b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphIndexClient.java
@@ -32,14 +32,17 @@ import org.apache.commons.lang.StringUtils;
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.SolrRequest;
+import org.apache.solr.client.solrj.SolrResponse;
import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.client.solrj.request.GenericSolrRequest;
import org.apache.solr.client.solrj.request.V2Request;
-import org.apache.solr.client.solrj.response.V2Response;
-import org.apache.solr.common.util.NamedList;
import org.apache.solr.client.solrj.response.FacetField;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.client.solrj.response.TermsResponse;
+import org.apache.solr.client.solrj.util.ClientUtils;
import org.apache.solr.common.params.CommonParams;
+import org.apache.solr.common.util.ContentStream;
+import org.apache.solr.common.util.NamedList;
import org.janusgraph.diskstorage.solr.Solr6Index;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -79,6 +82,8 @@ public class AtlasJanusGraphIndexClient implements
AtlasGraphIndexClient {
return;
}
+ Solr6Index.Mode solrMode = Solr6Index.getSolrMode();
+
//1) try updating request handler
//2) if update fails, try creating request handler
@@ -100,7 +105,7 @@ public class AtlasJanusGraphIndexClient implements
AtlasGraphIndexClient {
try {
LOG.info("Attempting to update free text request handler
{} for collection {}", FREETEXT_REQUEST_HANDLER, collectionName);
- updateFreeTextRequestHandler(solrClient, collectionName,
indexFieldName2SearchWeightMap);
+ updateFreeTextRequestHandler(solrClient, collectionName,
indexFieldName2SearchWeightMap, solrMode);
LOG.info("Successfully updated free text request handler
{} for collection {}..", FREETEXT_REQUEST_HANDLER, collectionName);
@@ -114,7 +119,7 @@ public class AtlasJanusGraphIndexClient implements
AtlasGraphIndexClient {
try {
LOG.info("Attempting to create free text request handler
{} for collection {}", FREETEXT_REQUEST_HANDLER, collectionName);
- createFreeTextRequestHandler(solrClient, collectionName,
indexFieldName2SearchWeightMap);
+ createFreeTextRequestHandler(solrClient, collectionName,
indexFieldName2SearchWeightMap, solrMode);
LOG.info("Successfully created free text request handler
{} for collection {}", FREETEXT_REQUEST_HANDLER, collectionName);
return;
@@ -256,10 +261,13 @@ public class AtlasJanusGraphIndexClient implements
AtlasGraphIndexClient {
return;
}
+ Solr6Index.Mode solrMode = Solr6Index.getSolrMode();
+
//update the request handler
performRequestHandlerAction(collectionName,
solrClient,
-
generatePayLoadForSuggestions(generateSuggestionsString(suggestionProperties)));
+
generatePayLoadForSuggestions(generateSuggestionsString(suggestionProperties)),
+ solrMode);
} catch (Throwable t) {
String msg = String.format("Error encountered in creating the
request handler '%s' for collection '%s'", Constants.TERMS_REQUEST_HANDLER,
collectionName);
@@ -458,35 +466,53 @@ public class AtlasJanusGraphIndexClient implements
AtlasGraphIndexClient {
return ret.toString();
}
- private V2Response updateFreeTextRequestHandler(SolrClient solrClient,
String collectionName,
- Map<String, Integer>
indexFieldName2SearchWeightMap) throws IOException, SolrServerException,
AtlasBaseException {
+ private SolrResponse updateFreeTextRequestHandler(SolrClient solrClient,
String collectionName,
+ Map<String, Integer>
indexFieldName2SearchWeightMap,
+ Solr6Index.Mode mode)
throws IOException, SolrServerException, AtlasBaseException {
String searchWeightString =
generateSearchWeightString(indexFieldName2SearchWeightMap);
String payLoadString =
generatePayLoadForFreeText("update-requesthandler", searchWeightString);
- return performRequestHandlerAction(collectionName, solrClient,
payLoadString);
+ return performRequestHandlerAction(collectionName, solrClient,
payLoadString, mode);
}
- private V2Response createFreeTextRequestHandler(SolrClient solrClient,
String collectionName,
- Map<String, Integer>
indexFieldName2SearchWeightMap) throws IOException, SolrServerException,
AtlasBaseException {
+ private SolrResponse createFreeTextRequestHandler(SolrClient solrClient,
String collectionName,
+ Map<String, Integer>
indexFieldName2SearchWeightMap,
+ Solr6Index.Mode mode)
throws IOException, SolrServerException, AtlasBaseException {
String searchWeightString =
generateSearchWeightString(indexFieldName2SearchWeightMap);
String payLoadString =
generatePayLoadForFreeText("create-requesthandler", searchWeightString);
- return performRequestHandlerAction(collectionName, solrClient,
payLoadString);
+ return performRequestHandlerAction(collectionName, solrClient,
payLoadString, mode);
}
- private V2Response performRequestHandlerAction(String collectionName,
+ private SolrResponse performRequestHandlerAction(String collectionName,
SolrClient solrClient,
- String actionPayLoad)
throws IOException, SolrServerException, AtlasBaseException {
- V2Request v2Request = new
V2Request.Builder(String.format("/collections/%s/config", collectionName))
-
.withMethod(SolrRequest.METHOD.POST)
- .withPayload(actionPayLoad)
- .build();
+ String actionPayLoad,
+ Solr6Index.Mode mode)
throws IOException, SolrServerException, AtlasBaseException {
+ switch (mode) {
+ case CLOUD:
+ V2Request v2request = new
V2Request.Builder(String.format("/collections/%s/config", collectionName))
+
.withMethod(SolrRequest.METHOD.POST)
+
.withPayload(actionPayLoad)
+ .build();
+
+ return
validateResponseForSuccess(v2request.process(solrClient));
- return validateResponseForSuccess(v2Request.process(solrClient));
+ case HTTP:
+ Collection<ContentStream> contentStreams =
ClientUtils.toContentStreams(actionPayLoad, "application/json; charset=UTF-8");
+ GenericSolrRequest request = new
GenericSolrRequest(SolrRequest.METHOD.POST, String.format("/%s/config",
collectionName), null);
+
+ request.setContentStreams(contentStreams);
+ request.setUseV2(false);
+
+ return validateResponseForSuccess(request.process(solrClient));
+
+ default:
+ throw new IllegalArgumentException("Unsupported Solr operation
mode: " + mode);
+ }
}
- private V2Response validateResponseForSuccess(V2Response v2Response)
throws AtlasBaseException {
- if(v2Response == null) {
+ private SolrResponse validateResponseForSuccess(SolrResponse solrResponse)
throws AtlasBaseException {
+ if(solrResponse == null) {
String msg = "Received null response .";
LOG.error(msg);
@@ -495,10 +521,10 @@ public class AtlasJanusGraphIndexClient implements
AtlasGraphIndexClient {
}
if (LOG.isDebugEnabled()) {
- LOG.debug("V2 Response is {}", v2Response.toString());
+ LOG.debug("V2 Response is {}", solrResponse.toString());
}
- NamedList<Object> response = v2Response.getResponse();
+ NamedList<Object> response = solrResponse.getResponse();
if(response != null) {
Object errorMessages = response.get("errorMessages");
@@ -523,7 +549,7 @@ public class AtlasJanusGraphIndexClient implements
AtlasGraphIndexClient {
throw new AtlasBaseException(msg);
} else {
if(LOG.isDebugEnabled()) {
- LOG.debug("Successfully performed response handler action.
V2 Response is {}", v2Response.toString());
+ LOG.debug("Successfully performed response handler action.
V2 Response is {}", solrResponse.toString());
}
}
@@ -533,7 +559,7 @@ public class AtlasJanusGraphIndexClient implements
AtlasGraphIndexClient {
}
}
- return v2Response;
+ return solrResponse;
}
static final class TermFreq {
diff --git
a/graphdb/janus/src/main/java/org/janusgraph/diskstorage/solr/Solr6Index.java
b/graphdb/janus/src/main/java/org/janusgraph/diskstorage/solr/Solr6Index.java
index 484c161..6509fef 100644
---
a/graphdb/janus/src/main/java/org/janusgraph/diskstorage/solr/Solr6Index.java
+++
b/graphdb/janus/src/main/java/org/janusgraph/diskstorage/solr/Solr6Index.java
@@ -143,7 +143,7 @@ public class Solr6Index implements IndexProvider {
private static Solr6Index instance = null;
public static final ConfigOption<Boolean> CREATE_SOLR_CLIENT_PER_REQUEST =
new ConfigOption(SOLR_NS, "create-client-per-request", "when false, allows the
sharing of solr client across other components.",
org.janusgraph.diskstorage.configuration.ConfigOption.Type.LOCAL, false);
- private enum Mode {
+ public enum Mode {
HTTP, CLOUD;
public static Mode parse(String mode) {
@@ -183,7 +183,6 @@ public class Solr6Index implements IndexProvider {
private final boolean waitSearcher;
private final boolean kerberosEnabled;
-
public Solr6Index(final Configuration config) throws BackendException {
// Add Kerberos-enabled SolrHttpClientBuilder
HttpClientUtil.setHttpClientBuilder(new
Krb5HttpClientBuilder().getBuilder());
@@ -217,6 +216,19 @@ public class Solr6Index implements IndexProvider {
Solr6Index.instance = this;
}
+ public static Mode getSolrMode() {
+ Solr6Index solr6Index = Solr6Index.instance;
+ Mode ret = (solr6Index != null) ?
Mode.parse(solr6Index.configuration.get(SOLR_MODE)) : null;
+
+ if (ret == null) {
+ logger.warn("SolrMode is not set. Assuming {}", Mode.CLOUD);
+
+ ret = Mode.CLOUD;
+ }
+
+ return ret;
+ }
+
public static SolrClient getSolrClient() {
if (Solr6Index.instance != null) {
if (createSolrClientPerRequest) {
