This is an automated email from the ASF dual-hosted git repository.
sarath pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/atlas.git
The following commit(s) were added to refs/heads/master by this push:
new 6ad0853 ATLAS-3417 Improve Atlas Initialization Performance for fresh
installation.
6ad0853 is described below
commit 6ad08536043d6192de3b91b7799efbc60461222f
Author: skoritala <[email protected]>
AuthorDate: Mon Oct 21 12:03:11 2019 -0700
ATLAS-3417 Improve Atlas Initialization Performance for fresh installation.
After recent changes in atlas quick search improvements, Atlas is
taking long time to start on fresh installations.
This checkin tries to improve initialization time for fresh
installations.
Signed-off-by: Sarath Subramanian <[email protected]>
---
.../repository/graph/GraphBackedSearchIndexer.java | 27 ++++++++++++++++-
.../repository/graph/IndexChangeListener.java | 2 ++
.../atlas/repository/graph/SolrIndexHelper.java | 35 +++++++++++++++-------
3 files changed, 53 insertions(+), 11 deletions(-)
diff --git
a/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java
b/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java
index 22fae7f..2bd7b94 100755
---
a/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java
+++
b/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java
@@ -126,6 +126,7 @@ public class GraphBackedSearchIndexer implements
SearchIndexer, ActiveStateChang
if (!HAConfiguration.isHAEnabled(configuration)) {
initialize(provider.get());
}
+ notifyInitializationStart();
}
public void addIndexListener(IndexChangeListener listener) {
@@ -221,7 +222,7 @@ public class GraphBackedSearchIndexer implements
SearchIndexer, ActiveStateChang
//Commit indexes
commit(management);
- notifyChangeListeners(changedTypeDefs);
+ notifyInitializationCompletion(changedTypeDefs);
} catch (RepositoryException | IndexException e) {
LOG.error("Failed to update indexes for changed typedefs", e);
attemptRollback(changedTypeDefs, management);
@@ -967,4 +968,28 @@ public class GraphBackedSearchIndexer implements
SearchIndexer, ActiveStateChang
}
}
+ private void notifyInitializationStart() {
+ for (IndexChangeListener indexChangeListener : indexChangeListeners) {
+ try {
+ indexChangeListener.onInitStart();
+ } catch (Throwable t) {
+ LOG.error("Error encountered in notifying the index change
listener {}.", indexChangeListener.getClass().getName(), t);
+ //we need to throw exception if any of the listeners throw
execption.
+ throw new RuntimeException("Error encountered in notifying the
index change listener " + indexChangeListener.getClass().getName(), t);
+ }
+ }
+ }
+
+ private void notifyInitializationCompletion(ChangedTypeDefs
changedTypeDefs) {
+ for (IndexChangeListener indexChangeListener : indexChangeListeners) {
+ try {
+ indexChangeListener.onInitCompletion(changedTypeDefs);
+ } catch (Throwable t) {
+ LOG.error("Error encountered in notifying the index change
listener {}.", indexChangeListener.getClass().getName(), t);
+ //we need to throw exception if any of the listeners throw
execption.
+ throw new RuntimeException("Error encountered in notifying the
index change listener " + indexChangeListener.getClass().getName(), t);
+ }
+ }
+ }
+
}
diff --git
a/repository/src/main/java/org/apache/atlas/repository/graph/IndexChangeListener.java
b/repository/src/main/java/org/apache/atlas/repository/graph/IndexChangeListener.java
index b6e6257..4aa16dd 100644
---
a/repository/src/main/java/org/apache/atlas/repository/graph/IndexChangeListener.java
+++
b/repository/src/main/java/org/apache/atlas/repository/graph/IndexChangeListener.java
@@ -21,4 +21,6 @@ import org.apache.atlas.listener.ChangedTypeDefs;
public interface IndexChangeListener {
void onChange(ChangedTypeDefs changedTypeDefs);
+ void onInitStart();
+ void onInitCompletion(ChangedTypeDefs changedTypeDefs);
}
diff --git
a/repository/src/main/java/org/apache/atlas/repository/graph/SolrIndexHelper.java
b/repository/src/main/java/org/apache/atlas/repository/graph/SolrIndexHelper.java
index 26121fe..7310861 100644
---
a/repository/src/main/java/org/apache/atlas/repository/graph/SolrIndexHelper.java
+++
b/repository/src/main/java/org/apache/atlas/repository/graph/SolrIndexHelper.java
@@ -58,6 +58,7 @@ public class SolrIndexHelper implements IndexChangeListener {
public SolrIndexHelper(AtlasTypeRegistry typeRegistry) {
this.typeRegistry = typeRegistry;
}
+ public boolean initializationCompleted = false;
@Override
public void onChange(ChangedTypeDefs changedTypeDefs) {
@@ -65,18 +66,32 @@ public class SolrIndexHelper implements IndexChangeListener
{
changedTypeDefs == null || !changedTypeDefs.hasEntityDef()) { //
nothing to do if there are no changes to entity-defs
return;
}
+ if(initializationCompleted) {
+ try {
+ AtlasGraph graph =
AtlasGraphProvider.getGraphInstance();
+ AtlasGraphIndexClient graphIndexClient =
graph.getGraphIndexClient();
+ Map<String, Integer> indexFieldName2SearchWeightMap =
geIndexFieldNamesWithSearchWeights();
+
+ graphIndexClient.applySearchWeight(Constants.VERTEX_INDEX,
indexFieldName2SearchWeightMap);
+ graphIndexClient.applySuggestionFields(Constants.VERTEX_INDEX,
getIndexFieldNamesForSuggestions(indexFieldName2SearchWeightMap));
+ } catch (AtlasException e) {
+ LOG.error("Error encountered in handling type system change
notification.", e);
+ throw new RuntimeException("Error encountered in handling type
system change notification.", e);
+ }
+ }
+ }
- try {
- AtlasGraph graph =
AtlasGraphProvider.getGraphInstance();
- AtlasGraphIndexClient graphIndexClient =
graph.getGraphIndexClient();
- Map<String, Integer> indexFieldName2SearchWeightMap =
geIndexFieldNamesWithSearchWeights();
+ @Override
+ public void onInitStart() {
+ LOG.info("SolrIndexHelper Initialization started.");
+ initializationCompleted = false;
+ }
- graphIndexClient.applySearchWeight(Constants.VERTEX_INDEX,
indexFieldName2SearchWeightMap);
- graphIndexClient.applySuggestionFields(Constants.VERTEX_INDEX,
getIndexFieldNamesForSuggestions(indexFieldName2SearchWeightMap));
- } catch (AtlasException e) {
- LOG.error("Error encountered in handling type system change
notification.", e);
- throw new RuntimeException("Error encountered in handling type
system change notification.", e);
- }
+ @Override
+ public void onInitCompletion(ChangedTypeDefs changedTypeDefs) {
+ LOG.info("SolrIndexHelper Initialization completed.");
+ initializationCompleted = true;
+ onChange(changedTypeDefs);
}
private List<String> getIndexFieldNamesForSuggestions(Map<String, Integer>
indexFieldName2SearchWeightMap) {
