This is an automated email from the ASF dual-hosted git repository. ilgrosso pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/syncope.git
commit 1b5e713f966dae2a2502771e96ab9b697e780fbc Author: Francesco Chicchiriccò <[email protected]> AuthorDate: Fri May 28 15:06:01 2021 +0200 Various Elasticsearch improvements and cleanup --- .../client/ElasticsearchClientFactoryBean.java | 74 ++++++++++++++++++++-- .../client/ElasticsearchIndexManager.java | 18 ++++-- .../elasticsearch/client/ElasticsearchUtils.java | 38 ++++++++--- .../core/persistence/jpa/DomainIndexLoader.java | 9 ++- .../jpa/dao/ElasticsearchAnySearchDAO.java | 21 ++++-- .../jpa/dao/ElasticsearchAnySearchDAOTest.java | 9 ++- .../java/job/ElasticsearchReindex.java | 47 +++++++++++--- pom.xml | 2 +- 8 files changed, 177 insertions(+), 41 deletions(-) diff --git a/ext/elasticsearch/client-elasticsearch/src/main/java/org/apache/syncope/ext/elasticsearch/client/ElasticsearchClientFactoryBean.java b/ext/elasticsearch/client-elasticsearch/src/main/java/org/apache/syncope/ext/elasticsearch/client/ElasticsearchClientFactoryBean.java index 7c35c99..39246a9 100644 --- a/ext/elasticsearch/client-elasticsearch/src/main/java/org/apache/syncope/ext/elasticsearch/client/ElasticsearchClientFactoryBean.java +++ b/ext/elasticsearch/client-elasticsearch/src/main/java/org/apache/syncope/ext/elasticsearch/client/ElasticsearchClientFactoryBean.java @@ -18,9 +18,19 @@ */ package org.apache.syncope.ext.elasticsearch.client; +import java.nio.charset.StandardCharsets; +import java.util.Base64; import java.util.List; +import org.apache.http.Header; +import org.apache.http.HttpHeaders; import org.apache.http.HttpHost; +import org.apache.http.auth.AuthScope; +import org.apache.http.auth.UsernamePasswordCredentials; +import org.apache.http.client.CredentialsProvider; +import org.apache.http.impl.client.BasicCredentialsProvider; +import org.apache.http.message.BasicHeader; import org.elasticsearch.client.RestClient; +import org.elasticsearch.client.RestClientBuilder; import org.elasticsearch.client.RestHighLevelClient; import org.springframework.beans.factory.DisposableBean; import org.springframework.beans.factory.FactoryBean; @@ -32,17 +42,74 @@ public class ElasticsearchClientFactoryBean implements FactoryBean<RestHighLevel private final List<HttpHost> hosts; + private String username; + + private String password; + + private String serviceToken; + + private String apiKeyId; + + private String apiKeySecret; + private RestHighLevelClient client; public ElasticsearchClientFactoryBean(final List<HttpHost> hosts) { this.hosts = hosts; } + public void setUsername(final String username) { + this.username = username; + } + + public void setPassword(final String password) { + this.password = password; + } + + public String getServiceToken() { + return serviceToken; + } + + public void setServiceToken(final String serviceToken) { + this.serviceToken = serviceToken; + } + + public String getApiKeyId() { + return apiKeyId; + } + + public void setApiKeyId(final String apiKeyId) { + this.apiKeyId = apiKeyId; + } + + public String getApiKeySecret() { + return apiKeySecret; + } + + public void setApiKeySecret(final String apiKeySecret) { + this.apiKeySecret = apiKeySecret; + } + @Override public RestHighLevelClient getObject() throws Exception { synchronized (this) { if (client == null) { - client = new RestHighLevelClient(RestClient.builder(hosts.toArray(new HttpHost[0]))); + RestClientBuilder restClient = RestClient.builder(hosts.toArray(new HttpHost[0])); + if (username != null && password != null) { + CredentialsProvider credentialsProvider = new BasicCredentialsProvider(); + credentialsProvider.setCredentials( + AuthScope.ANY, new UsernamePasswordCredentials(username, password)); + restClient.setHttpClientConfigCallback(b -> b.setDefaultCredentialsProvider(credentialsProvider)); + } else if (serviceToken != null) { + restClient.setDefaultHeaders( + new Header[] { new BasicHeader(HttpHeaders.AUTHORIZATION, "Bearer " + serviceToken) }); + } else if (apiKeyId != null && apiKeySecret != null) { + String apiKeyAuth = Base64.getEncoder().encodeToString( + (apiKeyId + ":" + apiKeySecret).getBytes(StandardCharsets.UTF_8)); + restClient.setDefaultHeaders( + new Header[] { new BasicHeader(HttpHeaders.AUTHORIZATION, "ApiKey " + apiKeyAuth) }); + } + client = new RestHighLevelClient(restClient); } } return client; @@ -54,11 +121,6 @@ public class ElasticsearchClientFactoryBean implements FactoryBean<RestHighLevel } @Override - public boolean isSingleton() { - return true; - } - - @Override public void destroy() throws Exception { if (client != null) { client.close(); diff --git a/ext/elasticsearch/client-elasticsearch/src/main/java/org/apache/syncope/ext/elasticsearch/client/ElasticsearchIndexManager.java b/ext/elasticsearch/client-elasticsearch/src/main/java/org/apache/syncope/ext/elasticsearch/client/ElasticsearchIndexManager.java index 8f32a48..69ebe5a 100644 --- a/ext/elasticsearch/client-elasticsearch/src/main/java/org/apache/syncope/ext/elasticsearch/client/ElasticsearchIndexManager.java +++ b/ext/elasticsearch/client-elasticsearch/src/main/java/org/apache/syncope/ext/elasticsearch/client/ElasticsearchIndexManager.java @@ -65,10 +65,8 @@ public class ElasticsearchIndexManager { new GetIndexRequest(ElasticsearchUtils.getContextDomainName(domain, kind)), RequestOptions.DEFAULT); } - public void createIndex(final String domain, final AnyTypeKind kind) - throws InterruptedException, ExecutionException, IOException { - - XContentBuilder settings = XContentFactory.jsonBuilder(). + public XContentBuilder defaultSettings() throws IOException { + return XContentFactory.jsonBuilder(). startObject(). startObject("analysis"). startObject("normalizer"). @@ -87,8 +85,10 @@ public class ElasticsearchIndexManager { field("number_of_replicas", elasticsearchUtils.getNumberOfReplicas()). endObject(). endObject(); + } - XContentBuilder mapping = XContentFactory.jsonBuilder(). + public XContentBuilder defaultMapping() throws IOException { + return XContentFactory.jsonBuilder(). startObject(). startArray("dynamic_templates"). startObject(). @@ -102,6 +102,14 @@ public class ElasticsearchIndexManager { endObject(). endArray(). endObject(); + } + + public void createIndex( + final String domain, + final AnyTypeKind kind, + final XContentBuilder settings, + final XContentBuilder mapping) + throws InterruptedException, ExecutionException, IOException { CreateIndexResponse response = client.indices().create( new CreateIndexRequest(ElasticsearchUtils.getContextDomainName(domain, kind)). diff --git a/ext/elasticsearch/client-elasticsearch/src/main/java/org/apache/syncope/ext/elasticsearch/client/ElasticsearchUtils.java b/ext/elasticsearch/client-elasticsearch/src/main/java/org/apache/syncope/ext/elasticsearch/client/ElasticsearchUtils.java index faea20a..47bc2c1 100644 --- a/ext/elasticsearch/client-elasticsearch/src/main/java/org/apache/syncope/ext/elasticsearch/client/ElasticsearchUtils.java +++ b/ext/elasticsearch/client-elasticsearch/src/main/java/org/apache/syncope/ext/elasticsearch/client/ElasticsearchUtils.java @@ -45,22 +45,26 @@ import org.springframework.transaction.annotation.Transactional; */ public class ElasticsearchUtils { + public static String getContextDomainName(final String domain, final AnyTypeKind kind) { + return domain.toLowerCase() + '_' + kind.name().toLowerCase(); + } + @Autowired - private UserDAO userDAO; + protected UserDAO userDAO; @Autowired - private GroupDAO groupDAO; + protected GroupDAO groupDAO; @Autowired - private AnyObjectDAO anyObjectDAO; + protected AnyObjectDAO anyObjectDAO; - private int indexMaxResultWindow = 10000; + protected int indexMaxResultWindow = 10000; - private int retryOnConflict = 5; + protected int retryOnConflict = 5; - private int numberOfShards = 1; + protected int numberOfShards = 1; - private int numberOfReplicas = 1; + protected int numberOfReplicas = 1; public void setIndexMaxResultWindow(final int indexMaxResultWindow) { this.indexMaxResultWindow = indexMaxResultWindow; @@ -143,6 +147,8 @@ public class ElasticsearchUtils { }); builder = builder.field("relationships", relationships); builder = builder.field("relationshipTypes", relationshipTypes); + + builder = customizeBuilder(builder, anyObject); } else if (any instanceof Group) { Group group = ((Group) any); builder = builder.field("name", group.getName()); @@ -160,6 +166,8 @@ public class ElasticsearchUtils { map(membership -> membership.getLeftEnd().getKey()).collect(Collectors.toList())); members.add(groupDAO.findADynMembers(group)); builder = builder.field("members", members); + + builder = customizeBuilder(builder, group); } else if (any instanceof User) { User user = ((User) any); builder = builder. @@ -192,6 +200,8 @@ public class ElasticsearchUtils { }); builder = builder.field("relationships", relationships); builder = builder.field("relationshipTypes", relationshipTypes); + + builder = customizeBuilder(builder, user); } for (PlainAttr<?> plainAttr : any.getPlainAttrs()) { @@ -208,7 +218,17 @@ public class ElasticsearchUtils { return builder.endObject(); } - public static String getContextDomainName(final String domain, final AnyTypeKind kind) { - return domain.toLowerCase() + '_' + kind.name().toLowerCase(); + protected XContentBuilder customizeBuilder(final XContentBuilder builder, final AnyObject anyObject) + throws IOException { + + return builder; + } + + protected XContentBuilder customizeBuilder(final XContentBuilder builder, final Group group) throws IOException { + return builder; + } + + protected XContentBuilder customizeBuilder(final XContentBuilder builder, final User user) throws IOException { + return builder; } } diff --git a/ext/elasticsearch/persistence-jpa/src/main/java/org/apache/syncope/core/persistence/jpa/DomainIndexLoader.java b/ext/elasticsearch/persistence-jpa/src/main/java/org/apache/syncope/core/persistence/jpa/DomainIndexLoader.java index 1697683..0a56061 100644 --- a/ext/elasticsearch/persistence-jpa/src/main/java/org/apache/syncope/core/persistence/jpa/DomainIndexLoader.java +++ b/ext/elasticsearch/persistence-jpa/src/main/java/org/apache/syncope/core/persistence/jpa/DomainIndexLoader.java @@ -45,13 +45,16 @@ public class DomainIndexLoader implements SyncopeCoreLoader { public void load(final String domain, final DataSource datasource) { try { if (!indexManager.existsIndex(domain, AnyTypeKind.USER)) { - indexManager.createIndex(domain, AnyTypeKind.USER); + indexManager.createIndex(domain, AnyTypeKind.USER, + indexManager.defaultSettings(), indexManager.defaultMapping()); } if (!indexManager.existsIndex(domain, AnyTypeKind.GROUP)) { - indexManager.createIndex(domain, AnyTypeKind.GROUP); + indexManager.createIndex(domain, AnyTypeKind.GROUP, + indexManager.defaultSettings(), indexManager.defaultMapping()); } if (!indexManager.existsIndex(domain, AnyTypeKind.ANY_OBJECT)) { - indexManager.createIndex(domain, AnyTypeKind.ANY_OBJECT); + indexManager.createIndex(domain, AnyTypeKind.ANY_OBJECT, + indexManager.defaultSettings(), indexManager.defaultMapping()); } } catch (Exception e) { LOG.error("While creating index for domain {}", domain, e); diff --git a/ext/elasticsearch/persistence-jpa/src/main/java/org/apache/syncope/core/persistence/jpa/dao/ElasticsearchAnySearchDAO.java b/ext/elasticsearch/persistence-jpa/src/main/java/org/apache/syncope/core/persistence/jpa/dao/ElasticsearchAnySearchDAO.java index 716ca14..2858436 100644 --- a/ext/elasticsearch/persistence-jpa/src/main/java/org/apache/syncope/core/persistence/jpa/dao/ElasticsearchAnySearchDAO.java +++ b/ext/elasticsearch/persistence-jpa/src/main/java/org/apache/syncope/core/persistence/jpa/dao/ElasticsearchAnySearchDAO.java @@ -23,7 +23,6 @@ import java.lang.reflect.Field; import java.util.ArrayList; import java.util.HashSet; import java.util.List; -import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; @@ -64,6 +63,7 @@ import org.elasticsearch.action.search.SearchType; import org.elasticsearch.client.RequestOptions; import org.elasticsearch.client.RestHighLevelClient; import org.elasticsearch.index.query.DisMaxQueryBuilder; +import org.elasticsearch.index.query.MatchAllQueryBuilder; import org.elasticsearch.index.query.MatchNoneQueryBuilder; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryBuilders; @@ -81,6 +81,8 @@ public class ElasticsearchAnySearchDAO extends AbstractAnySearchDAO { protected static final QueryBuilder MATCH_NONE_QUERY_BUILDER = new MatchNoneQueryBuilder(); + protected static final QueryBuilder MATCH_ALL_QUERY_BUILDER = new MatchAllQueryBuilder(); + @Autowired protected RestHighLevelClient client; @@ -88,7 +90,7 @@ public class ElasticsearchAnySearchDAO extends AbstractAnySearchDAO { protected ElasticsearchUtils elasticsearchUtils; protected Triple<Optional<QueryBuilder>, Set<String>, Set<String>> getAdminRealmsFilter( - final Set<String> adminRealms) { + final AnyTypeKind kind, final Set<String> adminRealms) { DisMaxQueryBuilder builder = QueryBuilders.disMaxQuery(); @@ -134,7 +136,7 @@ public class ElasticsearchAnySearchDAO extends AbstractAnySearchDAO { final int size, final List<SortBuilder<?>> sortBuilders) { - Triple<Optional<QueryBuilder>, Set<String>, Set<String>> filter = getAdminRealmsFilter(adminRealms); + Triple<Optional<QueryBuilder>, Set<String>, Set<String>> filter = getAdminRealmsFilter(kind, adminRealms); QueryBuilder queryBuilder; if (SyncopeConstants.FULL_ADMIN_REALMS.equals(adminRealms)) { queryBuilder = getQueryBuilder(cond, kind); @@ -228,8 +230,7 @@ public class ElasticsearchAnySearchDAO extends AbstractAnySearchDAO { return ArrayUtils.isEmpty(esResult) ? List.of() - : buildResult(Stream.of(Objects.requireNonNull(esResult)) - .map(SearchHit::getId).collect(Collectors.toList()), kind); + : buildResult(Stream.of(esResult).map(SearchHit::getId).collect(Collectors.toList()), kind); } protected QueryBuilder getQueryBuilder(final SearchCond cond, final AnyTypeKind kind) { @@ -314,9 +315,15 @@ public class ElasticsearchAnySearchDAO extends AbstractAnySearchDAO { } } + // allow for additional search conditions + if (builder == null) { + builder = getQueryBuilderForCustomConds(cond, kind); + } + if (builder == null) { builder = MATCH_NONE_QUERY_BUILDER; } + if (cond.getType() == SearchCond.Type.NOT_LEAF) { builder = QueryBuilders.boolQuery().mustNot(builder); } @@ -517,4 +524,8 @@ public class ElasticsearchAnySearchDAO extends AbstractAnySearchDAO { return fillAttrQuery(checked.getLeft(), checked.getMiddle(), checked.getRight()); } + + protected QueryBuilder getQueryBuilderForCustomConds(final SearchCond cond, final AnyTypeKind kind) { + return MATCH_ALL_QUERY_BUILDER; + } } diff --git a/ext/elasticsearch/persistence-jpa/src/test/java/org/apache/syncope/core/persistence/jpa/dao/ElasticsearchAnySearchDAOTest.java b/ext/elasticsearch/persistence-jpa/src/test/java/org/apache/syncope/core/persistence/jpa/dao/ElasticsearchAnySearchDAOTest.java index 6dd4d36..269967a 100644 --- a/ext/elasticsearch/persistence-jpa/src/test/java/org/apache/syncope/core/persistence/jpa/dao/ElasticsearchAnySearchDAOTest.java +++ b/ext/elasticsearch/persistence-jpa/src/test/java/org/apache/syncope/core/persistence/jpa/dao/ElasticsearchAnySearchDAOTest.java @@ -94,7 +94,8 @@ public class ElasticsearchAnySearchDAOTest { // 2. test Set<String> adminRealms = Set.of(SyncopeConstants.ROOT_REALM); - Triple<Optional<QueryBuilder>, Set<String>, Set<String>> filter = searchDAO.getAdminRealmsFilter(adminRealms); + Triple<Optional<QueryBuilder>, Set<String>, Set<String>> filter = + searchDAO.getAdminRealmsFilter(AnyTypeKind.USER, adminRealms); assertEquals( QueryBuilders.disMaxQuery().add(QueryBuilders.termQuery("realm", SyncopeConstants.ROOT_REALM)), filter.getLeft().get()); @@ -112,7 +113,8 @@ public class ElasticsearchAnySearchDAOTest { // 2. test Set<String> adminRealms = Set.of("dyn"); - Triple<Optional<QueryBuilder>, Set<String>, Set<String>> filter = searchDAO.getAdminRealmsFilter(adminRealms); + Triple<Optional<QueryBuilder>, Set<String>, Set<String>> filter = + searchDAO.getAdminRealmsFilter(AnyTypeKind.USER, adminRealms); assertFalse(filter.getLeft().isPresent()); assertEquals(Set.of("dyn"), filter.getMiddle()); assertEquals(Set.of(), filter.getRight()); @@ -121,7 +123,8 @@ public class ElasticsearchAnySearchDAOTest { @Test public void getAdminRealmsFilter_groupOwner() { Set<String> adminRealms = Set.of(RealmUtils.getGroupOwnerRealm("/any", "groupKey")); - Triple<Optional<QueryBuilder>, Set<String>, Set<String>> filter = searchDAO.getAdminRealmsFilter(adminRealms); + Triple<Optional<QueryBuilder>, Set<String>, Set<String>> filter = + searchDAO.getAdminRealmsFilter(AnyTypeKind.USER, adminRealms); assertFalse(filter.getLeft().isPresent()); assertEquals(Set.of(), filter.getMiddle()); assertEquals(Set.of("groupKey"), filter.getRight()); diff --git a/ext/elasticsearch/provisioning-java/src/main/java/org/apache/syncope/core/provisioning/java/job/ElasticsearchReindex.java b/ext/elasticsearch/provisioning-java/src/main/java/org/apache/syncope/core/provisioning/java/job/ElasticsearchReindex.java index 3423279..0e0c1e3 100644 --- a/ext/elasticsearch/provisioning-java/src/main/java/org/apache/syncope/core/provisioning/java/job/ElasticsearchReindex.java +++ b/ext/elasticsearch/provisioning-java/src/main/java/org/apache/syncope/core/provisioning/java/job/ElasticsearchReindex.java @@ -18,6 +18,7 @@ */ package org.apache.syncope.core.provisioning.java.job; +import java.io.IOException; import org.apache.syncope.common.lib.types.AnyTypeKind; import org.apache.syncope.core.persistence.api.dao.AnyDAO; import org.apache.syncope.core.persistence.api.dao.AnyObjectDAO; @@ -31,6 +32,7 @@ import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.index.IndexResponse; import org.elasticsearch.client.RequestOptions; import org.elasticsearch.client.RestHighLevelClient; +import org.elasticsearch.common.xcontent.XContentBuilder; import org.quartz.JobExecutionContext; import org.quartz.JobExecutionException; import org.springframework.beans.factory.annotation.Autowired; @@ -41,22 +43,46 @@ import org.springframework.beans.factory.annotation.Autowired; public class ElasticsearchReindex extends AbstractSchedTaskJobDelegate { @Autowired - private RestHighLevelClient client; + protected RestHighLevelClient client; @Autowired - private ElasticsearchIndexManager indexManager; + protected ElasticsearchIndexManager indexManager; @Autowired - private ElasticsearchUtils elasticsearchUtils; + protected ElasticsearchUtils elasticsearchUtils; @Autowired - private UserDAO userDAO; + protected UserDAO userDAO; @Autowired - private GroupDAO groupDAO; + protected GroupDAO groupDAO; @Autowired - private AnyObjectDAO anyObjectDAO; + protected AnyObjectDAO anyObjectDAO; + + protected XContentBuilder userSettings() throws IOException { + return indexManager.defaultSettings(); + } + + protected XContentBuilder groupSettings() throws IOException { + return indexManager.defaultSettings(); + } + + protected XContentBuilder anyObjectSettings() throws IOException { + return indexManager.defaultSettings(); + } + + protected XContentBuilder userMapping() throws IOException { + return indexManager.defaultMapping(); + } + + protected XContentBuilder groupMapping() throws IOException { + return indexManager.defaultMapping(); + } + + protected XContentBuilder anyObjectMapping() throws IOException { + return indexManager.defaultMapping(); + } @Override protected String doExecute(final boolean dryRun, final String executor, final JobExecutionContext context) @@ -69,17 +95,20 @@ public class ElasticsearchReindex extends AbstractSchedTaskJobDelegate { if (indexManager.existsIndex(AuthContextUtils.getDomain(), AnyTypeKind.USER)) { indexManager.removeIndex(AuthContextUtils.getDomain(), AnyTypeKind.USER); } - indexManager.createIndex(AuthContextUtils.getDomain(), AnyTypeKind.USER); + indexManager.createIndex( + AuthContextUtils.getDomain(), AnyTypeKind.USER, userSettings(), userMapping()); if (indexManager.existsIndex(AuthContextUtils.getDomain(), AnyTypeKind.GROUP)) { indexManager.removeIndex(AuthContextUtils.getDomain(), AnyTypeKind.GROUP); } - indexManager.createIndex(AuthContextUtils.getDomain(), AnyTypeKind.GROUP); + indexManager.createIndex( + AuthContextUtils.getDomain(), AnyTypeKind.GROUP, groupSettings(), groupMapping()); if (indexManager.existsIndex(AuthContextUtils.getDomain(), AnyTypeKind.ANY_OBJECT)) { indexManager.removeIndex(AuthContextUtils.getDomain(), AnyTypeKind.ANY_OBJECT); } - indexManager.createIndex(AuthContextUtils.getDomain(), AnyTypeKind.ANY_OBJECT); + indexManager.createIndex( + AuthContextUtils.getDomain(), AnyTypeKind.ANY_OBJECT, anyObjectSettings(), anyObjectMapping()); LOG.debug("Indexing users..."); for (int page = 1; page <= (userDAO.count() / AnyDAO.DEFAULT_PAGE_SIZE) + 1; page++) { diff --git a/pom.xml b/pom.xml index f106663..adc7a4b 100644 --- a/pom.xml +++ b/pom.xml @@ -477,7 +477,7 @@ under the License. <wicket-bootstrap.version>5.0.4</wicket-bootstrap.version> <wicket-spring-boot.version>3.0.4</wicket-spring-boot.version> - <netbeans.version>RELEASE122</netbeans.version> + <netbeans.version>RELEASE123</netbeans.version> <antlr4.version>4.9.2</antlr4.version>
