Author: mkataria
Date: Thu Jul 23 11:46:57 2020
New Revision: 1880198
URL: http://svn.apache.org/viewvc?rev=1880198&view=rev
Log:
OAK-9143: Use seed instead of reindexCount for elastic index suffix (Patch by
averma21)
Modified:
jackrabbit/oak/trunk/oak-benchmarks-elastic/src/main/java/org/apache/jackrabbit/oak/benchmark/util/TestHelper.java
jackrabbit/oak/trunk/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elastic/ElasticIndexDefinition.java
jackrabbit/oak/trunk/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elastic/ElasticIndexNameHelper.java
jackrabbit/oak/trunk/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elastic/index/ElasticIndexEditorContext.java
jackrabbit/oak/trunk/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elastic/index/ElasticIndexHelper.java
jackrabbit/oak/trunk/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elastic/index/ElasticIndexWriter.java
jackrabbit/oak/trunk/oak-search-elastic/src/test/java/org/apache/jackrabbit/oak/plugins/index/elastic/index/ElasticIndexHelperTest.java
Modified:
jackrabbit/oak/trunk/oak-benchmarks-elastic/src/main/java/org/apache/jackrabbit/oak/benchmark/util/TestHelper.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-benchmarks-elastic/src/main/java/org/apache/jackrabbit/oak/benchmark/util/TestHelper.java?rev=1880198&r1=1880197&r2=1880198&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-benchmarks-elastic/src/main/java/org/apache/jackrabbit/oak/benchmark/util/TestHelper.java
(original)
+++
jackrabbit/oak/trunk/oak-benchmarks-elastic/src/main/java/org/apache/jackrabbit/oak/benchmark/util/TestHelper.java
Thu Jul 23 11:46:57 2020
@@ -20,13 +20,18 @@ package org.apache.jackrabbit.oak.benchm
import org.apache.jackrabbit.oak.plugins.index.elastic.ElasticConnection;
import org.apache.jackrabbit.oak.plugins.index.elastic.ElasticIndexNameHelper;
+import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequest;
import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest;
import org.elasticsearch.action.support.master.AcknowledgedResponse;
+import org.elasticsearch.client.GetAliasesResponse;
import org.elasticsearch.client.RequestOptions;
+import org.elasticsearch.cluster.metadata.AliasMetadata;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
+import java.util.Map;
+import java.util.Set;
public class TestHelper {
@@ -45,15 +50,16 @@ public class TestHelper {
*/
public static void cleanupRemoteElastic(ElasticConnection connection,
String indexName) throws IOException {
String alias =
ElasticIndexNameHelper.getIndexAlias(connection.getIndexPrefix(), "/oak:index/"
+ indexName);
- /*
- Adding index suffix as -1 because reindex count will always be 1 here
(we are not doing any reindexing in the benchmark tests)
- TODO: If we write benchmarks for elastic reindex - this needs to be
changed to get the reindex count from the index def node
- */
- String remoteIndexName =
ElasticIndexNameHelper.getElasticSafeIndexName(alias + "-1");
- AcknowledgedResponse deleteIndexResponse =
connection.getClient().indices().
- delete(new DeleteIndexRequest(remoteIndexName),
RequestOptions.DEFAULT);
- if (!deleteIndexResponse.isAcknowledged()) {
- LOG.warn("Delete index call not acknowledged for index " +
remoteIndexName + " .Please check if remote index deleted or not.");
+ // get and delete the indexes which this alias is pointing to
+ GetAliasesRequest getAliasesRequest = new GetAliasesRequest(alias);
+ GetAliasesResponse aliasesResponse =
connection.getClient().indices().getAlias(getAliasesRequest,
RequestOptions.DEFAULT);
+ Map<String, Set<AliasMetadata>> aliases = aliasesResponse.getAliases();
+ for (String remoteIndexName : aliases.keySet()) {
+ AcknowledgedResponse deleteIndexResponse =
connection.getClient().indices().
+ delete(new DeleteIndexRequest(remoteIndexName),
RequestOptions.DEFAULT);
+ if (!deleteIndexResponse.isAcknowledged()) {
+ LOG.warn("Delete index call not acknowledged for index " +
remoteIndexName + " .Please check if remote index deleted or not.");
+ }
}
}
Modified:
jackrabbit/oak/trunk/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elastic/ElasticIndexDefinition.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elastic/ElasticIndexDefinition.java?rev=1880198&r1=1880197&r2=1880198&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elastic/ElasticIndexDefinition.java
(original)
+++
jackrabbit/oak/trunk/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elastic/ElasticIndexDefinition.java
Thu Jul 23 11:46:57 2020
@@ -52,6 +52,11 @@ public class ElasticIndexDefinition exte
public static final long BULK_RETRIES_BACKOFF_DEFAULT = 200;
/**
+ * Hidden property for storing a seed value to be used as suffix in remote
index name.
+ */
+ public static final String PROP_INDEX_NAME_SEED = ":nameSeed";
+
+ /**
* Node name under which various analyzers are configured
*/
private static final String ANALYZERS = "analyzers";
@@ -71,8 +76,6 @@ public class ElasticIndexDefinition exte
isAnalyzable = type -> Arrays.binarySearch(NOT_ANALYZED_TYPES, type) <
0;
}
- private final String remoteIndexName;
-
public final int bulkActions;
public final long bulkSizeBytes;
public final long bulkFlushIntervalMs;
@@ -84,9 +87,7 @@ public class ElasticIndexDefinition exte
public ElasticIndexDefinition(NodeState root, NodeState defn, String
indexPath, String indexPrefix) {
super(root, defn, determineIndexFormatVersion(defn),
determineUniqueId(defn), indexPath);
- String indexSuffix = "-" + getReindexCount();
this.remoteAlias = ElasticIndexNameHelper.getIndexAlias(indexPrefix !=
null ? indexPrefix : "", getIndexPath());
- this.remoteIndexName =
ElasticIndexNameHelper.getElasticSafeIndexName(this.remoteAlias + indexSuffix);
this.bulkActions = getOptionalValue(defn, BULK_ACTIONS,
BULK_ACTIONS_DEFAULT);
this.bulkSizeBytes = getOptionalValue(defn, BULK_SIZE_BYTES,
BULK_SIZE_BYTES_DEFAULT);
this.bulkFlushIntervalMs = getOptionalValue(defn,
BULK_FLUSH_INTERVAL_MS, BULK_FLUSH_INTERVAL_MS_DEFAULT);
@@ -109,16 +110,6 @@ public class ElasticIndexDefinition exte
return remoteAlias;
}
- /**
- * Returns the index identifier on the Elasticsearch cluster. Notice this
can be different from the value returned
- * from {@code getIndexName}. The index name shouldn't be used for index
read or updates. Alias obtained from {@link #getRemoteIndexAlias()}
- * should be used for such purposes.
- * @return the Elasticsearch index identifier
- */
- public String getRemoteIndexName() {
- return remoteIndexName;
- }
-
public Map<String, List<PropertyDefinition>> getPropertiesByName() {
return propertiesByName;
}
Modified:
jackrabbit/oak/trunk/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elastic/ElasticIndexNameHelper.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elastic/ElasticIndexNameHelper.java?rev=1880198&r1=1880197&r2=1880198&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elastic/ElasticIndexNameHelper.java
(original)
+++
jackrabbit/oak/trunk/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elastic/ElasticIndexNameHelper.java
Thu Jul 23 11:46:57 2020
@@ -19,6 +19,7 @@ package org.apache.jackrabbit.oak.plugin
import org.apache.jackrabbit.oak.commons.PathUtils;
+import java.util.UUID;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import java.util.stream.StreamSupport;
@@ -40,6 +41,26 @@ public class ElasticIndexNameHelper {
}
/**
+ * Create a name for remote elastic index from given index definition and
seed.
+ * @param indexDefinition elastic index definition to use
+ * @param seed seed to use
+ * @return remote elastic index name
+ */
+ public static String getRemoteIndexName(ElasticIndexDefinition
indexDefinition, long seed) {
+ return getElasticSafeIndexName(
+ indexDefinition.getRemoteIndexAlias() + "-" +
Long.toHexString(seed));
+ }
+
+ /**
+ * Create a name for remote elastic index from given index definition and
a randomly generated seed.
+ * @param indexDefinition elastic index definition to use
+ * @return remote elastic index name
+ */
+ public static String getRemoteIndexName(ElasticIndexDefinition
indexDefinition) {
+ return getRemoteIndexName(indexDefinition,
UUID.randomUUID().getMostSignificantBits());
+ }
+
+ /**
* <ul>
* <li>abc -> abc</li>
* <li>xy:abc -> xyabc</li>
@@ -48,7 +69,7 @@ public class ElasticIndexNameHelper {
* <p>
* The resulting file name would be truncated to MAX_NAME_LENGTH
*/
- public static String getElasticSafeIndexName(String indexPath) {
+ private static String getElasticSafeIndexName(String indexPath) {
String name = StreamSupport
.stream(PathUtils.elements(indexPath).spliterator(), false)
.limit(3) //Max 3 nodeNames including oak:index which is the
immediate parent for any indexPath
Modified:
jackrabbit/oak/trunk/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elastic/index/ElasticIndexEditorContext.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elastic/index/ElasticIndexEditorContext.java?rev=1880198&r1=1880197&r2=1880198&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elastic/index/ElasticIndexEditorContext.java
(original)
+++
jackrabbit/oak/trunk/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elastic/index/ElasticIndexEditorContext.java
Thu Jul 23 11:46:57 2020
@@ -28,6 +28,7 @@ import org.apache.jackrabbit.oak.spi.sta
import org.jetbrains.annotations.Nullable;
import java.io.IOException;
+import java.util.UUID;
class ElasticIndexEditorContext extends
FulltextIndexEditorContext<ElasticDocument> {
@@ -61,7 +62,10 @@ class ElasticIndexEditorContext extends
// Now, that index definition _might_ have been migrated by super
call, it would be ok to
// get writer and provision index settings and mappings
try {
- getWriter().provisionIndex();
+ long seed = UUID.randomUUID().getMostSignificantBits();
+ // merge gets called on node store later in the indexing flow
+
definitionBuilder.setProperty(ElasticIndexDefinition.PROP_INDEX_NAME_SEED,
seed);
+ getWriter().provisionIndex(seed);
} catch (IOException e) {
throw new IllegalStateException("Unable to provision index", e);
}
Modified:
jackrabbit/oak/trunk/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elastic/index/ElasticIndexHelper.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elastic/index/ElasticIndexHelper.java?rev=1880198&r1=1880197&r2=1880198&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elastic/index/ElasticIndexHelper.java
(original)
+++
jackrabbit/oak/trunk/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elastic/index/ElasticIndexHelper.java
Thu Jul 23 11:46:57 2020
@@ -34,8 +34,8 @@ import java.util.stream.Collectors;
*/
class ElasticIndexHelper {
- public static CreateIndexRequest createIndexRequest(ElasticIndexDefinition
indexDefinition) throws IOException {
- final CreateIndexRequest request = new
CreateIndexRequest(indexDefinition.getRemoteIndexName());
+ public static CreateIndexRequest createIndexRequest(String
remoteIndexName, ElasticIndexDefinition indexDefinition) throws IOException {
+ final CreateIndexRequest request = new
CreateIndexRequest(remoteIndexName);
// provision settings
request.settings(loadSettings(indexDefinition));
Modified:
jackrabbit/oak/trunk/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elastic/index/ElasticIndexWriter.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elastic/index/ElasticIndexWriter.java?rev=1880198&r1=1880197&r2=1880198&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elastic/index/ElasticIndexWriter.java
(original)
+++
jackrabbit/oak/trunk/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elastic/index/ElasticIndexWriter.java
Thu Jul 23 11:46:57 2020
@@ -18,6 +18,7 @@ package org.apache.jackrabbit.oak.plugin
import org.apache.jackrabbit.oak.plugins.index.elastic.ElasticConnection;
import org.apache.jackrabbit.oak.plugins.index.elastic.ElasticIndexDefinition;
+import org.apache.jackrabbit.oak.plugins.index.elastic.ElasticIndexNameHelper;
import org.apache.jackrabbit.oak.plugins.index.elastic.util.ElasticIndexUtils;
import
org.apache.jackrabbit.oak.plugins.index.search.spi.editor.FulltextIndexWriter;
import org.apache.jackrabbit.oak.spi.commit.CommitInfo;
@@ -97,21 +98,21 @@ class ElasticIndexWriter implements Full
return bulkProcessorHandler.close();
}
- protected void provisionIndex() throws IOException {
+ protected void provisionIndex(long seed) throws IOException {
// check if index already exists
+ final String indexName =
ElasticIndexNameHelper.getRemoteIndexName(indexDefinition, seed);
boolean exists = elasticConnection.getClient().indices().exists(
- new GetIndexRequest(indexDefinition.getRemoteIndexName()),
RequestOptions.DEFAULT
+ new GetIndexRequest(indexName), RequestOptions.DEFAULT
);
if (exists) {
- LOG.info("Index {} already exists. Skip index provision",
indexDefinition.getRemoteIndexName());
+ LOG.info("Index {} already exists. Skip index provision",
indexName);
return;
}
final IndicesClient indicesClient =
elasticConnection.getClient().indices();
- final String indexName = indexDefinition.getRemoteIndexName();
// create the new index
- final CreateIndexRequest request =
ElasticIndexHelper.createIndexRequest(indexDefinition);
+ final CreateIndexRequest request =
ElasticIndexHelper.createIndexRequest(indexName, indexDefinition);
try {
if (LOG.isDebugEnabled()) {
final String requestMsg =
Strings.toString(request.toXContent(jsonBuilder(), EMPTY_PARAMS));
@@ -119,7 +120,7 @@ class ElasticIndexWriter implements Full
}
CreateIndexResponse response = indicesClient.create(request,
RequestOptions.DEFAULT);
LOG.info("Updated settings for index {}. Response acknowledged:
{}",
- indexDefinition.getRemoteIndexName(),
response.isAcknowledged());
+ indexName, response.isAcknowledged());
checkResponseAcknowledgement(response, "Create index call not
acknowledged for index " + indexName);
} catch (ElasticsearchStatusException ese) {
// We already check index existence as first thing in this method,
if we get here it means we have got into
Modified:
jackrabbit/oak/trunk/oak-search-elastic/src/test/java/org/apache/jackrabbit/oak/plugins/index/elastic/index/ElasticIndexHelperTest.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-search-elastic/src/test/java/org/apache/jackrabbit/oak/plugins/index/elastic/index/ElasticIndexHelperTest.java?rev=1880198&r1=1880197&r2=1880198&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-search-elastic/src/test/java/org/apache/jackrabbit/oak/plugins/index/elastic/index/ElasticIndexHelperTest.java
(original)
+++
jackrabbit/oak/trunk/oak-search-elastic/src/test/java/org/apache/jackrabbit/oak/plugins/index/elastic/index/ElasticIndexHelperTest.java
Thu Jul 23 11:46:57 2020
@@ -19,6 +19,7 @@ package org.apache.jackrabbit.oak.plugin
import com.fasterxml.jackson.databind.ObjectMapper;
import org.apache.jackrabbit.oak.api.Tree;
import org.apache.jackrabbit.oak.plugins.index.elastic.ElasticIndexDefinition;
+import org.apache.jackrabbit.oak.plugins.index.elastic.ElasticIndexNameHelper;
import
org.apache.jackrabbit.oak.plugins.index.elastic.util.ElasticIndexDefinitionBuilder;
import
org.apache.jackrabbit.oak.plugins.index.search.util.IndexDefinitionBuilder;
import org.apache.jackrabbit.oak.spi.state.NodeState;
@@ -47,7 +48,7 @@ public class ElasticIndexHelperTest {
ElasticIndexDefinition definition =
new ElasticIndexDefinition(nodeState, nodeState, "path",
"prefix");
- CreateIndexRequest request =
ElasticIndexHelper.createIndexRequest(definition);
+ CreateIndexRequest request =
ElasticIndexHelper.createIndexRequest(ElasticIndexNameHelper.getRemoteIndexName(definition),
definition);
ObjectMapper mapper = new ObjectMapper();
Map<String, Object> jsonMap =
mapper.readValue(request.mappings().streamInput(), Map.class);
@@ -70,7 +71,7 @@ public class ElasticIndexHelperTest {
ElasticIndexDefinition definition =
new ElasticIndexDefinition(nodeState, nodeState, "path",
"prefix");
- ElasticIndexHelper.createIndexRequest(definition);
+
ElasticIndexHelper.createIndexRequest(ElasticIndexNameHelper.getRemoteIndexName(definition),
definition);
}
@Test
@@ -85,7 +86,7 @@ public class ElasticIndexHelperTest {
ElasticIndexDefinition definition =
new ElasticIndexDefinition(nodeState, nodeState, "path",
"prefix");
- CreateIndexRequest request =
ElasticIndexHelper.createIndexRequest(definition);
+ CreateIndexRequest request =
ElasticIndexHelper.createIndexRequest(ElasticIndexNameHelper.getRemoteIndexName(definition),
definition);
assertThat(request.settings().get("analysis.filter.oak_word_delimiter_graph_filter.preserve_original"),
is("false"));
@@ -113,7 +114,7 @@ public class ElasticIndexHelperTest {
ElasticIndexDefinition definition =
new ElasticIndexDefinition(nodeState, nodeState, "path",
"prefix");
- CreateIndexRequest request =
ElasticIndexHelper.createIndexRequest(definition);
+ CreateIndexRequest request =
ElasticIndexHelper.createIndexRequest(ElasticIndexNameHelper.getRemoteIndexName(definition),
definition);
assertThat(request.settings().get("analysis.filter.oak_word_delimiter_graph_filter.preserve_original"),
is("true"));
}
@@ -130,7 +131,7 @@ public class ElasticIndexHelperTest {
ElasticIndexDefinition definition =
new ElasticIndexDefinition(nodeState, nodeState, "path",
"prefix");
- CreateIndexRequest request =
ElasticIndexHelper.createIndexRequest(definition);
+ CreateIndexRequest request =
ElasticIndexHelper.createIndexRequest(ElasticIndexNameHelper.getRemoteIndexName(definition),
definition);
assertThat(request.settings().get("analysis.filter.shingle.type"),
is("shingle"));
assertThat(request.settings().get("analysis.analyzer.trigram.type"),
is("custom"));
