This is an automated email from the ASF dual-hosted git repository.
jkevan pushed a commit to branch reduceIndicesItems3
in repository https://gitbox.apache.org/repos/asf/unomi.git
The following commit(s) were added to refs/heads/reduceIndicesItems3 by this
push:
new 8361a522a UNOMI-736: provide base migration for indices reduction
8361a522a is described below
commit 8361a522a86b8fa296b979dba1944ac671cd7316
Author: Kevan <[email protected]>
AuthorDate: Mon Feb 27 18:10:37 2023 +0100
UNOMI-736: provide base migration for indices reduction
---
.../shell/migration/utils/MigrationUtils.java | 13 ++--
...-2.2.0-00-rolloverAndMigrateEventSession.groovy | 4 +-
.../migrate-2.2.0-05-indicesReduction.groovy | 78 ++++++++++++++++++++++
.../requestBody/2.2.0/base_reindex_request.json | 2 +-
.../requestBody/2.2.0/suffix_ids.painless | 19 ++++++
5 files changed, 108 insertions(+), 8 deletions(-)
diff --git
a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/utils/MigrationUtils.java
b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/utils/MigrationUtils.java
index a12fd6ec7..2050d6ba5 100644
---
a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/utils/MigrationUtils.java
+++
b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/utils/MigrationUtils.java
@@ -173,8 +173,8 @@ public class MigrationUtils {
return baseRequest.replace("#rolloverHotActions",
rolloverHotActions.toString());
}
- public static void moveToIndex(CloseableHttpClient httpClient,
BundleContext bundleContext, String esAddress, String sourceIndexName, String
targetIndexName) throws Exception {
- String reIndexRequest = resourceAsString(bundleContext,
"requestBody/2.2.0/base_reindex_request.json").replace("#source",
sourceIndexName).replace("#dest", targetIndexName);
+ public static void moveToIndex(CloseableHttpClient httpClient,
BundleContext bundleContext, String esAddress, String sourceIndexName, String
targetIndexName, String painlessScript) throws Exception {
+ String reIndexRequest = resourceAsString(bundleContext,
"requestBody/2.2.0/base_reindex_request.json").replace("#source",
sourceIndexName).replace("#dest", targetIndexName).replace("#painless",
StringUtils.isNotEmpty(painlessScript) ? getScriptPart(painlessScript) : "");
HttpUtils.executePostRequest(httpClient, esAddress + "/_reindex",
reIndexRequest, null);
}
@@ -225,10 +225,13 @@ public class MigrationUtils {
HttpUtils.executeDeleteRequest(httpClient, esAddress + "/" +
indexNameCloned, null);
}
});
- // Do a refresh
- HttpUtils.executePostRequest(httpClient, esAddress + "/" + indexName +
"/_refresh", null, null);
- waitForYellowStatus(httpClient, esAddress, migrationContext);
+ migrationContext.performMigrationStep("Reindex step for: " + indexName
+ " (refresh at the end)", () -> {
+ // Do a refresh
+ HttpUtils.executePostRequest(httpClient, esAddress + "/" +
indexName + "/_refresh", null, null);
+
+ waitForYellowStatus(httpClient, esAddress, migrationContext);
+ });
}
public static void scrollQuery(CloseableHttpClient httpClient, String
esAddress, String queryURL, String query, String scrollDuration, ScrollCallback
scrollCallback) throws IOException {
diff --git
a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.2.0-00-rolloverAndMigrateEventSession.groovy
b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.2.0-00-rolloverAndMigrateEventSession.groovy
index a04a4b097..c692fa7a2 100644
---
a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.2.0-00-rolloverAndMigrateEventSession.groovy
+++
b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.2.0-00-rolloverAndMigrateEventSession.groovy
@@ -58,7 +58,7 @@ Collections.sort(eventSortedIndices)
context.performMigrationStep("2.2.0-migrate-existing-events", () -> {
MigrationUtils.cleanAllIndexWithRollover(context.getHttpClient(),
bundleContext, esAddress, indexPrefix, "event")
eventSortedIndices.each { eventIndex ->
- MigrationUtils.moveToIndex(context.getHttpClient(), bundleContext,
esAddress, eventIndex, indexPrefix + "-event")
+ MigrationUtils.moveToIndex(context.getHttpClient(), bundleContext,
esAddress, eventIndex, indexPrefix + "-event", null)
sleep(3000)
}
})
@@ -86,7 +86,7 @@ Collections.sort(sessionSortedIndices)
context.performMigrationStep("2.2.0-migrate-existing-sessions", () -> {
MigrationUtils.cleanAllIndexWithRollover(context.getHttpClient(),
bundleContext, esAddress, indexPrefix, "session")
sessionSortedIndices.each { sessionIndex ->
- MigrationUtils.moveToIndex(context.getHttpClient(), bundleContext,
esAddress, sessionIndex, indexPrefix + "-session")
+ MigrationUtils.moveToIndex(context.getHttpClient(), bundleContext,
esAddress, sessionIndex, indexPrefix + "-session", null)
sleep(3000)
}
})
diff --git
a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.2.0-05-indicesReduction.groovy
b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.2.0-05-indicesReduction.groovy
new file mode 100644
index 000000000..8a07a5278
--- /dev/null
+++
b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.2.0-05-indicesReduction.groovy
@@ -0,0 +1,78 @@
+import org.apache.unomi.shell.migration.service.MigrationContext
+import org.apache.unomi.shell.migration.utils.HttpUtils
+import org.apache.unomi.shell.migration.utils.MigrationUtils
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+MigrationContext context = migrationContext
+String esAddress = context.getConfigString("esAddress")
+String indexPrefix = context.getConfigString("indexPrefix")
+String baseSettings = MigrationUtils.resourceAsString(bundleContext,
"requestBody/2.0.0/base_index_mapping.json")
+
+context.performMigrationStep("2.2.0-create-systemItems-index", () -> {
+ if (!MigrationUtils.indexExists(context.getHttpClient(), esAddress,
"${indexPrefix}-systemitems")) {
+ String mapping =
MigrationUtils.extractMappingFromBundles(bundleContext, "systemItems.json")
+ String newIndexSettings =
MigrationUtils.buildIndexCreationRequest(baseSettings, mapping, context, false)
+ HttpUtils.executePutRequest(context.getHttpClient(), esAddress +
"/${indexPrefix}-systemitems", newIndexSettings, null)
+ }
+})
+
+def indicesToReduce = [
+ actiontype: "systemitems",
+ campaign: "systemitems",
+ campaignevent: "systemitems",
+ goal: "systemitems",
+ userlist: "systemitems",
+ propertytype: "systemitems",
+ scope: "systemitems",
+ conditiontype: "systemitems",
+ rule: "systemitems",
+ scoring: "systemitems",
+ segment: "systemitems",
+ topic: "systemitems",
+ patch: "systemitems",
+ jsonschema: "systemitems",
+ importconfig: "systemitems",
+ exportconfig: "systemitems",
+ rulestats: "systemitems",
+ groovyaction: "systemitems",
+ persona: "profile",
+ personasession: "session"
+]
+def indicesToSuffixIds = [
+ rulestats: "-stat",
+ groovyaction: "-groovySourceCode"
+]
+indicesToReduce.each { indexToReduce ->
+ context.performMigrationStep("2.2.0-reduce-${indexToReduce.key}", () -> {
+ if (MigrationUtils.indexExists(context.getHttpClient(), esAddress,
"${indexPrefix}-${indexToReduce.key}")) {
+ def painless = null
+ // check if we need to update the ids of those items first
+ if (indicesToSuffixIds.containsKey(indexToReduce.key)) {
+ painless =
MigrationUtils.getFileWithoutComments(bundleContext,
"requestBody/2.2.0/suffix_ids.painless").replace("#ID_SUFFIX",
indicesToSuffixIds.get(indexToReduce.key))
+ }
+ // move items
+ MigrationUtils.moveToIndex(context.getHttpClient(), bundleContext,
esAddress, "${indexPrefix}-${indexToReduce.key}",
"${indexPrefix}-${indexToReduce.value}", painless)
+ MigrationUtils.deleteIndex(context.getHttpClient(), esAddress,
"${indexPrefix}-${indexToReduce.key}")
+ HttpUtils.executePostRequest(context.getHttpClient(), esAddress +
"/${indexPrefix}-${indexToReduce.value}/_refresh", null, null);
+ MigrationUtils.waitForYellowStatus(context.getHttpClient(),
esAddress, context);
+ }
+ })
+}
+
+
diff --git
a/tools/shell-commands/src/main/resources/requestBody/2.2.0/base_reindex_request.json
b/tools/shell-commands/src/main/resources/requestBody/2.2.0/base_reindex_request.json
index ddcb79a5e..589e71084 100644
---
a/tools/shell-commands/src/main/resources/requestBody/2.2.0/base_reindex_request.json
+++
b/tools/shell-commands/src/main/resources/requestBody/2.2.0/base_reindex_request.json
@@ -4,5 +4,5 @@
},
"dest": {
"index": "#dest"
- }
+ }#painless
}
diff --git
a/tools/shell-commands/src/main/resources/requestBody/2.2.0/suffix_ids.painless
b/tools/shell-commands/src/main/resources/requestBody/2.2.0/suffix_ids.painless
new file mode 100644
index 000000000..3b63549f6
--- /dev/null
+++
b/tools/shell-commands/src/main/resources/requestBody/2.2.0/suffix_ids.painless
@@ -0,0 +1,19 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ctx._id = ctx._id + '#ID_SUFFIX';
+ctx._source.itemId = ctx._id;
\ No newline at end of file
