This is an automated email from the ASF dual-hosted git repository. asi pushed a commit to branch issue-505-v2 in repository https://gitbox.apache.org/repos/asf/unomi.git
commit 75d596532ab5c21280142fdc703cdebf64c0952f Author: Anatol Sialitski <[email protected]> AuthorDate: Wed Jun 15 18:35:58 2022 +0200 UNOMI-505 Study replication of existing profileIDs into new alias index --- .../unomi/shell/migration/impl/MigrationTo200.java | 112 ++++++++++++++++++++- .../requestBody/bulkSaveProfileAliases.ndjson | 2 + 2 files changed, 113 insertions(+), 1 deletion(-) diff --git a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/impl/MigrationTo200.java b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/impl/MigrationTo200.java index 9dc873b6e..732f8f795 100644 --- a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/impl/MigrationTo200.java +++ b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/impl/MigrationTo200.java @@ -19,15 +19,18 @@ package org.apache.unomi.shell.migration.impl; import org.apache.commons.io.IOUtils; import org.apache.http.HttpStatus; import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpEntityEnclosingRequestBase; import org.apache.http.client.methods.HttpGet; import org.apache.http.client.methods.HttpPost; import org.apache.http.client.methods.HttpPut; +import org.apache.http.client.methods.HttpUriRequest; import org.apache.http.entity.StringEntity; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.util.EntityUtils; import org.apache.karaf.shell.api.console.Session; import org.apache.unomi.shell.migration.Migration; import org.apache.unomi.shell.migration.utils.ConsoleUtils; +import org.json.JSONArray; import org.json.JSONObject; import org.osgi.framework.BundleContext; import org.osgi.framework.Version; @@ -35,8 +38,10 @@ import org.osgi.service.component.annotations.Component; import java.io.IOException; import java.io.InputStream; +import java.net.URI; import java.net.URL; import java.nio.charset.StandardCharsets; +import java.time.Instant; import java.util.Collections; import java.util.HashSet; import java.util.Set; @@ -65,7 +70,8 @@ public class MigrationTo200 implements Migration { public String getDescription() { return "Updates mapping for an index \"event\" with prefix \"context\" by default. Adds the \"sourceId\" field and copies value " + "from the \"scope\" field to it." - + "Creates the scope entries in the index \"scope\" from the existing sopes of the events"; + + "Creates the scope entries in the index \"scope\" from the existing scopes of the events. " + + "Creates the \"profileAlias\" documents based on \"profile\"."; } @Override @@ -86,6 +92,7 @@ public class MigrationTo200 implements Migration { } createScopeMapping(indexPrefix); createScopes(getSetOfScopes(indexes), indexPrefix); + createProfileAliasDocumentsFromProfile(); } private void updateMapping(final String indexName) throws IOException { @@ -246,6 +253,109 @@ public class MigrationTo200 implements Migration { return scopes; } + private void createProfileAliasDocumentsFromProfile() throws IOException { + System.out.println("Migration \"Create profileAlias from profile\" started"); + Instant migrationTime = Instant.now(); + int initialOffset = 1000; + int size = 1000; + doProcessResponse(migrationTime, initialOffset, size, null); + System.out.println("Migration \"Create profileAlias from profile\" completed."); + } + + private void doProcessResponse(Instant migrationTime, int initialOffset, int offset, String scrollId) throws IOException { + HttpUriRequest request = scrollId == null + ? createSearchRequest(offset) + : createSearchRequestWithScrollId(scrollId); + + try (CloseableHttpResponse response = httpClient.execute(request)) { + if (response.getStatusLine().getStatusCode() == HttpStatus.SC_OK) { + JSONObject responseAsJson = new JSONObject(EntityUtils.toString(response.getEntity())); + + if (responseAsJson.has("hits")) { + JSONObject hitsObject = responseAsJson.getJSONObject("hits"); + if (hitsObject.has("hits")) { + StringBuilder bulkCreateRequest = new StringBuilder(); + JSONArray hits = hitsObject.getJSONArray("hits"); + for (Object o : hits) { + JSONObject hit = (JSONObject) o; + if (hit.has("_source")) { + JSONObject hitSource = hit.getJSONObject("_source"); + if (hitSource.has("itemId")) { + String itemId = hitSource.getString("itemId"); + String bulkSaveProfileAliases = resourceAsString("requestBody/bulkSaveProfileAliases.ndjson"); + bulkCreateRequest.append(bulkSaveProfileAliases. + replace("$itemId", itemId). + replace("$migrationTime", migrationTime.toString())); + } + } + } + + CloseableHttpResponse bulkResponse = httpClient.execute(createProfileAliasRequest(bulkCreateRequest.toString())); + if (bulkResponse != null) { + bulkResponse.close(); + } + } + + if (hitsObject.getJSONObject("total").getInt("value") > offset) { + doProcessResponse(migrationTime, initialOffset, offset + initialOffset, responseAsJson.getString("_scroll_id")); + } else { + if (scrollId != null) { + CloseableHttpResponse deleteScrollResponse = httpClient.execute(createDeleteScrollRequest(scrollId)); + if (deleteScrollResponse != null) { + deleteScrollResponse.close(); + } + } + } + } + } + } + } + + private HttpPost createSearchRequestWithScrollId(final String scrollId) throws IOException { + final String requestBody = "{\n" + + " \"scroll_id\": \"" + scrollId + "\",\n" + + " \"scroll\": \"1h\"\n" + + "}"; + + final HttpPost request = new HttpPost(esAddress + "/_search/scroll"); + + request.addHeader("Accept", "application/json"); + request.addHeader("Content-Type", "application/json"); + request.setEntity(new StringEntity(requestBody)); + + return request; + } + + private HttpGet createSearchRequest(int size) { + return new HttpGet(esAddress + "/context-profile/_search?&scroll=1h&_source_includes=itemId&size=" + size); + } + + private HttpEntityEnclosingRequestBase createDeleteScrollRequest(final String scrollId) throws IOException { + final HttpEntityEnclosingRequestBase deleteRequest = new HttpEntityEnclosingRequestBase() { + @Override + public String getMethod() { + return "DELETE"; + } + }; + + deleteRequest.setURI(URI.create(esAddress + "/_search/scroll")); + deleteRequest.setEntity(new StringEntity("{ \"scroll_id\": \"" + scrollId + "\" }")); + deleteRequest.addHeader("Accept", "application/json"); + deleteRequest.addHeader("Content-Type", "application/json"); + + return deleteRequest; + } + + private HttpPost createProfileAliasRequest(String bulkRequestAsString) throws IOException { + final HttpPost bulkRequest = new HttpPost(esAddress + "/context-profilealias/_bulk"); + + bulkRequest.addHeader("Accept", "application/json"); + bulkRequest.addHeader("Content-Type", "application/json"); + bulkRequest.setEntity(new StringEntity(bulkRequestAsString)); + + return bulkRequest; + } + protected String resourceAsString(final String resource) { final URL url = bundleContext.getBundle().getResource(resource); try (InputStream stream = url.openStream()) { diff --git a/tools/shell-commands/src/main/resources/requestBody/bulkSaveProfileAliases.ndjson b/tools/shell-commands/src/main/resources/requestBody/bulkSaveProfileAliases.ndjson new file mode 100644 index 000000000..ad886ca35 --- /dev/null +++ b/tools/shell-commands/src/main/resources/requestBody/bulkSaveProfileAliases.ndjson @@ -0,0 +1,2 @@ +{ "create" : { "_id": "$itemId" }} +{ "itemId": "$itemId", "itemType": "profileAlias", "profileID": "$itemId", "scope": null, "clientID": "defaultClientId", "creationTime": "$migrationTime", "modifiedTime": "$migrationTime"}
