This is an automated email from the ASF dual-hosted git repository.

asi pushed a commit to branch issue-505-v2
in repository https://gitbox.apache.org/repos/asf/unomi.git

commit 75d596532ab5c21280142fdc703cdebf64c0952f
Author: Anatol Sialitski <[email protected]>
AuthorDate: Wed Jun 15 18:35:58 2022 +0200

    UNOMI-505 Study replication of existing profileIDs into new alias index
---
 .../unomi/shell/migration/impl/MigrationTo200.java | 112 ++++++++++++++++++++-
 .../requestBody/bulkSaveProfileAliases.ndjson      |   2 +
 2 files changed, 113 insertions(+), 1 deletion(-)

diff --git 
a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/impl/MigrationTo200.java
 
b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/impl/MigrationTo200.java
index 9dc873b6e..732f8f795 100644
--- 
a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/impl/MigrationTo200.java
+++ 
b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/impl/MigrationTo200.java
@@ -19,15 +19,18 @@ package org.apache.unomi.shell.migration.impl;
 import org.apache.commons.io.IOUtils;
 import org.apache.http.HttpStatus;
 import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.methods.HttpEntityEnclosingRequestBase;
 import org.apache.http.client.methods.HttpGet;
 import org.apache.http.client.methods.HttpPost;
 import org.apache.http.client.methods.HttpPut;
+import org.apache.http.client.methods.HttpUriRequest;
 import org.apache.http.entity.StringEntity;
 import org.apache.http.impl.client.CloseableHttpClient;
 import org.apache.http.util.EntityUtils;
 import org.apache.karaf.shell.api.console.Session;
 import org.apache.unomi.shell.migration.Migration;
 import org.apache.unomi.shell.migration.utils.ConsoleUtils;
+import org.json.JSONArray;
 import org.json.JSONObject;
 import org.osgi.framework.BundleContext;
 import org.osgi.framework.Version;
@@ -35,8 +38,10 @@ import org.osgi.service.component.annotations.Component;
 
 import java.io.IOException;
 import java.io.InputStream;
+import java.net.URI;
 import java.net.URL;
 import java.nio.charset.StandardCharsets;
+import java.time.Instant;
 import java.util.Collections;
 import java.util.HashSet;
 import java.util.Set;
@@ -65,7 +70,8 @@ public class MigrationTo200 implements Migration {
     public String getDescription() {
         return "Updates mapping for an index \"event\" with prefix \"context\" 
by default. Adds the \"sourceId\" field and copies value "
                 + "from the \"scope\" field to it."
-                + "Creates the scope entries in the index \"scope\" from the 
existing sopes of the events";
+                + "Creates the scope entries in the index \"scope\" from the 
existing scopes of the events. "
+                + "Creates the \"profileAlias\" documents based on 
\"profile\".";
     }
 
     @Override
@@ -86,6 +92,7 @@ public class MigrationTo200 implements Migration {
         }
         createScopeMapping(indexPrefix);
         createScopes(getSetOfScopes(indexes), indexPrefix);
+        createProfileAliasDocumentsFromProfile();
     }
 
     private void updateMapping(final String indexName) throws IOException {
@@ -246,6 +253,109 @@ public class MigrationTo200 implements Migration {
         return scopes;
     }
 
+    private void createProfileAliasDocumentsFromProfile() throws IOException {
+        System.out.println("Migration \"Create profileAlias from profile\" 
started");
+        Instant migrationTime = Instant.now();
+        int initialOffset = 1000;
+        int size = 1000;
+        doProcessResponse(migrationTime, initialOffset, size, null);
+        System.out.println("Migration \"Create profileAlias from profile\" 
completed.");
+    }
+
+    private void doProcessResponse(Instant migrationTime, int initialOffset, 
int offset, String scrollId) throws IOException {
+        HttpUriRequest request = scrollId == null
+                ? createSearchRequest(offset)
+                : createSearchRequestWithScrollId(scrollId);
+
+        try (CloseableHttpResponse response = httpClient.execute(request)) {
+            if (response.getStatusLine().getStatusCode() == HttpStatus.SC_OK) {
+                JSONObject responseAsJson = new 
JSONObject(EntityUtils.toString(response.getEntity()));
+
+                if (responseAsJson.has("hits")) {
+                    JSONObject hitsObject = 
responseAsJson.getJSONObject("hits");
+                    if (hitsObject.has("hits")) {
+                        StringBuilder bulkCreateRequest = new StringBuilder();
+                        JSONArray hits = hitsObject.getJSONArray("hits");
+                        for (Object o : hits) {
+                            JSONObject hit = (JSONObject) o;
+                            if (hit.has("_source")) {
+                                JSONObject hitSource = 
hit.getJSONObject("_source");
+                                if (hitSource.has("itemId")) {
+                                    String itemId = 
hitSource.getString("itemId");
+                                    String bulkSaveProfileAliases = 
resourceAsString("requestBody/bulkSaveProfileAliases.ndjson");
+                                    
bulkCreateRequest.append(bulkSaveProfileAliases.
+                                            replace("$itemId", itemId).
+                                            replace("$migrationTime", 
migrationTime.toString()));
+                                }
+                            }
+                        }
+
+                        CloseableHttpResponse bulkResponse = 
httpClient.execute(createProfileAliasRequest(bulkCreateRequest.toString()));
+                        if (bulkResponse != null) {
+                            bulkResponse.close();
+                        }
+                    }
+
+                    if (hitsObject.getJSONObject("total").getInt("value") > 
offset) {
+                        doProcessResponse(migrationTime, initialOffset, offset 
+ initialOffset, responseAsJson.getString("_scroll_id"));
+                    } else {
+                        if (scrollId != null) {
+                            CloseableHttpResponse deleteScrollResponse = 
httpClient.execute(createDeleteScrollRequest(scrollId));
+                            if (deleteScrollResponse != null) {
+                                deleteScrollResponse.close();
+                            }
+                        }
+                    }
+                }
+            }
+        }
+    }
+
+    private HttpPost createSearchRequestWithScrollId(final String scrollId) 
throws IOException {
+        final String requestBody = "{\n" +
+                "  \"scroll_id\": \"" + scrollId + "\",\n" +
+                "  \"scroll\": \"1h\"\n" +
+                "}";
+
+        final HttpPost request = new HttpPost(esAddress + "/_search/scroll");
+
+        request.addHeader("Accept", "application/json");
+        request.addHeader("Content-Type", "application/json");
+        request.setEntity(new StringEntity(requestBody));
+
+        return request;
+    }
+
+    private HttpGet createSearchRequest(int size) {
+        return new HttpGet(esAddress + 
"/context-profile/_search?&scroll=1h&_source_includes=itemId&size=" + size);
+    }
+
+    private HttpEntityEnclosingRequestBase createDeleteScrollRequest(final 
String scrollId) throws IOException {
+        final HttpEntityEnclosingRequestBase deleteRequest = new 
HttpEntityEnclosingRequestBase() {
+            @Override
+            public String getMethod() {
+                return "DELETE";
+            }
+        };
+
+        deleteRequest.setURI(URI.create(esAddress + "/_search/scroll"));
+        deleteRequest.setEntity(new StringEntity("{ \"scroll_id\": \"" + 
scrollId + "\" }"));
+        deleteRequest.addHeader("Accept", "application/json");
+        deleteRequest.addHeader("Content-Type", "application/json");
+
+        return deleteRequest;
+    }
+
+    private HttpPost createProfileAliasRequest(String bulkRequestAsString) 
throws IOException {
+        final HttpPost bulkRequest = new HttpPost(esAddress + 
"/context-profilealias/_bulk");
+
+        bulkRequest.addHeader("Accept", "application/json");
+        bulkRequest.addHeader("Content-Type", "application/json");
+        bulkRequest.setEntity(new StringEntity(bulkRequestAsString));
+
+        return bulkRequest;
+    }
+
     protected String resourceAsString(final String resource) {
         final URL url = bundleContext.getBundle().getResource(resource);
         try (InputStream stream = url.openStream()) {
diff --git 
a/tools/shell-commands/src/main/resources/requestBody/bulkSaveProfileAliases.ndjson
 
b/tools/shell-commands/src/main/resources/requestBody/bulkSaveProfileAliases.ndjson
new file mode 100644
index 000000000..ad886ca35
--- /dev/null
+++ 
b/tools/shell-commands/src/main/resources/requestBody/bulkSaveProfileAliases.ndjson
@@ -0,0 +1,2 @@
+{ "create" : { "_id": "$itemId" }}
+{ "itemId":  "$itemId", "itemType":  "profileAlias", "profileID":  "$itemId", 
"scope":  null, "clientID": "defaultClientId", "creationTime":  
"$migrationTime", "modifiedTime":  "$migrationTime"}

Reply via email to