jkevan commented on code in PR #427:
URL: https://github.com/apache/unomi/pull/427#discussion_r878247541
##########
tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/impl/MigrationTo200.java:
##########
@@ -126,25 +113,88 @@ private void copyValueScopeToSourceId(final String
indexName, final CloseableHtt
httpPost.addHeader("Accept", "application/json");
httpPost.addHeader("Content-Type", "application/json");
- String request = "{\n" +
- " \"script\": {\n" +
- " \"source\": \"ctx._source.sourceId =
ctx._source.scope\",\n" +
- " \"lang\": \"painless\"\n" +
- " }\n" +
- "}";
+ String request = "{\n" + " \"script\": {\n" + " \"source\":
\"ctx._source.sourceId = ctx._source.scope\",\n"
+ + " \"lang\": \"painless\"\n" + " }\n" + "}";
httpPost.setEntity(new StringEntity(request));
try (CloseableHttpResponse response = httpClient.execute(httpPost)) {
JSONObject responseAsJson = new
JSONObject(EntityUtils.toString(response.getEntity()));
if (response.getStatusLine().getStatusCode() == HttpStatus.SC_OK) {
- System.out.println("Copying the \"scope\" field to the
\"sourceId\" field for index = \"" + indexName + "\" successfully completed.
Total: " +
- responseAsJson.get("total") + ", updated: " +
responseAsJson.get("updated") + ".");
+ System.out.println("Copying the \"scope\" field to the
\"sourceId\" field for index = \"" + indexName
+ + "\" successfully completed. Total: " +
responseAsJson.get("total") + ", updated: " + responseAsJson.get("updated")
+ + ".");
} else {
System.out.println("Copying the \"scope\" field to the
\"sourceId\" field for index = \"" + indexName + "\" failed.");
}
}
}
+ private Set<String> getEventIndexes(String indexPrefix) throws IOException
{
+ try (CloseableHttpResponse response = httpClient.execute(new
HttpGet(esAddress + "/_aliases"))) {
+ if (response.getStatusLine().getStatusCode() == HttpStatus.SC_OK) {
+ JSONObject indexesAsJson = new
JSONObject(EntityUtils.toString(response.getEntity()));
+ return indexesAsJson.keySet().stream().
+ filter(alias -> alias.startsWith(indexPrefix +
"-event")).
+ collect(Collectors.toSet());
+ }
+ }
+ return Collections.emptySet();
+ }
+
+ private void createScopes(Set<String> scopes, String indexPrefix) throws
IOException {
+ final StringBuilder body = new StringBuilder();
+ scopes.forEach(scope -> {
+ body.append("{\"index\": {\"_id\": \"" + scope + "\"}}\n");
+ body.append("{\"itemId\": \"" + scope + "\", \"itemType\":
\"scope\", \"metadata\": { \"id\": \"" + scope + "\" }}\n");
+ });
+ final HttpPost httpPost = new HttpPost(esAddress + "/" + indexPrefix +
"-scope/_bulk");
+
+ httpPost.addHeader("Accept", "application/json");
+ httpPost.addHeader("Content-Type", "application/x-ndjson");
+
+ httpPost.setEntity(new StringEntity(body.toString()));
+
+ try (CloseableHttpResponse response = httpClient.execute(httpPost)) {
+ if (response.getStatusLine().getStatusCode() == HttpStatus.SC_OK) {
+ System.out.println("Creating the \"scopes\" into the index " +
indexPrefix + "-scope successfully finished");
+ } else {
+ System.out.println("Creating the \"scopes\" into the index " +
indexPrefix + "-scope has failed" + response.getStatusLine()
+ .getStatusCode());
+ }
+ }
+ }
+
+ private Set<String> getSetOfScopes(Set<String> indices) throws IOException
{
+ String joinedIndices = String.join(",", indices);
+ final HttpPost httpPost = new HttpPost(esAddress + "/" + joinedIndices
+ "/_search");
+
+ httpPost.addHeader("Accept", "application/json");
+ httpPost.addHeader("Content-Type", "application/json");
+
+ String request =
+ "{\n" + " \"_source\": false,\n" + " \"size\": 0,\n" + "
\"aggs\": {\n" + " \"scopes\": {\n" + " \"terms\": {\n"
+ + " \"field\": \"scope.keyword\"\n" + "
}\n" + " },\n" + " \"bucketInfos\": {\n"
+ + " \"stats_bucket\": {\n" + "
\"buckets_path\": \"scopes._count\"\n" + " }\n" + " }\n"
+ + " }\n" + "}";
Review Comment:
same
##########
tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/impl/MigrationTo200.java:
##########
@@ -126,25 +113,88 @@ private void copyValueScopeToSourceId(final String
indexName, final CloseableHtt
httpPost.addHeader("Accept", "application/json");
httpPost.addHeader("Content-Type", "application/json");
- String request = "{\n" +
- " \"script\": {\n" +
- " \"source\": \"ctx._source.sourceId =
ctx._source.scope\",\n" +
- " \"lang\": \"painless\"\n" +
- " }\n" +
- "}";
+ String request = "{\n" + " \"script\": {\n" + " \"source\":
\"ctx._source.sourceId = ctx._source.scope\",\n"
+ + " \"lang\": \"painless\"\n" + " }\n" + "}";
httpPost.setEntity(new StringEntity(request));
try (CloseableHttpResponse response = httpClient.execute(httpPost)) {
JSONObject responseAsJson = new
JSONObject(EntityUtils.toString(response.getEntity()));
if (response.getStatusLine().getStatusCode() == HttpStatus.SC_OK) {
- System.out.println("Copying the \"scope\" field to the
\"sourceId\" field for index = \"" + indexName + "\" successfully completed.
Total: " +
- responseAsJson.get("total") + ", updated: " +
responseAsJson.get("updated") + ".");
+ System.out.println("Copying the \"scope\" field to the
\"sourceId\" field for index = \"" + indexName
+ + "\" successfully completed. Total: " +
responseAsJson.get("total") + ", updated: " + responseAsJson.get("updated")
+ + ".");
} else {
System.out.println("Copying the \"scope\" field to the
\"sourceId\" field for index = \"" + indexName + "\" failed.");
}
}
}
+ private Set<String> getEventIndexes(String indexPrefix) throws IOException
{
+ try (CloseableHttpResponse response = httpClient.execute(new
HttpGet(esAddress + "/_aliases"))) {
+ if (response.getStatusLine().getStatusCode() == HttpStatus.SC_OK) {
+ JSONObject indexesAsJson = new
JSONObject(EntityUtils.toString(response.getEntity()));
+ return indexesAsJson.keySet().stream().
+ filter(alias -> alias.startsWith(indexPrefix +
"-event")).
+ collect(Collectors.toSet());
+ }
+ }
+ return Collections.emptySet();
+ }
+
+ private void createScopes(Set<String> scopes, String indexPrefix) throws
IOException {
+ final StringBuilder body = new StringBuilder();
Review Comment:
Careful here it seem's you forgot to create the scope index with the mapping
provided in:
persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/scope.json
##########
tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/impl/MigrationTo200.java:
##########
@@ -126,25 +113,88 @@ private void copyValueScopeToSourceId(final String
indexName, final CloseableHtt
httpPost.addHeader("Accept", "application/json");
httpPost.addHeader("Content-Type", "application/json");
- String request = "{\n" +
- " \"script\": {\n" +
- " \"source\": \"ctx._source.sourceId =
ctx._source.scope\",\n" +
- " \"lang\": \"painless\"\n" +
- " }\n" +
- "}";
+ String request = "{\n" + " \"script\": {\n" + " \"source\":
\"ctx._source.sourceId = ctx._source.scope\",\n"
+ + " \"lang\": \"painless\"\n" + " }\n" + "}";
Review Comment:
same
##########
tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/impl/MigrationTo200.java:
##########
@@ -67,50 +73,31 @@ public void execute(Session session, CloseableHttpClient
httpClient, String esAd
}
private void doExecute() throws IOException {
- try (CloseableHttpResponse response = httpClient.execute(new
HttpGet(esAddress + "/_aliases"))) {
-
- if (response.getStatusLine().getStatusCode() == HttpStatus.SC_OK) {
- JSONObject indicesAsJson = new
JSONObject(EntityUtils.toString(response.getEntity()));
-
- final Set<String> indices = indicesAsJson.keySet().stream().
- filter(alias -> alias.startsWith("context-event")).
- collect(Collectors.toSet());
-
- for (String indexName : indices) {
- updateMapping(indexName, httpClient);
- }
- }
+ String indexPrefix = ConsoleUtils.askUserWithDefaultAnswer(session,
"SOURCE index name (default: context) : ", "context");
+ Set<String> indexes = getEventIndexes(indexPrefix);
+ for (String index : indexes) {
+ updateMapping(index);
}
+ createScopes(getSetOfScopes(indexes), indexPrefix);
}
- private void updateMapping(final String indexName, final
CloseableHttpClient httpClient) throws IOException {
+ private void updateMapping(final String indexName) throws IOException {
HttpPut httpPut = new HttpPut(esAddress + "/" + indexName +
"/_mapping");
httpPut.addHeader("Accept", "application/json");
httpPut.addHeader("Content-Type", "application/json");
- String request = "{\n" +
- "\"properties\": {\n" +
- " \"sourceId\": {\n" +
- " \"analyzer\": \"folding\",\n" +
- " \"type\": \"text\",\n" +
- " \"fields\": {\n" +
- " \"keyword\": {\n" +
- " \"type\": \"keyword\",\n" +
- " \"ignore_above\": 256\n" +
- " }\n" +
- " }\n" +
- " }\n" +
- " }\n" +
- "}";
+ String request = "{\n" + "\"properties\": {\n" + " \"sourceId\": {\n"
+ " \"analyzer\": \"folding\",\n" + " \"type\": \"text\",\n"
+ + " \"fields\": {\n" + " \"keyword\": {\n" + " \"type\":
\"keyword\",\n" + " \"ignore_above\": 256\n" + " }\n"
+ + " }\n" + " }\n" + " }\n" + "}";
Review Comment:
Difficult to maintain this kind of stuff, would be create to have them in
separate file as resources like in the itests.
I know that we dont update them after release, but in case of any issue or
troubleshoot this code will be painful
##########
tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/impl/MigrationTo200.java:
##########
@@ -126,25 +113,88 @@ private void copyValueScopeToSourceId(final String
indexName, final CloseableHtt
httpPost.addHeader("Accept", "application/json");
httpPost.addHeader("Content-Type", "application/json");
- String request = "{\n" +
- " \"script\": {\n" +
- " \"source\": \"ctx._source.sourceId =
ctx._source.scope\",\n" +
- " \"lang\": \"painless\"\n" +
- " }\n" +
- "}";
+ String request = "{\n" + " \"script\": {\n" + " \"source\":
\"ctx._source.sourceId = ctx._source.scope\",\n"
+ + " \"lang\": \"painless\"\n" + " }\n" + "}";
httpPost.setEntity(new StringEntity(request));
try (CloseableHttpResponse response = httpClient.execute(httpPost)) {
JSONObject responseAsJson = new
JSONObject(EntityUtils.toString(response.getEntity()));
if (response.getStatusLine().getStatusCode() == HttpStatus.SC_OK) {
- System.out.println("Copying the \"scope\" field to the
\"sourceId\" field for index = \"" + indexName + "\" successfully completed.
Total: " +
- responseAsJson.get("total") + ", updated: " +
responseAsJson.get("updated") + ".");
+ System.out.println("Copying the \"scope\" field to the
\"sourceId\" field for index = \"" + indexName
+ + "\" successfully completed. Total: " +
responseAsJson.get("total") + ", updated: " + responseAsJson.get("updated")
+ + ".");
} else {
System.out.println("Copying the \"scope\" field to the
\"sourceId\" field for index = \"" + indexName + "\" failed.");
}
}
}
+ private Set<String> getEventIndexes(String indexPrefix) throws IOException
{
+ try (CloseableHttpResponse response = httpClient.execute(new
HttpGet(esAddress + "/_aliases"))) {
+ if (response.getStatusLine().getStatusCode() == HttpStatus.SC_OK) {
+ JSONObject indexesAsJson = new
JSONObject(EntityUtils.toString(response.getEntity()));
+ return indexesAsJson.keySet().stream().
+ filter(alias -> alias.startsWith(indexPrefix +
"-event")).
+ collect(Collectors.toSet());
+ }
+ }
+ return Collections.emptySet();
+ }
+
+ private void createScopes(Set<String> scopes, String indexPrefix) throws
IOException {
+ final StringBuilder body = new StringBuilder();
+ scopes.forEach(scope -> {
+ body.append("{\"index\": {\"_id\": \"" + scope + "\"}}\n");
+ body.append("{\"itemId\": \"" + scope + "\", \"itemType\":
\"scope\", \"metadata\": { \"id\": \"" + scope + "\" }}\n");
+ });
+ final HttpPost httpPost = new HttpPost(esAddress + "/" + indexPrefix +
"-scope/_bulk");
+
+ httpPost.addHeader("Accept", "application/json");
+ httpPost.addHeader("Content-Type", "application/x-ndjson");
+
+ httpPost.setEntity(new StringEntity(body.toString()));
+
+ try (CloseableHttpResponse response = httpClient.execute(httpPost)) {
+ if (response.getStatusLine().getStatusCode() == HttpStatus.SC_OK) {
+ System.out.println("Creating the \"scopes\" into the index " +
indexPrefix + "-scope successfully finished");
+ } else {
+ System.out.println("Creating the \"scopes\" into the index " +
indexPrefix + "-scope has failed" + response.getStatusLine()
+ .getStatusCode());
+ }
+ }
+ }
+
+ private Set<String> getSetOfScopes(Set<String> indices) throws IOException
{
+ String joinedIndices = String.join(",", indices);
+ final HttpPost httpPost = new HttpPost(esAddress + "/" + joinedIndices
+ "/_search");
+
+ httpPost.addHeader("Accept", "application/json");
+ httpPost.addHeader("Content-Type", "application/json");
+
+ String request =
+ "{\n" + " \"_source\": false,\n" + " \"size\": 0,\n" + "
\"aggs\": {\n" + " \"scopes\": {\n" + " \"terms\": {\n"
+ + " \"field\": \"scope.keyword\"\n" + "
}\n" + " },\n" + " \"bucketInfos\": {\n"
+ + " \"stats_bucket\": {\n" + "
\"buckets_path\": \"scopes._count\"\n" + " }\n" + " }\n"
+ + " }\n" + "}";
+
+ httpPost.setEntity(new StringEntity(request));
+
+ Set<String> scopes = new HashSet<>();
+ try (CloseableHttpResponse response = httpClient.execute(httpPost)) {
+ JSONObject responseAsJson = new
JSONObject(EntityUtils.toString(response.getEntity()));
+ if (response.getStatusLine().getStatusCode() == HttpStatus.SC_OK) {
+ System.out.println("Getting the \"scope\" values from the
events successfully finished. " + "Number of scope to create: "
+ +
responseAsJson.getJSONObject("aggregations").getJSONObject("bucketInfos").get("count").toString());
+ scopes = StreamSupport
+
.stream(responseAsJson.getJSONObject("aggregations").getJSONObject("scopes").getJSONArray("buckets").spliterator(),
+ false).map(bucketElement -> ((JSONObject)
bucketElement).getString("key")).collect(Collectors.toSet());
Review Comment:
is it null safe ? if one of the element is null during the chain of
functions will it break the entire migration ?
Migration code should be as safe as possible.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
