jsinovassin commented on code in PR #427:
URL: https://github.com/apache/unomi/pull/427#discussion_r879382754
##########
tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/impl/MigrationTo200.java:
##########
@@ -126,25 +113,88 @@ private void copyValueScopeToSourceId(final String
indexName, final CloseableHtt
httpPost.addHeader("Accept", "application/json");
httpPost.addHeader("Content-Type", "application/json");
- String request = "{\n" +
- " \"script\": {\n" +
- " \"source\": \"ctx._source.sourceId =
ctx._source.scope\",\n" +
- " \"lang\": \"painless\"\n" +
- " }\n" +
- "}";
+ String request = "{\n" + " \"script\": {\n" + " \"source\":
\"ctx._source.sourceId = ctx._source.scope\",\n"
+ + " \"lang\": \"painless\"\n" + " }\n" + "}";
httpPost.setEntity(new StringEntity(request));
try (CloseableHttpResponse response = httpClient.execute(httpPost)) {
JSONObject responseAsJson = new
JSONObject(EntityUtils.toString(response.getEntity()));
if (response.getStatusLine().getStatusCode() == HttpStatus.SC_OK) {
- System.out.println("Copying the \"scope\" field to the
\"sourceId\" field for index = \"" + indexName + "\" successfully completed.
Total: " +
- responseAsJson.get("total") + ", updated: " +
responseAsJson.get("updated") + ".");
+ System.out.println("Copying the \"scope\" field to the
\"sourceId\" field for index = \"" + indexName
+ + "\" successfully completed. Total: " +
responseAsJson.get("total") + ", updated: " + responseAsJson.get("updated")
+ + ".");
} else {
System.out.println("Copying the \"scope\" field to the
\"sourceId\" field for index = \"" + indexName + "\" failed.");
}
}
}
+ private Set<String> getEventIndexes(String indexPrefix) throws IOException
{
+ try (CloseableHttpResponse response = httpClient.execute(new
HttpGet(esAddress + "/_aliases"))) {
+ if (response.getStatusLine().getStatusCode() == HttpStatus.SC_OK) {
+ JSONObject indexesAsJson = new
JSONObject(EntityUtils.toString(response.getEntity()));
+ return indexesAsJson.keySet().stream().
+ filter(alias -> alias.startsWith(indexPrefix +
"-event")).
+ collect(Collectors.toSet());
+ }
+ }
+ return Collections.emptySet();
+ }
+
+ private void createScopes(Set<String> scopes, String indexPrefix) throws
IOException {
+ final StringBuilder body = new StringBuilder();
+ scopes.forEach(scope -> {
+ body.append("{\"index\": {\"_id\": \"" + scope + "\"}}\n");
+ body.append("{\"itemId\": \"" + scope + "\", \"itemType\":
\"scope\", \"metadata\": { \"id\": \"" + scope + "\" }}\n");
+ });
+ final HttpPost httpPost = new HttpPost(esAddress + "/" + indexPrefix +
"-scope/_bulk");
+
+ httpPost.addHeader("Accept", "application/json");
+ httpPost.addHeader("Content-Type", "application/x-ndjson");
+
+ httpPost.setEntity(new StringEntity(body.toString()));
+
+ try (CloseableHttpResponse response = httpClient.execute(httpPost)) {
+ if (response.getStatusLine().getStatusCode() == HttpStatus.SC_OK) {
+ System.out.println("Creating the \"scopes\" into the index " +
indexPrefix + "-scope successfully finished");
+ } else {
+ System.out.println("Creating the \"scopes\" into the index " +
indexPrefix + "-scope has failed" + response.getStatusLine()
+ .getStatusCode());
+ }
+ }
+ }
+
+ private Set<String> getSetOfScopes(Set<String> indices) throws IOException
{
+ String joinedIndices = String.join(",", indices);
+ final HttpPost httpPost = new HttpPost(esAddress + "/" + joinedIndices
+ "/_search");
+
+ httpPost.addHeader("Accept", "application/json");
+ httpPost.addHeader("Content-Type", "application/json");
+
+ String request =
+ "{\n" + " \"_source\": false,\n" + " \"size\": 0,\n" + "
\"aggs\": {\n" + " \"scopes\": {\n" + " \"terms\": {\n"
+ + " \"field\": \"scope.keyword\"\n" + "
}\n" + " },\n" + " \"bucketInfos\": {\n"
+ + " \"stats_bucket\": {\n" + "
\"buckets_path\": \"scopes._count\"\n" + " }\n" + " }\n"
+ + " }\n" + "}";
+
+ httpPost.setEntity(new StringEntity(request));
+
+ Set<String> scopes = new HashSet<>();
+ try (CloseableHttpResponse response = httpClient.execute(httpPost)) {
+ JSONObject responseAsJson = new
JSONObject(EntityUtils.toString(response.getEntity()));
+ if (response.getStatusLine().getStatusCode() == HttpStatus.SC_OK) {
+ System.out.println("Getting the \"scope\" values from the
events successfully finished. " + "Number of scope to create: "
+ +
responseAsJson.getJSONObject("aggregations").getJSONObject("bucketInfos").get("count").toString());
+ scopes = StreamSupport
+
.stream(responseAsJson.getJSONObject("aggregations").getJSONObject("scopes").getJSONArray("buckets").spliterator(),
+ false).map(bucketElement -> ((JSONObject)
bucketElement).getString("key")).collect(Collectors.toSet());
Review Comment:
Yes, it's null safe an exception would be thrown if the attribute is not
found and the migration will not crash
##########
tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/impl/MigrationTo200.java:
##########
@@ -126,25 +113,88 @@ private void copyValueScopeToSourceId(final String
indexName, final CloseableHtt
httpPost.addHeader("Accept", "application/json");
httpPost.addHeader("Content-Type", "application/json");
- String request = "{\n" +
- " \"script\": {\n" +
- " \"source\": \"ctx._source.sourceId =
ctx._source.scope\",\n" +
- " \"lang\": \"painless\"\n" +
- " }\n" +
- "}";
+ String request = "{\n" + " \"script\": {\n" + " \"source\":
\"ctx._source.sourceId = ctx._source.scope\",\n"
+ + " \"lang\": \"painless\"\n" + " }\n" + "}";
httpPost.setEntity(new StringEntity(request));
try (CloseableHttpResponse response = httpClient.execute(httpPost)) {
JSONObject responseAsJson = new
JSONObject(EntityUtils.toString(response.getEntity()));
if (response.getStatusLine().getStatusCode() == HttpStatus.SC_OK) {
- System.out.println("Copying the \"scope\" field to the
\"sourceId\" field for index = \"" + indexName + "\" successfully completed.
Total: " +
- responseAsJson.get("total") + ", updated: " +
responseAsJson.get("updated") + ".");
+ System.out.println("Copying the \"scope\" field to the
\"sourceId\" field for index = \"" + indexName
+ + "\" successfully completed. Total: " +
responseAsJson.get("total") + ", updated: " + responseAsJson.get("updated")
+ + ".");
} else {
System.out.println("Copying the \"scope\" field to the
\"sourceId\" field for index = \"" + indexName + "\" failed.");
}
}
}
+ private Set<String> getEventIndexes(String indexPrefix) throws IOException
{
+ try (CloseableHttpResponse response = httpClient.execute(new
HttpGet(esAddress + "/_aliases"))) {
+ if (response.getStatusLine().getStatusCode() == HttpStatus.SC_OK) {
+ JSONObject indexesAsJson = new
JSONObject(EntityUtils.toString(response.getEntity()));
+ return indexesAsJson.keySet().stream().
+ filter(alias -> alias.startsWith(indexPrefix +
"-event")).
+ collect(Collectors.toSet());
+ }
+ }
+ return Collections.emptySet();
+ }
+
+ private void createScopes(Set<String> scopes, String indexPrefix) throws
IOException {
+ final StringBuilder body = new StringBuilder();
+ scopes.forEach(scope -> {
+ body.append("{\"index\": {\"_id\": \"" + scope + "\"}}\n");
+ body.append("{\"itemId\": \"" + scope + "\", \"itemType\":
\"scope\", \"metadata\": { \"id\": \"" + scope + "\" }}\n");
+ });
+ final HttpPost httpPost = new HttpPost(esAddress + "/" + indexPrefix +
"-scope/_bulk");
+
+ httpPost.addHeader("Accept", "application/json");
+ httpPost.addHeader("Content-Type", "application/x-ndjson");
+
+ httpPost.setEntity(new StringEntity(body.toString()));
+
+ try (CloseableHttpResponse response = httpClient.execute(httpPost)) {
+ if (response.getStatusLine().getStatusCode() == HttpStatus.SC_OK) {
+ System.out.println("Creating the \"scopes\" into the index " +
indexPrefix + "-scope successfully finished");
+ } else {
+ System.out.println("Creating the \"scopes\" into the index " +
indexPrefix + "-scope has failed" + response.getStatusLine()
+ .getStatusCode());
+ }
+ }
+ }
+
+ private Set<String> getSetOfScopes(Set<String> indices) throws IOException
{
+ String joinedIndices = String.join(",", indices);
+ final HttpPost httpPost = new HttpPost(esAddress + "/" + joinedIndices
+ "/_search");
+
+ httpPost.addHeader("Accept", "application/json");
+ httpPost.addHeader("Content-Type", "application/json");
+
+ String request =
+ "{\n" + " \"_source\": false,\n" + " \"size\": 0,\n" + "
\"aggs\": {\n" + " \"scopes\": {\n" + " \"terms\": {\n"
+ + " \"field\": \"scope.keyword\"\n" + "
}\n" + " },\n" + " \"bucketInfos\": {\n"
+ + " \"stats_bucket\": {\n" + "
\"buckets_path\": \"scopes._count\"\n" + " }\n" + " }\n"
+ + " }\n" + "}";
+
+ httpPost.setEntity(new StringEntity(request));
+
+ Set<String> scopes = new HashSet<>();
+ try (CloseableHttpResponse response = httpClient.execute(httpPost)) {
+ JSONObject responseAsJson = new
JSONObject(EntityUtils.toString(response.getEntity()));
+ if (response.getStatusLine().getStatusCode() == HttpStatus.SC_OK) {
+ System.out.println("Getting the \"scope\" values from the
events successfully finished. " + "Number of scope to create: "
+ +
responseAsJson.getJSONObject("aggregations").getJSONObject("bucketInfos").get("count").toString());
+ scopes = StreamSupport
+
.stream(responseAsJson.getJSONObject("aggregations").getJSONObject("scopes").getJSONArray("buckets").spliterator(),
+ false).map(bucketElement -> ((JSONObject)
bucketElement).getString("key")).collect(Collectors.toSet());
Review Comment:
Yes, it's null safe, an exception would be thrown if the attribute is not
found and the migration will not crash
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
