This is an automated email from the ASF dual-hosted git repository.
dgriffon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/unomi.git
The following commit(s) were added to refs/heads/master by this push:
new 06048715f UNOMI-810 : add log on ES operation onto Metadata items.
(#651)
06048715f is described below
commit 06048715f5cb0b384f28072f46f3866f16627a24
Author: David Griffon <[email protected]>
AuthorDate: Mon Dec 11 16:50:25 2023 +0100
UNOMI-810 : add log on ES operation onto Metadata items. (#651)
* UNOMI-810 : add log on ES operation onto Metadata items.
* UNOMI-810 : add logs on metadata items ES operations - refresh index
after scope migration
---
.github/workflows/unomi-ci-build-tests.yml | 5 ++++-
.../ElasticSearchPersistenceServiceImpl.java | 19 ++++++++++++++++---
.../apache/unomi/shell/migration/utils/HttpUtils.java | 2 +-
.../cxs/migration/migrate-2.0.0-20-scopes.groovy | 2 ++
4 files changed, 23 insertions(+), 5 deletions(-)
diff --git a/.github/workflows/unomi-ci-build-tests.yml
b/.github/workflows/unomi-ci-build-tests.yml
index 4ee27ca0e..ce3cb60b2 100644
--- a/.github/workflows/unomi-ci-build-tests.yml
+++ b/.github/workflows/unomi-ci-build-tests.yml
@@ -54,7 +54,10 @@ jobs:
if: failure()
with:
name: unomi-log-jdk${{ matrix.java }}-${{ github.run_number }}
- path: itests/target/exam/**/data/log
+ path: |
+ itests/target/exam/**/data/log
+ itests/target/elasticsearch0/data
+ itests/target/elasticsearch0/logs
- name: Publish Test Report
uses: mikepenz/action-junit-report@v3
if: failure()
diff --git
a/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java
b/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java
index 8ca6526f1..0da7bc14b 100644
---
a/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java
+++
b/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java
@@ -890,7 +890,7 @@ public class ElasticSearchPersistenceServiceImpl implements
PersistenceService,
}.catchingExecuteInClassLoader(true);
}
-
+
private void setMetadata(Item item, String itemId, long version, long
seqNo, long primaryTerm, String index) {
if (!systemItems.contains(item.getItemType()) && item.getItemId() ==
null) {
item.setItemId(itemId);
@@ -970,6 +970,7 @@ public class ElasticSearchPersistenceServiceImpl implements
PersistenceService,
} else {
bulkProcessor.add(indexRequest);
}
+ logMetadataItemOperation("saved", item);
} catch (IndexNotFoundException e) {
logger.error("Could not find index {}, could not
register item type {} with id {} ", index, itemType, item.getItemId(), e);
return false;
@@ -1026,6 +1027,7 @@ public class ElasticSearchPersistenceServiceImpl
implements PersistenceService,
} else {
bulkProcessor.add(updateRequest);
}
+ logMetadataItemOperation("updated", item);
return true;
} catch (IndexNotFoundException e) {
throw new Exception("No index found for itemType=" +
clazz.getName() + "itemId=" + item.getItemId(), e);
@@ -1322,6 +1324,9 @@ public class ElasticSearchPersistenceServiceImpl
implements PersistenceService,
DeleteRequest deleteRequest = new DeleteRequest(index,
documentId);
client.delete(deleteRequest, RequestOptions.DEFAULT);
+ if (MetadataItem.class.isAssignableFrom(clazz)) {
+ logger.info("Item of type {} with ID {} has been
removed", customItemType != null ? customItemType : clazz.getSimpleName(),
itemId);
+ }
return true;
} catch (Exception e) {
throw new Exception("Cannot remove", e);
@@ -1352,6 +1357,7 @@ public class ElasticSearchPersistenceServiceImpl
implements PersistenceService,
public <T extends Item> boolean removeByQuery(QueryBuilder queryBuilder,
final Class<T> clazz) throws Exception {
try {
String itemType = Item.getItemType(clazz);
+ logger.debug("Remove item of type {} using a query", itemType);
final DeleteByQueryRequest deleteByQueryRequest = new
DeleteByQueryRequest(getIndexNameForQuery(itemType))
.setQuery(wrapWithItemTypeQuery(itemType, queryBuilder))
// Setting slices to auto will let Elasticsearch choose
the number of slices to use.
@@ -1442,7 +1448,7 @@ public class ElasticSearchPersistenceServiceImpl
implements PersistenceService,
}
public boolean createIndex(final String itemType) {
-
+ logger.debug("Create index {}", itemType);
Boolean result = new InClassLoaderExecute<Boolean>(metricsService,
this.getClass().getName() + ".createIndex", this.bundleContext,
this.fatalIllegalStateErrors, throwExceptions) {
protected Boolean execute(Object... args) throws IOException {
String index = getIndex(itemType);
@@ -2477,6 +2483,7 @@ public class ElasticSearchPersistenceServiceImpl
implements PersistenceService,
@Override
public void purge(final String scope) {
+ logger.debug("Purge scope {}", scope);
new InClassLoaderExecute<Void>(metricsService,
this.getClass().getName() + ".purgeWithScope", this.bundleContext,
this.fatalIllegalStateErrors, throwExceptions) {
@Override
protected Void execute(Object... args) throws IOException {
@@ -2672,7 +2679,7 @@ public class ElasticSearchPersistenceServiceImpl
implements PersistenceService,
private String getIndexNameForQuery(String itemType) {
return isItemTypeRollingOver(itemType) ?
getRolloverIndexForQuery(itemType) : getIndex(itemType);
}
-
+
private String getRolloverIndexForQuery(String itemType) {
return indexPrefix + "-" + itemType.toLowerCase() + "-*";
}
@@ -2737,4 +2744,10 @@ public class ElasticSearchPersistenceServiceImpl
implements PersistenceService,
}
return WriteRequest.RefreshPolicy.NONE;
}
+
+ private void logMetadataItemOperation (String operation, Item item) {
+ if (item instanceof MetadataItem) {
+ logger.info("Item of type {} with ID {} has been {}",
item.getItemType(), item.getItemId(), operation);
+ }
+ }
}
diff --git
a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/utils/HttpUtils.java
b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/utils/HttpUtils.java
index ec7a8a6dc..795d9849a 100644
---
a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/utils/HttpUtils.java
+++
b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/utils/HttpUtils.java
@@ -44,7 +44,6 @@ import java.security.KeyManagementException;
import java.security.NoSuchAlgorithmException;
import java.security.SecureRandom;
import java.security.cert.X509Certificate;
-import java.util.Arrays;
import java.util.Map;
/**
@@ -182,6 +181,7 @@ public class HttpUtils {
String stringResponse = EntityUtils.toString(entity);
EntityUtils.consumeQuietly(entity);
+
return stringResponse;
}
}
diff --git
a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-20-scopes.groovy
b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-20-scopes.groovy
index 7f2300e2d..7b291abcd 100644
---
a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-20-scopes.groovy
+++
b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-20-scopes.groovy
@@ -73,6 +73,8 @@
context.performMigrationStep("2.0.0-create-scopes-from-existing-events", () -> {
if (bulkSaveRequest.length() > 0) {
HttpUtils.executePostRequest(context.getHttpClient(), esAddress +
"/" + scopeIndex + "/_bulk", bulkSaveRequest.toString(), null)
+ // Refresh index
+ HttpUtils.executeGetRequest(context.getHttpClient(), esAddress +
"/" + scopeIndex + "/_refresh", null)
}
}
})
\ No newline at end of file
