This is an automated email from the ASF dual-hosted git repository.
wusheng pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/skywalking.git
The following commit(s) were added to refs/heads/master by this push:
new c91f7e3952 Fix Elasticsearch storage: In the merged index, column's
property `indexOnly` not applied and cannot be updated. (#10060)
c91f7e3952 is described below
commit c91f7e3952954a299227dbab77662ec6de517803
Author: Wan Kai <[email protected]>
AuthorDate: Wed Nov 30 17:24:35 2022 +0800
Fix Elasticsearch storage: In the merged index, column's property
`indexOnly` not applied and cannot be updated. (#10060)
---
docs/en/changes/changes.md | 1 +
.../plugin/elasticsearch/base/IndexStructures.java | 26 +++++++++++++++++++---
.../elasticsearch/base/IndexStructuresTest.java | 2 ++
.../elasticsearch/base/MockEsInstallTest.java | 19 ++++++++--------
4 files changed, 36 insertions(+), 12 deletions(-)
diff --git a/docs/en/changes/changes.md b/docs/en/changes/changes.md
index 3e54e32e7a..fefcb17362 100644
--- a/docs/en/changes/changes.md
+++ b/docs/en/changes/changes.md
@@ -137,6 +137,7 @@
* Rename `BanyanDB.ShardingKey` to `BanyanDB.SeriesID`.
* Self-Observability: Add counters for metrics reading from DB or cached.
Dashboard:`Metrics Persistent Cache Count`.
* Self-Observability: Fix `GC Time` calculation.
+* Fix Elasticsearch storage: In `No-Sharding Mode`, column's property
`indexOnly` not applied and cannot be updated.
* Update the `trace_id` field as storage only(cannot be queried) in
`top_n_database_statement`, `top_n_cache_read_command`,
`top_n_cache_read_command` index.
#### UI
diff --git
a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/IndexStructures.java
b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/IndexStructures.java
index 63edd50ad6..11c27f47d4 100644
---
a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/IndexStructures.java
+++
b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/IndexStructures.java
@@ -21,6 +21,7 @@ package
org.apache.skywalking.oap.server.storage.plugin.elasticsearch.base;
import java.util.HashMap;
import java.util.Map;
import java.util.Objects;
+import java.util.Set;
import java.util.stream.Collectors;
import lombok.EqualsAndHashCode;
import org.apache.skywalking.library.elasticsearch.response.Mappings;
@@ -143,17 +144,36 @@ public class IndexStructures {
if (!isContains) {
return false;
}
- return
fields.source.getExcludes().containsAll(this.source.getExcludes());
+ Set<String> inputExcludes = fields.source.getExcludes();
+ Set<String> excludes = source.getExcludes();
+ //need to add new excludes
+ if (!excludes.containsAll(inputExcludes)) {
+ return false;
+ }
+ //need to delete existing excludes
+ for (String p : fields.properties.keySet()) {
+ if (!inputExcludes.contains(p) && excludes.contains(p)) {
+ return false;
+ }
+ }
+ return true;
}
/**
* Append new fields and update.
* Properties combine input and exist, update property's attribute,
won't remove old one.
- * Source will be updated to the input.
+ * If the existed `excludes` contains a not existing property, the
excluded field would be removed.
*/
private void appendNewFields(Fields fields) {
properties.putAll(fields.properties);
- source = fields.source;
+ Set<String> inputExcludes = fields.source.getExcludes();
+ Set<String> excludes = source.getExcludes();
+ excludes.addAll(inputExcludes);
+ fields.properties.keySet().forEach(p -> {
+ if (!inputExcludes.contains(p) && excludes.contains(p)) {
+ excludes.remove(p);
+ }
+ });
}
/**
diff --git
a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/IndexStructuresTest.java
b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/IndexStructuresTest.java
index 059af8abb9..13e08aafb7 100644
---
a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/IndexStructuresTest.java
+++
b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/IndexStructuresTest.java
@@ -74,6 +74,7 @@ public class IndexStructuresTest {
"test", Mappings.builder()
.type(ElasticSearchClient.TYPE)
.properties(properties)
+ .source(new Mappings.Source())
.build(), new HashMap<>());
Mappings mapping = structures.getMapping("test");
Assert.assertEquals(properties, mapping.getProperties());
@@ -84,6 +85,7 @@ public class IndexStructuresTest {
"test", Mappings.builder()
.type(ElasticSearchClient.TYPE)
.properties(properties2)
+ .source(new Mappings.Source())
.build(), new HashMap<>());
mapping = structures.getMapping("test");
HashMap<String, Object> res = new HashMap<>();
diff --git
a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/MockEsInstallTest.java
b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/MockEsInstallTest.java
index e08e6a10de..66ea0f0d37 100644
---
a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/MockEsInstallTest.java
+++
b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/MockEsInstallTest.java
@@ -127,8 +127,8 @@ public class MockEsInstallTest {
.source(new Mappings.Source())
.build(),
new HashSet<>(ImmutableSet.of("a", "b")),
- new HashSet<>(ImmutableSet.of("b")),
-
"{\"properties\":{\"a\":{\"type\":\"keyword\"},\"b\":{\"type\":\"keyword\"}},\"_source\":{\"excludes\":[\"b\"]}}",
+ new HashSet<>(),
+
"{\"properties\":{\"a\":{\"type\":\"keyword\"},\"b\":{\"type\":\"keyword\"}},\"_source\":{\"excludes\":[\"a\"]}}",
"{\"properties\":{},\"_source\":null}",
false
},
@@ -145,13 +145,14 @@ public class MockEsInstallTest {
Mappings.builder()
.type(ElasticSearchClient.TYPE)
.properties(new HashMap<>(
- ImmutableMap.of("b", ImmutableMap.of("type",
"keyword"))))
+ ImmutableMap.of("c", ImmutableMap.of("type",
"keyword"))))
.source(new Mappings.Source())
.build(),
new HashSet<>(ImmutableSet.of("a", "b")),
- null,
-
"{\"properties\":{\"a\":{\"type\":\"keyword\"},\"b\":{\"type\":\"keyword\"}},\"_source\":{\"excludes\":[]}}",
- "{\"properties\":{},\"_source\":null}",
+ new HashSet<>(ImmutableSet.of("c")),
+
"{\"properties\":{\"a\":{\"type\":\"keyword\"},\"b\":{\"type\":\"keyword\"},\"c\":{\"type\":\"keyword\"}},"
+
+ "\"_source\":{\"excludes\":[\"a\",\"b\",\"c\"]}}",
+
"{\"properties\":{\"c\":{\"type\":\"keyword\"}},\"_source\":null}",
false
},
{
@@ -224,8 +225,8 @@ public class MockEsInstallTest {
.build(),
new HashSet<>(ImmutableSet.of("a")),
new HashSet<>(ImmutableSet.of("b")),
-
"{\"properties\":{\"a\":{\"type\":\"keyword\"},\"b\":{\"type\":\"keyword\"},\"c\":{\"type\":\"keyword\",\"index\":false}},\""
+
- "_source\":{\"excludes\":[\"b\"]}}",
+
"{\"properties\":{\"a\":{\"type\":\"keyword\"},\"b\":{\"type\":\"keyword\"},\"c\":{\"type\":\"keyword\",\"index\":false}},"
+
+ "\"_source\":{\"excludes\":[\"a\",\"b\"]}}",
"{\"properties\":{\"c\":{\"type\":\"keyword\",\"index\":false}},\"_source\":null}",
false
}
@@ -257,7 +258,7 @@ public class MockEsInstallTest {
Mappings mappings = structures.getMapping(this.name);
Assert.assertEquals(this.combineResult,
mapper.writeValueAsString(mappings));
- //diff the hisMapping and new, if has new item will update current
index
+ //diff the hisMapping and new, if it has new item will update current
index
structures.putStructure(this.name, this.newMappings, new HashMap<>());
Mappings diff = structures.diffMappings(this.name, hisMappingsClone);
Assert.assertEquals(this.diffResult, mapper.writeValueAsString(diff));
