This is an automated email from the ASF dual-hosted git repository.
wusheng pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/skywalking.git
The following commit(s) were added to refs/heads/master by this push:
new 2bf7f0f Support ElasticSearch template mappings `properties
parameters` and `_source` update (#8706)
2bf7f0f is described below
commit 2bf7f0f4e779e276d241cbf71773a3bba43735b8
Author: Kai <[email protected]>
AuthorDate: Sat Mar 19 00:55:47 2022 +0800
Support ElasticSearch template mappings `properties parameters` and
`_source` update (#8706)
Previous logic can't update the mappings `properties parameters` and
`_source` only support adding a new property.
### Now
1. As the OAP core merges the index of metrics, properties would not be
deleted still.
2. Would not update the current index to avoid conflict, a new index
follows the new template to create.
**Warning**, if there are properties conflicts, same key with different
values, they would override each other according to booting sequence. Unless we
change booting process as merging modules before merging with existing
templates and indices, we can't add check mechanism and warnings.
---
CHANGES.md | 2 +
.../library/elasticsearch/response/Mappings.java | 1 +
.../plugin/elasticsearch/base/IndexStructures.java | 37 ++-
.../elasticsearch/base/IndexStructuresTest.java | 14 +-
.../elasticsearch/base/MockEsInstallTest.java | 275 +++++++++++++++++++++
5 files changed, 301 insertions(+), 28 deletions(-)
diff --git a/CHANGES.md b/CHANGES.md
index f796f8b..29e605b 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -136,6 +136,8 @@ BanyanDB is the 1st storage implementation supporting this.
It would make contin
NOTICE, this sharding concept is NOT just for splitting data into different
database instances or physical files.
```
+* Support ElasticSearch template mappings `properties parameters` and
`_source` update.
+
#### UI
* Remove unused jars (log4j-api.jar) in classpath.
diff --git
a/oap-server/server-library/library-elasticsearch-client/src/main/java/org/apache/skywalking/library/elasticsearch/response/Mappings.java
b/oap-server/server-library/library-elasticsearch-client/src/main/java/org/apache/skywalking/library/elasticsearch/response/Mappings.java
index 03eb5a9..b0ab993 100644
---
a/oap-server/server-library/library-elasticsearch-client/src/main/java/org/apache/skywalking/library/elasticsearch/response/Mappings.java
+++
b/oap-server/server-library/library-elasticsearch-client/src/main/java/org/apache/skywalking/library/elasticsearch/response/Mappings.java
@@ -51,6 +51,7 @@ public final class Mappings {
@Setter
private Source source = new Source();
+ @EqualsAndHashCode
public static class Source {
@JsonProperty("excludes")
@Getter
diff --git
a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/IndexStructures.java
b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/IndexStructures.java
index 74423c8..97d9719 100644
---
a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/IndexStructures.java
+++
b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/IndexStructures.java
@@ -21,7 +21,6 @@ package
org.apache.skywalking.oap.server.storage.plugin.elasticsearch.base;
import java.util.HashMap;
import java.util.Map;
import java.util.Objects;
-import java.util.Set;
import java.util.stream.Collectors;
import org.apache.skywalking.library.elasticsearch.response.Mappings;
import
org.apache.skywalking.oap.server.library.client.elasticsearch.ElasticSearchClient;
@@ -67,7 +66,8 @@ public class IndexStructures {
/**
* Returns mappings with fields that not exist in the input mappings.
- * do not return _source config to avoid index update conflict.
+ * The input mappings should be history mapping from current index.
+ * Do not return _source config to avoid current index update conflict.
*/
public Mappings diffStructure(String tableName, Mappings mappings) {
if (!structures.containsKey(tableName)) {
@@ -102,7 +102,7 @@ public class IndexStructures {
*/
public static class Fields {
private final Map<String, Object> properties;
- Mappings.Source source;
+ private Mappings.Source source;
private Fields(Mappings mapping) {
this.properties = mapping.getProperties();
@@ -113,28 +113,25 @@ public class IndexStructures {
* Returns ture when the input fields have already been stored in the
properties.
*/
private boolean containsAllFields(Fields fields) {
- return fields.properties.entrySet().stream()
- .allMatch(item ->
this.properties.containsKey(item.getKey()));
+ if (this.properties.size() < fields.properties.size()) {
+ return false;
+ }
+ boolean isContains = fields.properties.entrySet().stream()
+ .allMatch(item ->
Objects.equals(properties.get(item.getKey()), item.getValue()));
+ if (!isContains) {
+ return false;
+ }
+ return Objects.equals(this.source, fields.source);
}
/**
- * Append new fields to the properties when have new fields.
+ * Append new fields and update.
+ * Properties combine input and exist, update property's attribute,
won't remove old one.
+ * Source will be updated to the input.
*/
private void appendNewFields(Fields fields) {
- Map<String, Object> newFields =
- fields.properties.entrySet()
- .stream()
- .filter(e ->
!this.properties.containsKey(e.getKey()))
- .collect(Collectors.toMap(
- Map.Entry::getKey,
- Map.Entry::getValue
- ));
- properties.putAll(newFields);
- if (source != null) {
- Set<String> exclude = source.getExcludes();
- Set<String> newExclude = fields.source.getExcludes();
- exclude.addAll(newExclude);
- }
+ properties.putAll(fields.properties);
+ source = fields.source;
}
/**
diff --git
a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/IndexStructuresTest.java
b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/IndexStructuresTest.java
index fe5d7eb..4c06148 100644
---
a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/IndexStructuresTest.java
+++
b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/IndexStructuresTest.java
@@ -21,7 +21,6 @@ package
org.apache.skywalking.oap.server.storage.plugin.elasticsearch.base;
import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
-import java.util.Set;
import org.apache.skywalking.library.elasticsearch.response.Mappings;
import
org.apache.skywalking.oap.server.library.client.elasticsearch.ElasticSearchClient;
import org.junit.Assert;
@@ -61,8 +60,8 @@ public class IndexStructuresTest {
.properties(properties)
.source(source)
.build());
-
Assert.assertEquals(structuresSource.getMapping("test").getProperties(),
properties);
-
Assert.assertEquals(structuresSource.getMapping("test").getSource().getExcludes(),
source.getExcludes());
+ Assert.assertEquals(properties,
structuresSource.getMapping("test").getProperties());
+ Assert.assertEquals(source.getExcludes(),
structuresSource.getMapping("test").getSource().getExcludes());
}
@Test
@@ -103,8 +102,8 @@ public class IndexStructuresTest {
.properties(properties)
.source(source)
.build());
-
Assert.assertEquals(structuresSource.getMapping("test").getProperties(),
properties);
-
Assert.assertEquals(structuresSource.getMapping("test").getSource().getExcludes(),
source.getExcludes());
+ Assert.assertEquals(properties,
structuresSource.getMapping("test").getProperties());
+ Assert.assertEquals(source.getExcludes(),
structuresSource.getMapping("test").getSource().getExcludes());
Mappings.Source source2 = new Mappings.Source();
source.getExcludes().addAll(Arrays.asList("b", "c", "d", "e"));
@@ -114,9 +113,8 @@ public class IndexStructuresTest {
.properties(properties2)
.source(source2)
.build());
- Set<String> excludes = new HashSet<>(Arrays.asList("a", "b", "c", "d",
"e"));
-
Assert.assertEquals(structuresSource.getMapping("test").getProperties(), res);
-
Assert.assertEquals(structuresSource.getMapping("test").getSource().getExcludes(),
excludes);
+ Assert.assertEquals(res,
structuresSource.getMapping("test").getProperties());
+ Assert.assertEquals(new HashSet<>(),
structuresSource.getMapping("test").getSource().getExcludes());
}
@Test
diff --git
a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/MockEsInstallTest.java
b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/MockEsInstallTest.java
new file mode 100644
index 0000000..61b461f
--- /dev/null
+++
b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/MockEsInstallTest.java
@@ -0,0 +1,275 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.skywalking.oap.server.storage.plugin.elasticsearch.base;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Set;
+import org.apache.skywalking.library.elasticsearch.response.Mappings;
+import
org.apache.skywalking.oap.server.library.client.elasticsearch.ElasticSearchClient;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.testcontainers.shaded.com.google.common.collect.ImmutableMap;
+import org.testcontainers.shaded.com.google.common.collect.ImmutableSet;
+
+@RunWith(Parameterized.class)
+public class MockEsInstallTest {
+ private final ObjectMapper mapper = new ObjectMapper();
+ @Parameterized.Parameter
+ public String name;
+
+ @Parameterized.Parameter(1)
+ public Mappings hisMappings;
+
+ @Parameterized.Parameter(2)
+ public Mappings newMappings;
+
+ @Parameterized.Parameter(3)
+ public Set<String> excludes;
+
+ @Parameterized.Parameter(4)
+ public Set<String> newExcludes;
+
+ @Parameterized.Parameter(5)
+ public String combineResult;
+
+ @Parameterized.Parameter(6)
+ public String diffResult;
+
+ @Parameterized.Parameter(7)
+ public boolean contains;
+
+ @Parameterized.Parameters(name = "{index}: {0}")
+
+ public static Collection<Object[]> data() {
+ return Arrays.asList(new Object[][] {
+ {
+ "contains_properties_true",
+ Mappings.builder()
+ .type(ElasticSearchClient.TYPE)
+ .properties(new HashMap<>(
+ ImmutableMap.of("a", ImmutableMap.of("type",
"keyword"),
+ "b", ImmutableMap.of("type",
"keyword")
+ )))
+ .source(new Mappings.Source())
+ .build(),
+ Mappings.builder()
+ .type(ElasticSearchClient.TYPE)
+ .properties(new HashMap<>(
+ ImmutableMap.of("b", ImmutableMap.of("type",
"keyword"))))
+ .source(new Mappings.Source())
+ .build(),
+ null,
+ null,
+
"{\"properties\":{\"a\":{\"type\":\"keyword\"},\"b\":{\"type\":\"keyword\"}},\"_source\":{\"excludes\":[]}}",
+ "{\"properties\":{},\"_source\":null}",
+ true
+ },
+ {
+ "contains_source_true",
+ Mappings.builder()
+ .type(ElasticSearchClient.TYPE)
+ .properties(new HashMap<>(
+ ImmutableMap.of("a", ImmutableMap.of("type",
"keyword"),
+ "b", ImmutableMap.of("type",
"keyword")
+ )))
+ .source(new Mappings.Source())
+ .build(),
+ Mappings.builder()
+ .type(ElasticSearchClient.TYPE)
+ .properties(new HashMap<>(
+ ImmutableMap.of("b", ImmutableMap.of("type",
"keyword"))))
+ .source(new Mappings.Source())
+ .build(),
+ new HashSet<>(ImmutableSet.of("b")),
+ new HashSet<>(ImmutableSet.of("b")),
+
"{\"properties\":{\"a\":{\"type\":\"keyword\"},\"b\":{\"type\":\"keyword\"}},\"_source\":{\"excludes\":[\"b\"]}}",
+ "{\"properties\":{},\"_source\":null}",
+ true
+ },
+ {
+ "contains_source_false",
+ Mappings.builder()
+ .type(ElasticSearchClient.TYPE)
+ .properties(new HashMap<>(
+ ImmutableMap.of("a", ImmutableMap.of("type",
"keyword"),
+ "b", ImmutableMap.of("type",
"keyword")
+ )))
+ .source(new Mappings.Source())
+ .build(),
+ Mappings.builder()
+ .type(ElasticSearchClient.TYPE)
+ .properties(new HashMap<>(
+ ImmutableMap.of("b", ImmutableMap.of("type",
"keyword"))))
+ .source(new Mappings.Source())
+ .build(),
+ new HashSet<>(ImmutableSet.of("a", "b")),
+ new HashSet<>(ImmutableSet.of("b")),
+
"{\"properties\":{\"a\":{\"type\":\"keyword\"},\"b\":{\"type\":\"keyword\"}},\"_source\":{\"excludes\":[\"b\"]}}",
+ "{\"properties\":{},\"_source\":null}",
+ false
+ },
+ {
+ "contains_source_false",
+ Mappings.builder()
+ .type(ElasticSearchClient.TYPE)
+ .properties(new HashMap<>(
+ ImmutableMap.of("a", ImmutableMap.of("type",
"keyword"),
+ "b", ImmutableMap.of("type",
"keyword")
+ )))
+ .source(new Mappings.Source())
+ .build(),
+ Mappings.builder()
+ .type(ElasticSearchClient.TYPE)
+ .properties(new HashMap<>(
+ ImmutableMap.of("b", ImmutableMap.of("type",
"keyword"))))
+ .source(new Mappings.Source())
+ .build(),
+ new HashSet<>(ImmutableSet.of("a", "b")),
+ null,
+
"{\"properties\":{\"a\":{\"type\":\"keyword\"},\"b\":{\"type\":\"keyword\"}},\"_source\":{\"excludes\":[]}}",
+ "{\"properties\":{},\"_source\":null}",
+ false
+ },
+ {
+ "combineAndUpdate_properties",
+ Mappings.builder()
+ .type(ElasticSearchClient.TYPE)
+ .properties(new HashMap<>(
+ ImmutableMap.of("a", ImmutableMap.of("type",
"keyword"),
+ "b", ImmutableMap.of("type",
"keyword")
+ )))
+ .source(new Mappings.Source())
+ .build(),
+ Mappings.builder()
+ .type(ElasticSearchClient.TYPE)
+ .properties(new HashMap<>(
+ ImmutableMap.of("b", ImmutableMap.of("type",
"keyword", "index", false),
+ "c", ImmutableMap.of("type",
"keyword")
+ )))
+ .source(new Mappings.Source())
+ .build(),
+ null,
+ null,
+
"{\"properties\":{\"a\":{\"type\":\"keyword\"},\"b\":{\"type\":\"keyword\",\"index\":false},\"c\":{\"type\":\"keyword\"}},\""
+
+ "_source\":{\"excludes\":[]}}",
+
"{\"properties\":{\"c\":{\"type\":\"keyword\"}},\"_source\":null}",
+ false
+ },
+ {
+ "combineAndUpdate_properties",
+ Mappings.builder()
+ .type(ElasticSearchClient.TYPE)
+ .properties(new HashMap<>(
+ ImmutableMap.of("a", ImmutableMap.of("type",
"keyword"),
+ "b", ImmutableMap.of("type",
"keyword", "index", false)
+ )))
+ .source(new Mappings.Source())
+ .build(),
+ Mappings.builder()
+ .type(ElasticSearchClient.TYPE)
+ .properties(new HashMap<>(
+ ImmutableMap.of("b", ImmutableMap.of("type",
"keyword"),
+ "c", ImmutableMap.of("type",
"keyword", "index", false)
+ )))
+ .source(new Mappings.Source())
+ .build(),
+ null,
+ null,
+
"{\"properties\":{\"a\":{\"type\":\"keyword\"},\"b\":{\"type\":\"keyword\"},\"c\":{\"type\":\"keyword\",\"index\":false}},\""
+
+ "_source\":{\"excludes\":[]}}",
+
"{\"properties\":{\"c\":{\"type\":\"keyword\",\"index\":false}},\"_source\":null}",
+ false
+ },
+ {
+ "update_source",
+ Mappings.builder()
+ .type(ElasticSearchClient.TYPE)
+ .properties(new HashMap<>(
+ ImmutableMap.of("a", ImmutableMap.of("type",
"keyword"),
+ "b", ImmutableMap.of("type",
"keyword", "index", false)
+ )))
+ .source(new Mappings.Source())
+ .build(),
+ Mappings.builder()
+ .type(ElasticSearchClient.TYPE)
+ .properties(new HashMap<>(
+ ImmutableMap.of("b", ImmutableMap.of("type",
"keyword"),
+ "c", ImmutableMap.of("type",
"keyword", "index", false)
+ )))
+ .source(new Mappings.Source())
+ .build(),
+ new HashSet<>(ImmutableSet.of("a")),
+ new HashSet<>(ImmutableSet.of("b")),
+
"{\"properties\":{\"a\":{\"type\":\"keyword\"},\"b\":{\"type\":\"keyword\"},\"c\":{\"type\":\"keyword\",\"index\":false}},\""
+
+ "_source\":{\"excludes\":[\"b\"]}}",
+
"{\"properties\":{\"c\":{\"type\":\"keyword\",\"index\":false}},\"_source\":null}",
+ false
+ }
+ });
+ }
+
+ @Before
+ public void init() {
+ if (this.excludes != null) {
+ this.hisMappings.getSource().setExcludes(this.excludes);
+ }
+ if (this.newExcludes != null) {
+ this.newMappings.getSource().setExcludes(this.newExcludes);
+ }
+ }
+
+ @Test
+ public void mockEsInstallTest() throws JsonProcessingException {
+ IndexStructures structures = new IndexStructures();
+ //clone it since the items will be changed after combine
+ Mappings hisMappingsClone = cloneMappings(this.hisMappings);
+ //put the current mappings
+ structures.putStructure(this.name, this.hisMappings);
+ //if current mappings already contains new mappings items
+ Assert.assertEquals(this.contains,
structures.containsStructure(this.name, this.newMappings));
+
+ //put the new mappings and combine
+ structures.putStructure(this.name, this.newMappings);
+ Mappings mappings = structures.getMapping(this.name);
+ Assert.assertEquals(this.combineResult,
mapper.writeValueAsString(mappings));
+
+ //diff the hisMapping and new, if has new item will update current
index
+ structures.putStructure(this.name, this.newMappings);
+ Mappings diff = structures.diffStructure(this.name, hisMappingsClone);
+ Assert.assertEquals(this.diffResult, mapper.writeValueAsString(diff));
+ }
+
+ private Mappings cloneMappings(Mappings mappings) {
+ Mappings.Source source = new Mappings.Source();
+ source.setExcludes(new HashSet<>(mappings.getSource().getExcludes()));
+ return Mappings.builder()
+ .type(mappings.getType())
+ .properties(new HashMap<>(mappings.getProperties()))
+ .source(source)
+ .build();
+ }
+}
