This is an automated email from the ASF dual-hosted git repository. kezhenxu94 pushed a commit to branch es7.14 in repository https://gitbox.apache.org/repos/asf/skywalking.git
commit 15928ac1ef8b27ac1fdb5e17f64cd99942f2bc12 Author: kezhenxu94 <[email protected]> AuthorDate: Tue Aug 10 12:35:45 2021 +0800 Fix issues in ES7.14 and add log cases to ES7.14 --- .github/workflows/e2e.log.yaml | 2 +- CHANGES.md | 1 + .../oap/server/core/alarm/AlarmRecord.java | 5 +- .../analysis/manual/endpoint/EndpointTraffic.java | 6 ++- .../analysis/manual/log/AbstractLogRecord.java | 7 ++- .../analysis/manual/segment/SegmentRecord.java | 6 ++- .../analysis/manual/service/ServiceTraffic.java | 5 +- .../manual/errorlog/BrowserErrorLogRecord.java | 6 ++- .../storage/plugin/zipkin/ZipkinSpanRecord.java | 6 ++- .../e2e-test/docker/log/docker-compose.es7.14.yml | 56 ++++++++++++++++++++++ 10 files changed, 91 insertions(+), 9 deletions(-) diff --git a/.github/workflows/e2e.log.yaml b/.github/workflows/e2e.log.yaml index c8a4d9a..3eea86d 100644 --- a/.github/workflows/e2e.log.yaml +++ b/.github/workflows/e2e.log.yaml @@ -39,7 +39,7 @@ jobs: timeout-minutes: 90 strategy: matrix: - storage: ['h2', 'mysql', 'es6', 'es7', 'influxdb'] + storage: ['h2', 'mysql', 'es6', 'es7', 'es7.14', 'influxdb'] env: SW_STORAGE: ${{ matrix.storage }} steps: diff --git a/CHANGES.md b/CHANGES.md index 3e4bc9c..7998e75 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -23,6 +23,7 @@ Release Notes. * Trace segment wouldn't be recognized as a TopN sample service. Add through #4694 experimentally, but it caused performance impact. * Remove `version` and `endTime` in the segment entity. Reduce indexing payload. +* Fix `mapper_parsing_exception` in ElasticSearch 7.14. #### UI diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/alarm/AlarmRecord.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/alarm/AlarmRecord.java index 8fc26a3..b935005 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/alarm/AlarmRecord.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/alarm/AlarmRecord.java @@ -37,6 +37,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import static java.util.Objects.nonNull; import static org.apache.skywalking.oap.server.core.source.DefaultScopeDefine.ALARM; @Getter @@ -92,7 +93,9 @@ public class AlarmRecord extends Record { map.put(NAME, storageData.getName()); map.put(ID0, storageData.getId0()); map.put(ID1, storageData.getId1()); - map.put(ALARM_MESSAGE, storageData.getAlarmMessage()); + if (nonNull(storageData.getAlarmMessage())) { + map.put(ALARM_MESSAGE, storageData.getAlarmMessage()); + } map.put(START_TIME, storageData.getStartTime()); map.put(TIME_BUCKET, storageData.getTimeBucket()); map.put(RULE_NAME, storageData.getRuleName()); diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/endpoint/EndpointTraffic.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/endpoint/EndpointTraffic.java index e302904..90690ee 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/endpoint/EndpointTraffic.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/endpoint/EndpointTraffic.java @@ -35,6 +35,8 @@ import org.apache.skywalking.oap.server.core.source.DefaultScopeDefine; import org.apache.skywalking.oap.server.core.storage.StorageHashMapBuilder; import org.apache.skywalking.oap.server.core.storage.annotation.Column; +import static java.util.Objects.nonNull; + @Stream(name = EndpointTraffic.INDEX_NAME, scopeId = DefaultScopeDefine.ENDPOINT, builder = EndpointTraffic.Builder.class, processor = MetricsStreamProcessor.class) @MetricsExtension(supportDownSampling = false, supportUpdate = false) @@ -101,7 +103,9 @@ public class EndpointTraffic extends Metrics { public Map<String, Object> entity2Storage(EndpointTraffic storageData) { Map<String, Object> map = new HashMap<>(); map.put(SERVICE_ID, storageData.getServiceId()); - map.put(NAME, storageData.getName()); + if (nonNull(storageData.getName())) { + map.put(NAME, storageData.getName()); + } map.put(TIME_BUCKET, storageData.getTimeBucket()); return map; } diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/log/AbstractLogRecord.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/log/AbstractLogRecord.java index 8988d00..66557d7 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/log/AbstractLogRecord.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/log/AbstractLogRecord.java @@ -33,6 +33,8 @@ import org.apache.skywalking.oap.server.core.storage.StorageHashMapBuilder; import org.apache.skywalking.oap.server.core.storage.annotation.Column; import org.apache.skywalking.oap.server.library.util.CollectionUtils; +import static java.util.Objects.nonNull; + public abstract class AbstractLogRecord extends Record { public static final String SERVICE_ID = "service_id"; @@ -44,7 +46,6 @@ public abstract class AbstractLogRecord extends Record { public static final String SPAN_ID = "span_id"; public static final String CONTENT_TYPE = "content_type"; public static final String CONTENT = "content"; - public static final String CONTENT_TYPE_CLASS = "content_type_class"; public static final String TAGS_RAW_DATA = "tags_raw_data"; public static final String TIMESTAMP = "timestamp"; public static final String TAGS = "tags"; @@ -121,7 +122,9 @@ public abstract class AbstractLogRecord extends Record { map.put(SERVICE_ID, record.getServiceId()); map.put(SERVICE_INSTANCE_ID, record.getServiceInstanceId()); map.put(ENDPOINT_ID, record.getEndpointId()); - map.put(ENDPOINT_NAME, record.getEndpointName()); + if (nonNull(record.getEndpointName())) { + map.put(ENDPOINT_NAME, record.getEndpointName()); + } map.put(TRACE_ID, record.getTraceId()); map.put(TRACE_SEGMENT_ID, record.getTraceSegmentId()); map.put(SPAN_ID, record.getSpanId()); diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/segment/SegmentRecord.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/segment/SegmentRecord.java index 7af4419..3885b79 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/segment/SegmentRecord.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/segment/SegmentRecord.java @@ -36,6 +36,8 @@ import org.apache.skywalking.oap.server.core.storage.annotation.Column; import org.apache.skywalking.oap.server.core.storage.annotation.SuperDataset; import org.apache.skywalking.oap.server.library.util.CollectionUtils; +import static java.util.Objects.nonNull; + @SuperDataset @Stream(name = SegmentRecord.INDEX_NAME, scopeId = DefaultScopeDefine.SEGMENT, builder = SegmentRecord.Builder.class, processor = RecordStreamProcessor.class) public class SegmentRecord extends Record { @@ -119,7 +121,9 @@ public class SegmentRecord extends Record { map.put(TRACE_ID, storageData.getTraceId()); map.put(SERVICE_ID, storageData.getServiceId()); map.put(SERVICE_INSTANCE_ID, storageData.getServiceInstanceId()); - map.put(ENDPOINT_NAME, storageData.getEndpointName()); + if (nonNull(storageData.getEndpointName())) { + map.put(ENDPOINT_NAME, storageData.getEndpointName()); + } map.put(ENDPOINT_ID, storageData.getEndpointId()); map.put(START_TIME, storageData.getStartTime()); map.put(LATENCY, storageData.getLatency()); diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/service/ServiceTraffic.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/service/ServiceTraffic.java index d5ab808..623b002 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/service/ServiceTraffic.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/service/ServiceTraffic.java @@ -35,6 +35,7 @@ import org.apache.skywalking.oap.server.core.source.DefaultScopeDefine; import org.apache.skywalking.oap.server.core.storage.StorageHashMapBuilder; import org.apache.skywalking.oap.server.core.storage.annotation.Column; +import static java.util.Objects.nonNull; import static org.apache.skywalking.oap.server.core.Const.DOUBLE_COLONS_SPLIT; @Stream(name = ServiceTraffic.INDEX_NAME, scopeId = DefaultScopeDefine.SERVICE, @@ -115,7 +116,9 @@ public class ServiceTraffic extends Metrics { } } Map<String, Object> map = new HashMap<>(); - map.put(NAME, serviceName); + if (nonNull(serviceName)) { + map.put(NAME, serviceName); + } map.put(NODE_TYPE, storageData.getNodeType().value()); map.put(GROUP, storageData.getGroup()); return map; diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/browser/manual/errorlog/BrowserErrorLogRecord.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/browser/manual/errorlog/BrowserErrorLogRecord.java index 7cc404b..7a856c5 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/browser/manual/errorlog/BrowserErrorLogRecord.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/browser/manual/errorlog/BrowserErrorLogRecord.java @@ -33,6 +33,8 @@ import org.apache.skywalking.oap.server.core.storage.annotation.Column; import org.apache.skywalking.oap.server.core.storage.annotation.SuperDataset; import org.apache.skywalking.oap.server.library.util.CollectionUtils; +import static java.util.Objects.nonNull; + @SuperDataset @Stream(name = BrowserErrorLogRecord.INDEX_NAME, scopeId = DefaultScopeDefine.BROWSER_ERROR_LOG, builder = BrowserErrorLogRecord.Builder.class, processor = RecordStreamProcessor.class) public class BrowserErrorLogRecord extends Record { @@ -119,7 +121,9 @@ public class BrowserErrorLogRecord extends Record { map.put(SERVICE_ID, storageData.getServiceId()); map.put(SERVICE_VERSION_ID, storageData.getServiceVersionId()); map.put(PAGE_PATH_ID, storageData.getPagePathId()); - map.put(PAGE_PATH, storageData.getPagePath()); + if (nonNull(storageData.getPagePath())) { + map.put(PAGE_PATH, storageData.getPagePath()); + } map.put(TIMESTAMP, storageData.getTimestamp()); map.put(TIME_BUCKET, storageData.getTimeBucket()); map.put(ERROR_CATEGORY, storageData.getErrorCategory()); diff --git a/oap-server/server-storage-plugin/storage-zipkin-elasticsearch7-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/zipkin/ZipkinSpanRecord.java b/oap-server/server-storage-plugin/storage-zipkin-elasticsearch7-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/zipkin/ZipkinSpanRecord.java index 434b3eb..c3f90ef 100644 --- a/oap-server/server-storage-plugin/storage-zipkin-elasticsearch7-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/zipkin/ZipkinSpanRecord.java +++ b/oap-server/server-storage-plugin/storage-zipkin-elasticsearch7-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/zipkin/ZipkinSpanRecord.java @@ -35,6 +35,8 @@ import org.apache.skywalking.oap.server.core.storage.annotation.Column; import org.apache.skywalking.oap.server.core.storage.annotation.SuperDataset; import org.apache.skywalking.oap.server.library.util.CollectionUtils; +import static java.util.Objects.nonNull; + @SuperDataset @Stream(name = ZipkinSpanRecord.INDEX_NAME, scopeId = DefaultScopeDefine.ZIPKIN_SPAN, builder = ZipkinSpanRecord.Builder.class, processor = RecordStreamProcessor.class) public class ZipkinSpanRecord extends Record { @@ -120,7 +122,9 @@ public class ZipkinSpanRecord extends Record { map.put(SPAN_ID, storageData.getSpanId()); map.put(SERVICE_ID, storageData.getServiceId()); map.put(SERVICE_INSTANCE_ID, storageData.getServiceInstanceId()); - map.put(ENDPOINT_NAME, storageData.getEndpointName()); + if (nonNull(storageData.getEndpointName())) { + map.put(ENDPOINT_NAME, storageData.getEndpointName()); + } map.put(ENDPOINT_ID, storageData.getEndpointId()); map.put(START_TIME, storageData.getStartTime()); map.put(END_TIME, storageData.getEndTime()); diff --git a/test/e2e/e2e-test/docker/log/docker-compose.es7.14.yml b/test/e2e/e2e-test/docker/log/docker-compose.es7.14.yml new file mode 100644 index 0000000..2a7cadc --- /dev/null +++ b/test/e2e/e2e-test/docker/log/docker-compose.es7.14.yml @@ -0,0 +1,56 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +version: '2.1' + +services: + es: + image: elastic/elasticsearch:7.14.0 + expose: + - 9200 + networks: + - e2e + environment: + - discovery.type=single-node + healthcheck: + test: ["CMD", "bash", "-c", "cat < /dev/null > /dev/tcp/127.0.0.1/9200"] + interval: 5s + timeout: 60s + retries: 120 + + oap: + extends: + file: ../base-compose.yml + service: oap-es7 + environment: + SW_STORAGE: elasticsearch7 + SW_LOG_LAL_FILES: test + SW_LOG_MAL_FILES: test + volumes: + - ./lal.yaml:/skywalking/config/lal/test.yaml + - ./log-mal.yaml:/skywalking/config/log-mal-rules/test.yaml + depends_on: + es: + condition: service_healthy + + provider: + extends: + file: ../base-compose.yml + service: provider + depends_on: + oap: + condition: service_healthy +networks: + e2e:
