This is an automated email from the ASF dual-hosted git repository. ningjiang pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-servicecomb-java-chassis.git
The following commit(s) were added to refs/heads/master by this push: new a3e96ec [SCB-124] Fix 0.5.x metrics update patch bug (#462) a3e96ec is described below commit a3e96ecaf2a8e9421dd3410e16c9811aeb1a0c23 Author: zhengyangyong <yangyong.zh...@huawei.com> AuthorDate: Mon Dec 25 18:28:43 2017 +0800 [SCB-124] Fix 0.5.x metrics update patch bug (#462) * fix 0.5.x metrics to latest Signed-off-by: zhengyangyong <yangyong.zh...@huawei.com> * add metrics it test in springmvc demo Signed-off-by: zhengyangyong <yangyong.zh...@huawei.com> --- .../common/rest/AbstractRestInvocation.java | 61 ++++++++++++++++++++- .../common/rest/TestAbstractRestInvocation.java | 12 +++-- .../main/java/io/servicecomb/core/Invocation.java | 12 +++++ .../foundation/metrics/MetricsServoRegistry.java | 2 +- .../io/servicecomb/bizkeeper/BizkeeperHandler.java | 10 ++++ .../springmvc/consumer/SpringmvcConsumerMain.java | 24 +++++++++ .../src/main/resources/microservice.yaml | 5 +- .../springmvc/provider/SpringmvcHelloImpl.java | 32 +++++++++++ .../src/main/resources/microservice.yaml | 6 ++- .../transport/highway/HighwayServerInvoke.java | 63 +++++++++++++++++++--- 10 files changed, 209 insertions(+), 18 deletions(-) diff --git a/common/common-rest/src/main/java/io/servicecomb/common/rest/AbstractRestInvocation.java b/common/common-rest/src/main/java/io/servicecomb/common/rest/AbstractRestInvocation.java index 622448b..f7f5134 100644 --- a/common/common-rest/src/main/java/io/servicecomb/common/rest/AbstractRestInvocation.java +++ b/common/common-rest/src/main/java/io/servicecomb/common/rest/AbstractRestInvocation.java @@ -45,6 +45,9 @@ import io.servicecomb.core.definition.OperationMeta; import io.servicecomb.core.metrics.InvocationStartedEvent; import io.servicecomb.foundation.common.utils.EventUtils; import io.servicecomb.foundation.common.utils.JsonUtils; +import io.servicecomb.foundation.metrics.MetricsServoRegistry; +import io.servicecomb.foundation.metrics.performance.QueueMetrics; +import io.servicecomb.foundation.metrics.performance.QueueMetricsData; import io.servicecomb.foundation.vertx.http.HttpServletRequestEx; import io.servicecomb.foundation.vertx.http.HttpServletResponseEx; import io.servicecomb.foundation.vertx.stream.BufferOutputStream; @@ -109,6 +112,8 @@ public abstract class AbstractRestInvocation { System.nanoTime()); EventUtils.triggerEvent(startedEvent); + QueueMetrics metricsData = initMetrics(operationMeta); + operationMeta.getExecutor().execute(() -> { synchronized (this.requestEx) { try { @@ -121,7 +126,7 @@ public abstract class AbstractRestInvocation { return; } - runOnExecutor(startedEvent); + runOnExecutor(metricsData,startedEvent); } catch (Throwable e) { LOGGER.error("rest server onRequest error", e); sendFailResponse(e); @@ -130,10 +135,13 @@ public abstract class AbstractRestInvocation { }); } - protected void runOnExecutor(InvocationStartedEvent startedEvent) { + protected void runOnExecutor(QueueMetrics metricsData,InvocationStartedEvent startedEvent) { Object[] args = RestCodec.restToArgs(requestEx, restOperationMeta); createInvocation(args); + this.invocation.setMetricsData(metricsData); + updateMetrics(); + //立刻设置开始时间,否则Finished时无法计算TotalTime invocation.setStartTime(startedEvent.getStartedTime()); invocation.triggerStartProcessingEvent(); @@ -181,6 +189,7 @@ public abstract class AbstractRestInvocation { sendResponseQuietly(resp); invocation.triggerFinishedEvent(); + endMetrics(); }); } @@ -235,4 +244,52 @@ public abstract class AbstractRestInvocation { responseEx.flushBuffer(); } } + + /** + * Init the metrics. Note down the queue count and start time. + * @param operationMeta Operation data + * @return QueueMetrics + */ + private QueueMetrics initMetrics(OperationMeta operationMeta) { + QueueMetrics metricsData = new QueueMetrics(); + metricsData.setQueueStartTime(System.currentTimeMillis()); + metricsData.setOperQualifiedName(operationMeta.getMicroserviceQualifiedName()); + QueueMetricsData reqQueue = MetricsServoRegistry.getOrCreateLocalMetrics() + .getOrCreateQueueMetrics(operationMeta.getMicroserviceQualifiedName()); + reqQueue.incrementCountInQueue(); + return metricsData; + } + + /** + * Update the queue metrics. + */ + private void updateMetrics() { + QueueMetrics metricsData = (QueueMetrics) this.invocation.getMetricsData(); + if (null != metricsData) { + metricsData.setQueueEndTime(System.currentTimeMillis()); + QueueMetricsData reqQueue = MetricsServoRegistry.getOrCreateLocalMetrics() + .getOrCreateQueueMetrics(restOperationMeta.getOperationMeta().getMicroserviceQualifiedName()); + reqQueue.incrementTotalCount(); + Long timeInQueue = metricsData.getQueueEndTime() - metricsData.getQueueStartTime(); + reqQueue.setTotalTime(reqQueue.getTotalTime() + timeInQueue); + reqQueue.setMinLifeTimeInQueue(timeInQueue); + reqQueue.setMaxLifeTimeInQueue(timeInQueue); + reqQueue.decrementCountInQueue(); + } + } + + /** + * Prepare the end time of queue metrics. + */ + private void endMetrics() { + QueueMetrics metricsData = (QueueMetrics) this.invocation.getMetricsData(); + if (null != metricsData) { + metricsData.setEndOperTime(System.currentTimeMillis()); + QueueMetricsData reqQueue = MetricsServoRegistry.getOrCreateLocalMetrics() + .getOrCreateQueueMetrics(restOperationMeta.getOperationMeta().getMicroserviceQualifiedName()); + reqQueue.incrementTotalServExecutionCount(); + reqQueue.setTotalServExecutionTime( + reqQueue.getTotalServExecutionTime() + (metricsData.getEndOperTime() - metricsData.getQueueEndTime())); + } + } } diff --git a/common/common-rest/src/test/java/io/servicecomb/common/rest/TestAbstractRestInvocation.java b/common/common-rest/src/test/java/io/servicecomb/common/rest/TestAbstractRestInvocation.java index 589de7d..ddab596 100644 --- a/common/common-rest/src/test/java/io/servicecomb/common/rest/TestAbstractRestInvocation.java +++ b/common/common-rest/src/test/java/io/servicecomb/common/rest/TestAbstractRestInvocation.java @@ -51,6 +51,9 @@ import io.servicecomb.core.definition.SchemaMeta; import io.servicecomb.core.executor.ReactiveExecutor; import io.servicecomb.core.provider.consumer.ReferenceConfig; import io.servicecomb.foundation.common.utils.JsonUtils; +import io.servicecomb.foundation.metrics.MetricsServoRegistry; +import io.servicecomb.foundation.metrics.performance.QueueMetrics; +import io.servicecomb.foundation.metrics.performance.QueueMetricsData; import io.servicecomb.foundation.vertx.http.AbstractHttpServletRequest; import io.servicecomb.foundation.vertx.http.HttpServletRequestEx; import io.servicecomb.foundation.vertx.http.HttpServletResponseEx; @@ -371,6 +374,7 @@ public class TestAbstractRestInvocation { restInvocation.sendResponseQuietly(response); Assert.assertSame(response, result.value); + } @Test @@ -621,7 +625,7 @@ public class TestAbstractRestInvocation { Error error = new Error("run on executor"); restInvocation = new AbstractRestInvocationForTest() { @Override - protected void runOnExecutor(InvocationStartedEvent startedEvent) { + protected void runOnExecutor(QueueMetrics metricsData,InvocationStartedEvent startedEvent) { throw error; } @@ -658,7 +662,7 @@ public class TestAbstractRestInvocation { restInvocation = new AbstractRestInvocationForTest() { @Override - protected void runOnExecutor(InvocationStartedEvent startedEvent) { + protected void runOnExecutor(QueueMetrics metricsData,InvocationStartedEvent startedEvent) { throw new Error("run on executor"); } @@ -694,7 +698,7 @@ public class TestAbstractRestInvocation { Holder<Boolean> result = new Holder<>(); restInvocation = new AbstractRestInvocationForTest() { @Override - protected void runOnExecutor(InvocationStartedEvent startedEvent) { + protected void runOnExecutor(QueueMetrics metricsData,InvocationStartedEvent startedEvent) { result.value = true; } }; @@ -725,7 +729,7 @@ public class TestAbstractRestInvocation { restInvocation.requestEx = requestEx; restInvocation.restOperationMeta = restOperation; - restInvocation.runOnExecutor(new InvocationStartedEvent("", System.nanoTime())); + restInvocation.runOnExecutor(null, new InvocationStartedEvent("", System.nanoTime())); Assert.assertTrue(result.value); Assert.assertSame(invocation, restInvocation.invocation); } diff --git a/core/src/main/java/io/servicecomb/core/Invocation.java b/core/src/main/java/io/servicecomb/core/Invocation.java index 7cc5f36..ec9abe6 100644 --- a/core/src/main/java/io/servicecomb/core/Invocation.java +++ b/core/src/main/java/io/servicecomb/core/Invocation.java @@ -53,6 +53,18 @@ public class Invocation extends SwaggerInvocation { private int handlerIndex; + //start,end of queue and operation time after queue for operation level metrics. + private Object metricsData; + + public Object getMetricsData() { + return metricsData; + } + + public void setMetricsData(Object metricsData) { + this.metricsData = metricsData; + } + + // 应答的处理器 // 同步模式:避免应答在网络线程中处理解码等等业务级逻辑 private Executor responseExecutor; diff --git a/foundations/foundation-metrics/src/main/java/io/servicecomb/foundation/metrics/MetricsServoRegistry.java b/foundations/foundation-metrics/src/main/java/io/servicecomb/foundation/metrics/MetricsServoRegistry.java index 6d19905..2e53265 100644 --- a/foundations/foundation-metrics/src/main/java/io/servicecomb/foundation/metrics/MetricsServoRegistry.java +++ b/foundations/foundation-metrics/src/main/java/io/servicecomb/foundation/metrics/MetricsServoRegistry.java @@ -532,7 +532,7 @@ public class MetricsServoRegistry implements InitializingBean { * * @return List of monitors */ - private List<Monitor<?>> getMetricsMonitors() { + public List<Monitor<?>> getMetricsMonitors() { List<Monitor<?>> monitors = new ArrayList<>(); monitors.add(getRequestValuesGaugeMonitor("totalRequestsPerProvider INSTANCE_LEVEL", diff --git a/handlers/handler-bizkeeper/src/main/java/io/servicecomb/bizkeeper/BizkeeperHandler.java b/handlers/handler-bizkeeper/src/main/java/io/servicecomb/bizkeeper/BizkeeperHandler.java index 0c53be7..5c3f166 100644 --- a/handlers/handler-bizkeeper/src/main/java/io/servicecomb/bizkeeper/BizkeeperHandler.java +++ b/handlers/handler-bizkeeper/src/main/java/io/servicecomb/bizkeeper/BizkeeperHandler.java @@ -29,6 +29,7 @@ import com.netflix.hystrix.strategy.executionhook.HystrixCommandExecutionHook; import io.servicecomb.core.Handler; import io.servicecomb.core.Invocation; +import io.servicecomb.foundation.metrics.performance.MetricsDataMonitorUtil; import io.servicecomb.swagger.invocation.AsyncResponse; import io.servicecomb.swagger.invocation.Response; import rx.Observable; @@ -79,10 +80,19 @@ public abstract class BizkeeperHandler implements Handler { public void handle(Invocation invocation, AsyncResponse asyncResp) throws Exception { HystrixObservable<Response> command = delegate.createBizkeeperCommand(invocation); + new MetricsDataMonitorUtil() + .setAllReqProviderAndConsumer(invocation.getOperationMeta().getMicroserviceQualifiedName(), + String.valueOf(invocation.getInvocationType())); + Observable<Response> observable = command.toObservable(); observable.subscribe(asyncResp::complete, error -> { LOG.warn("catch error in bizkeeper:" + error.getMessage()); asyncResp.fail(invocation.getInvocationType(), error); + + new MetricsDataMonitorUtil() + .setAllFailReqProviderAndConsumer(invocation.getOperationMeta().getMicroserviceQualifiedName(), + String.valueOf(invocation.getInvocationType())); + }, () -> { }); diff --git a/samples/springmvc-sample/springmvc-consumer/src/main/java/io/servicecomb/samples/springmvc/consumer/SpringmvcConsumerMain.java b/samples/springmvc-sample/springmvc-consumer/src/main/java/io/servicecomb/samples/springmvc/consumer/SpringmvcConsumerMain.java index 9ad4875..929f38d 100644 --- a/samples/springmvc-sample/springmvc-consumer/src/main/java/io/servicecomb/samples/springmvc/consumer/SpringmvcConsumerMain.java +++ b/samples/springmvc-sample/springmvc-consumer/src/main/java/io/servicecomb/samples/springmvc/consumer/SpringmvcConsumerMain.java @@ -16,10 +16,15 @@ */ package io.servicecomb.samples.springmvc.consumer; +import java.util.HashMap; +import java.util.Map; + import org.springframework.stereotype.Component; import org.springframework.web.client.RestTemplate; +import io.servicecomb.foundation.common.exceptions.ServiceCombException; import io.servicecomb.foundation.common.utils.BeanUtils; +import io.servicecomb.foundation.common.utils.JsonUtils; import io.servicecomb.foundation.common.utils.Log4jUtils; import io.servicecomb.provider.pojo.RpcReference; import io.servicecomb.provider.springmvc.reference.RestTemplateBuilder; @@ -51,6 +56,25 @@ public class SpringmvcConsumerMain { // POJO Consumer System.out.println("POJO consumer sayhi services: " + hello.sayHi("Java Chassis")); System.out.println("POJO consumer sayhi services: " + hello.sayHello(person)); + + String metricsResult = restTemplate.getForObject("cse://springmvc/springmvchello/metricsForTest/", String.class); + + Map<String, String> resultMap = JsonUtils.OBJ_MAPPER.readValue(metricsResult, HashMap.class); + + if (!resultMap.get("CPU and Memory").contains("heapUsed=")) { + throw new ServiceCombException("check metrics error"); + } + + if (!resultMap.get("totalRequestProvider OPERATIONAL_LEVEL") + .equals("{springmvc.springmvcHello.sayHi=2, springmvc.springmvcHello.sayHello=2, springmvc.springmvcHello.metricsForTest=1}")) { + throw new ServiceCombException("check metrics error"); + } + + if (!resultMap.get("RequestQueueRelated").contains("springmvc.springmvcHello.sayHi") || + !resultMap.get("RequestQueueRelated").contains("springmvc.springmvcHello.sayHello")){ + throw new ServiceCombException("check metrics error"); + } + } public static void init() throws Exception { diff --git a/samples/springmvc-sample/springmvc-consumer/src/main/resources/microservice.yaml b/samples/springmvc-sample/springmvc-consumer/src/main/resources/microservice.yaml index 253e9cc..8c25b99 100644 --- a/samples/springmvc-sample/springmvc-consumer/src/main/resources/microservice.yaml +++ b/samples/springmvc-sample/springmvc-consumer/src/main/resources/microservice.yaml @@ -18,7 +18,7 @@ APPLICATION_ID: springmvc-sample service_description: name: springmvcConsumer - version: 0.0.1 + version: 0.0.2 cse: service: registry: @@ -30,6 +30,3 @@ cse: isolation: Consumer: enabled: false - references: - springmvc: - version-rule: 0.0.1 diff --git a/samples/springmvc-sample/springmvc-provider/src/main/java/io/servicecomb/samples/springmvc/provider/SpringmvcHelloImpl.java b/samples/springmvc-sample/springmvc-provider/src/main/java/io/servicecomb/samples/springmvc/provider/SpringmvcHelloImpl.java index cceeac6..a5f2317 100644 --- a/samples/springmvc-sample/springmvc-provider/src/main/java/io/servicecomb/samples/springmvc/provider/SpringmvcHelloImpl.java +++ b/samples/springmvc-sample/springmvc-provider/src/main/java/io/servicecomb/samples/springmvc/provider/SpringmvcHelloImpl.java @@ -18,13 +18,23 @@ package io.servicecomb.samples.springmvc.provider; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + import javax.ws.rs.core.MediaType; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestMethod; import org.springframework.web.bind.annotation.RequestParam; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.netflix.servo.monitor.Monitor; + +import io.servicecomb.foundation.common.utils.JsonUtils; +import io.servicecomb.foundation.metrics.MetricsServoRegistry; import io.servicecomb.provider.rest.common.RestSchema; import io.servicecomb.samples.common.schema.Hello; import io.servicecomb.samples.common.schema.models.Person; @@ -33,6 +43,13 @@ import io.servicecomb.samples.common.schema.models.Person; @RequestMapping(path = "/springmvchello", produces = MediaType.APPLICATION_JSON) public class SpringmvcHelloImpl implements Hello { + private MetricsServoRegistry registry; + + @Autowired + public SpringmvcHelloImpl(MetricsServoRegistry registry) { + this.registry = registry; + } + @Override @RequestMapping(path = "/sayhi", method = RequestMethod.POST) public String sayHi(@RequestParam(name = "name") String name) { @@ -44,4 +61,19 @@ public class SpringmvcHelloImpl implements Hello { public String sayHello(@RequestBody Person person) { return "Hello person " + person.getName(); } + + //metrics it test + @RequestMapping(path = "/metricsForTest", method = RequestMethod.GET) + public String metricsForTest() { + List<Monitor<?>> monitors = registry.getMetricsMonitors(); + Map<String, String> values = new HashMap<>(); + for (Monitor<?> monitor : monitors) { + values.put(monitor.getConfig().getName(), monitor.getValue().toString()); + } + try { + return JsonUtils.writeValueAsString(values); + } catch (JsonProcessingException e) { + return "{}"; + } + } } diff --git a/samples/springmvc-sample/springmvc-provider/src/main/resources/microservice.yaml b/samples/springmvc-sample/springmvc-provider/src/main/resources/microservice.yaml index e4fe45f..fb9d26f 100644 --- a/samples/springmvc-sample/springmvc-provider/src/main/resources/microservice.yaml +++ b/samples/springmvc-sample/springmvc-provider/src/main/resources/microservice.yaml @@ -18,7 +18,7 @@ APPLICATION_ID: springmvc-sample service_description: name: springmvc - version: 0.0.1 + version: 0.0.2 cse: service: registry: @@ -27,3 +27,7 @@ cse: address: 0.0.0.0:8080 highway: address: 0.0.0.0:7070 + handler: + chain: + Provider: + default: bizkeeper-provider diff --git a/transports/transport-highway/src/main/java/io/servicecomb/transport/highway/HighwayServerInvoke.java b/transports/transport-highway/src/main/java/io/servicecomb/transport/highway/HighwayServerInvoke.java index 1ead7a4..68e95e3 100644 --- a/transports/transport-highway/src/main/java/io/servicecomb/transport/highway/HighwayServerInvoke.java +++ b/transports/transport-highway/src/main/java/io/servicecomb/transport/highway/HighwayServerInvoke.java @@ -35,6 +35,9 @@ import io.servicecomb.core.definition.OperationMeta; import io.servicecomb.core.definition.SchemaMeta; import io.servicecomb.core.metrics.InvocationStartedEvent; import io.servicecomb.foundation.common.utils.EventUtils; +import io.servicecomb.foundation.metrics.MetricsServoRegistry; +import io.servicecomb.foundation.metrics.performance.QueueMetrics; +import io.servicecomb.foundation.metrics.performance.QueueMetricsData; import io.servicecomb.foundation.vertx.tcp.TcpConnection; import io.servicecomb.swagger.invocation.Response; import io.servicecomb.swagger.invocation.exception.InvocationException; @@ -106,9 +109,9 @@ public class HighwayServerInvoke { this.bodyBuffer = bodyBuffer; } - private void runInExecutor(InvocationStartedEvent startedEvent) { + private void runInExecutor(QueueMetrics metricsData,InvocationStartedEvent startedEvent) { try { - doRunInExecutor(startedEvent); + doRunInExecutor(metricsData,startedEvent); } catch (Throwable e) { String msg = String.format("handle request error, %s, msgId=%d", operationMeta.getMicroserviceQualifiedName(), @@ -119,17 +122,17 @@ public class HighwayServerInvoke { } } - private void doRunInExecutor(InvocationStartedEvent startedEvent) throws Exception { + private void doRunInExecutor(QueueMetrics metricsData,InvocationStartedEvent startedEvent) throws Exception { Invocation invocation = HighwayCodec.decodeRequest(header, operationProtobuf, bodyBuffer, protobufFeature); invocation.getHandlerContext().put(Const.REMOTE_ADDRESS, this.connection.getNetSocket().remoteAddress()); - + updateMetrics(invocation); //立刻设置开始时间,否则Finished时无法计算TotalTime invocation.setStartTime(startedEvent.getStartedTime()); invocation.triggerStartProcessingEvent(); invocation.next(response -> { sendResponse(invocation.getContext(), response); - + endMetrics(invocation); invocation.triggerFinishedEvent(); }); } @@ -166,7 +169,55 @@ public class HighwayServerInvoke { InvocationStartedEvent startedEvent = new InvocationStartedEvent(operationMeta.getMicroserviceQualifiedName(), System.nanoTime()); EventUtils.triggerEvent(startedEvent); + QueueMetrics metricsData = initMetrics(operationMeta); + operationMeta.getExecutor().execute(() -> runInExecutor(metricsData,startedEvent)); + } + + /** + * Init the metrics. Note down the queue count and start time. + * @param operationMeta Operation data + * @return QueueMetrics + */ + private QueueMetrics initMetrics(OperationMeta operationMeta) { + QueueMetrics metricsData = new QueueMetrics(); + metricsData.setQueueStartTime(System.currentTimeMillis()); + metricsData.setOperQualifiedName(operationMeta.getMicroserviceQualifiedName()); + QueueMetricsData reqQueue = MetricsServoRegistry.getOrCreateLocalMetrics() + .getOrCreateQueueMetrics(operationMeta.getMicroserviceQualifiedName()); + reqQueue.incrementCountInQueue(); + return metricsData; + } - operationMeta.getExecutor().execute(() -> runInExecutor(startedEvent)); + /** + * Update the queue metrics. + */ + private void updateMetrics(Invocation invocation) { + QueueMetrics metricsData = (QueueMetrics) invocation.getMetricsData(); + if (null != metricsData) { + metricsData.setQueueEndTime(System.currentTimeMillis()); + QueueMetricsData reqQueue = MetricsServoRegistry.getOrCreateLocalMetrics() + .getOrCreateQueueMetrics(operationMeta.getMicroserviceQualifiedName()); + reqQueue.incrementTotalCount(); + Long timeInQueue = metricsData.getQueueEndTime() - metricsData.getQueueStartTime(); + reqQueue.setTotalTime(reqQueue.getTotalTime() + timeInQueue); + reqQueue.setMinLifeTimeInQueue(timeInQueue); + reqQueue.setMaxLifeTimeInQueue(timeInQueue); + reqQueue.decrementCountInQueue(); + } + } + + /** + * Prepare the end time of queue metrics. + */ + private void endMetrics(Invocation invocation) { + QueueMetrics metricsData = (QueueMetrics) invocation.getMetricsData(); + if (null != metricsData) { + metricsData.setEndOperTime(System.currentTimeMillis()); + QueueMetricsData reqQueue = MetricsServoRegistry.getOrCreateLocalMetrics() + .getOrCreateQueueMetrics(operationMeta.getMicroserviceQualifiedName()); + reqQueue.incrementTotalServExecutionCount(); + reqQueue.setTotalServExecutionTime( + reqQueue.getTotalServExecutionTime() + (metricsData.getEndOperTime() - metricsData.getQueueEndTime())); + } } } -- To stop receiving notification emails like this one, please contact ['"commits@servicecomb.apache.org" <commits@servicecomb.apache.org>'].