[
https://issues.apache.org/jira/browse/BEAM-3802?focusedWorklogId=79025&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-79025
]
ASF GitHub Bot logged work on BEAM-3802:
----------------------------------------
Author: ASF GitHub Bot
Created on: 09/Mar/18 21:28
Start Date: 09/Mar/18 21:28
Worklog Time Spent: 10m
Work Description: kennknowles closed pull request #4824: [BEAM-3802] Move
metrics caching up a level
URL: https://github.com/apache/beam/pull/4824
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git
a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowMetrics.java
b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowMetrics.java
index 0983674b983..8c93e97e754 100644
---
a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowMetrics.java
+++
b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowMetrics.java
@@ -63,7 +63,7 @@
* After the job has finished running, Metrics no longer will change, so
their results are
* cached here.
*/
- private MetricQueryResults cachedMetricResults = null;
+ private JobMetrics cachedMetricResults = null;
/**
* Constructor for the DataflowMetrics class.
@@ -88,14 +88,15 @@ private MetricQueryResults populateMetricQueryResults(
.build();
}
- private MetricQueryResults queryServiceForMetrics(MetricsFilter filter) {
+ @Override
+ public MetricQueryResults queryMetrics(MetricsFilter filter) {
List<com.google.api.services.dataflow.model.MetricUpdate> metricUpdates;
ImmutableList<MetricResult<Long>> counters = ImmutableList.of();
ImmutableList<MetricResult<DistributionResult>> distributions =
ImmutableList.of();
ImmutableList<MetricResult<GaugeResult>> gauges = ImmutableList.of();
JobMetrics jobMetrics;
try {
- jobMetrics = dataflowClient.getJobMetrics(dataflowPipelineJob.jobId);
+ jobMetrics = getJobMetrics();
} catch (IOException e) {
LOG.warn("Unable to query job metrics.\n");
return DataflowMetricQueryResults.create(counters, distributions,
gauges);
@@ -106,17 +107,12 @@ private MetricQueryResults
queryServiceForMetrics(MetricsFilter filter) {
return populateMetricQueryResults(metricUpdates, filter);
}
- public MetricQueryResults queryMetrics() {
- return queryMetrics(null);
- }
-
- @Override
- public MetricQueryResults queryMetrics(MetricsFilter filter) {
+ private JobMetrics getJobMetrics() throws IOException {
if (cachedMetricResults != null) {
// Metric results have been cached after the job ran.
return cachedMetricResults;
}
- MetricQueryResults result = queryServiceForMetrics(filter);
+ JobMetrics result =
dataflowClient.getJobMetrics(dataflowPipelineJob.jobId);
if (dataflowPipelineJob.getState().isTerminal()) {
// Add current query result to the cache.
cachedMetricResults = result;
diff --git
a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowMetricsTest.java
b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowMetricsTest.java
index baf02114179..571c2481a45 100644
---
a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowMetricsTest.java
+++
b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowMetricsTest.java
@@ -109,7 +109,7 @@ public void testEmptyMetricUpdates() throws IOException {
when(dataflowClient.getJobMetrics(JOB_ID)).thenReturn(jobMetrics);
DataflowMetrics dataflowMetrics = new DataflowMetrics(job, dataflowClient);
- MetricQueryResults result = dataflowMetrics.queryMetrics();
+ MetricQueryResults result = dataflowMetrics.queryMetrics(null);
assertThat(ImmutableList.copyOf(result.counters()), is(empty()));
assertThat(ImmutableList.copyOf(result.distributions()), is(empty()));
}
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
Issue Time Tracking
-------------------
Worklog Id: (was: 79025)
Time Spent: 1h 20m (was: 1h 10m)
> Should be able to call dataflow queryMetrics more then once
> -----------------------------------------------------------
>
> Key: BEAM-3802
> URL: https://issues.apache.org/jira/browse/BEAM-3802
> Project: Beam
> Issue Type: Bug
> Components: runner-dataflow
> Reporter: Andrew Pilloud
> Assignee: Andrew Pilloud
> Priority: Major
> Time Spent: 1h 20m
> Remaining Estimate: 0h
>
> When you queryMetrics on the dataflow runner against a batch job, you always
> get the set of metrics as filtered by the first call to queryMetrics.
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
