This is an automated email from the ASF dual-hosted git repository.
anandinguva pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 357ff1fc012 Refactor code to make the UI more readable. Look at
https://github.com/apache/beam/issues/27005#issuecomment-1599358169 (#27192)
357ff1fc012 is described below
commit 357ff1fc01249692abae59036ba8e73fc099aeeb
Author: Anand Inguva <[email protected]>
AuthorDate: Thu Jun 22 18:18:35 2023 -0400
Refactor code to make the UI more readable. Look at
https://github.com/apache/beam/issues/27005#issuecomment-1599358169 (#27192)
---
.../apache_beam/testing/analyzers/constants.py | 6 +-
.../testing/analyzers/github_issues_utils.py | 19 ++++-
.../apache_beam/testing/analyzers/perf_analysis.py | 24 +++---
.../testing/analyzers/perf_analysis_utils.py | 6 +-
.../testing/analyzers/tests_config.yaml | 87 ++++++++++++++--------
5 files changed, 92 insertions(+), 50 deletions(-)
diff --git a/sdks/python/apache_beam/testing/analyzers/constants.py
b/sdks/python/apache_beam/testing/analyzers/constants.py
index c0df05f61db..2a52fab563d 100644
--- a/sdks/python/apache_beam/testing/analyzers/constants.py
+++ b/sdks/python/apache_beam/testing/analyzers/constants.py
@@ -37,7 +37,11 @@ _DEFAULT_MIN_RUNS_BETWEEN_CHANGE_POINTS = 3
_DEFAULT_NUM_RUMS_IN_CHANGE_POINT_WINDOW = 14
_PERF_TEST_KEYS = {
- 'test_name', 'metrics_dataset', 'metrics_table', 'project', 'metric_name'
+ 'test_description',
+ 'metrics_dataset',
+ 'metrics_table',
+ 'project',
+ 'metric_name'
}
_SCHEMA = [{
diff --git a/sdks/python/apache_beam/testing/analyzers/github_issues_utils.py
b/sdks/python/apache_beam/testing/analyzers/github_issues_utils.py
index b53768edcab..f6e2939161e 100644
--- a/sdks/python/apache_beam/testing/analyzers/github_issues_utils.py
+++ b/sdks/python/apache_beam/testing/analyzers/github_issues_utils.py
@@ -54,7 +54,7 @@ _ISSUE_DESCRIPTION_TEMPLATE = """
For more information on how to triage the alerts, please look at
`Triage performance alert issues` section of the
[README](https://github.com/apache/beam/tree/master/sdks/python/apache_beam/testing/analyzers/README.md#triage-performance-alert-issues).
"""
-_METRIC_INFO_TEMPLATE = "timestamp: {}, metric_value: `{}`"
+_METRIC_INFO_TEMPLATE = "timestamp: {}, metric_value: {}"
_AWAITING_TRIAGE_LABEL = 'awaiting triage'
_PERF_ALERT_LABEL = 'perf-alert'
@@ -123,6 +123,7 @@ def comment_on_issue(issue_number: int,
'body': comment_description,
issue_number: issue_number,
}
+
response = requests.post(
open_issue_response['comments_url'], json.dumps(data),
headers=_HEADERS)
return True, response.json()['html_url']
@@ -142,7 +143,9 @@ def get_issue_description(
timestamps: List[pd.Timestamp],
metric_values: List,
change_point_index: int,
- max_results_to_display: int = 5) -> str:
+ max_results_to_display: int = 5,
+ test_description: Optional[str] = None,
+) -> str:
"""
Args:
metric_name: Metric name used for the Change Point Analysis.
@@ -166,13 +169,21 @@ def get_issue_description(
description = _ISSUE_DESCRIPTION_TEMPLATE.format(
test_name, metric_name) + 2 * '\n'
+ description += (
+ "`Test description:` " + f'{test_description}' +
+ 2 * '\n') if test_description else ''
+
+ description += '```' + '\n'
runs_to_display = [
- _METRIC_INFO_TEMPLATE.format(timestamps[i].ctime(), metric_values[i])
+ _METRIC_INFO_TEMPLATE.format(
+ timestamps[i].ctime(), format(metric_values[i], '.2f'))
for i in reversed(range(min_timestamp_index, max_timestamp_index + 1))
]
runs_to_display[change_point_index - min_timestamp_index] += " <---- Anomaly"
- return description + '\n'.join(runs_to_display)
+ description += '\n'.join(runs_to_display) + '\n'
+ description += '```' + '\n'
+ return description
def report_change_point_on_issues(
diff --git a/sdks/python/apache_beam/testing/analyzers/perf_analysis.py
b/sdks/python/apache_beam/testing/analyzers/perf_analysis.py
index ee00e8abf42..e5e8a3a7b2f 100644
--- a/sdks/python/apache_beam/testing/analyzers/perf_analysis.py
+++ b/sdks/python/apache_beam/testing/analyzers/perf_analysis.py
@@ -46,7 +46,7 @@ from apache_beam.testing.analyzers.perf_analysis_utils import
validate_config
from apache_beam.testing.load_tests.load_test_metrics_utils import
BigQueryMetricsFetcher
-def run_change_point_analysis(params, test_id, big_query_metrics_fetcher):
+def run_change_point_analysis(params, test_name, big_query_metrics_fetcher):
"""
Args:
params: Dict containing parameters to run change point analysis.
@@ -56,13 +56,13 @@ def run_change_point_analysis(params, test_id,
big_query_metrics_fetcher):
Returns:
bool indicating if a change point is observed and alerted on GitHub.
"""
+ logging.info("Running change point analysis for test %s" % test_name)
if not validate_config(params.keys()):
raise ValueError(
f"Please make sure all these keys {constants._PERF_TEST_KEYS} "
- f"are specified for the {test_id}")
+ f"are specified for the {test_name}")
metric_name = params['metric_name']
- test_name = params['test_name'].replace('.', '_') + f'_{metric_name}'
min_runs_between_change_points = (
constants._DEFAULT_MIN_RUNS_BETWEEN_CHANGE_POINTS)
@@ -96,7 +96,7 @@ def run_change_point_analysis(params, test_id,
big_query_metrics_fetcher):
'on metric %s. Since the change point run %s '
'lies outside the num_runs_in_change_point_window distance: %s, '
'alert is not raised.' % (
- params['test_name'],
+ test_name,
metric_name,
latest_change_point_run + 1,
num_runs_in_change_point_window))
@@ -114,27 +114,29 @@ def run_change_point_analysis(params, test_id,
big_query_metrics_fetcher):
constants._CHANGE_POINT_TIMESTAMP_LABEL].tolist()
last_reported_issue_number = existing_issue_data[
constants._ISSUE_NUMBER].tolist()[0]
+ # convert numpy.int64 to int
+ last_reported_issue_number = last_reported_issue_number.item()
is_alert = is_perf_alert(
previous_change_point_timestamps=existing_issue_timestamps,
change_point_index=change_point_index,
timestamps=timestamps,
min_runs_between_change_points=min_runs_between_change_points)
- logging.debug(
- "Performance alert is %s for test %s" % (is_alert, params['test_name']))
+ logging.debug("Performance alert is %s for test %s" % (is_alert, test_name))
if is_alert:
issue_number, issue_url = create_performance_alert(
- metric_name, params['test_name'], timestamps,
+ metric_name, test_name, timestamps,
metric_values, change_point_index,
params.get('labels', None),
last_reported_issue_number,
- test_target=params['test_target'] if 'test_target' in params else None
+ test_description = params.get('test_description', None),
)
issue_metadata = GitHubIssueMetaData(
issue_timestamp=pd.Timestamp(
datetime.now().replace(tzinfo=timezone.utc)),
- test_name=test_name,
+ # BQ doesn't allow '.' in table name
+ test_name=test_name.replace('.', '_'),
metric_name=metric_name,
test_id=uuid.uuid4().hex,
change_point=metric_values[change_point_index],
@@ -170,8 +172,8 @@ def run(config_file_path: Optional[str] = None) -> None:
big_query_metrics_fetcher = BigQueryMetricsFetcher()
- for test_id, params in tests_config.items():
- run_change_point_analysis(params, test_id, big_query_metrics_fetcher)
+ for test_name, params in tests_config.items():
+ run_change_point_analysis(params, test_name, big_query_metrics_fetcher)
if __name__ == '__main__':
diff --git a/sdks/python/apache_beam/testing/analyzers/perf_analysis_utils.py
b/sdks/python/apache_beam/testing/analyzers/perf_analysis_utils.py
index ec74f206ce8..9e8d4d6f27f 100644
--- a/sdks/python/apache_beam/testing/analyzers/perf_analysis_utils.py
+++ b/sdks/python/apache_beam/testing/analyzers/perf_analysis_utils.py
@@ -187,14 +187,14 @@ def create_performance_alert(
change_point_index: int,
labels: List[str],
existing_issue_number: Optional[int],
- test_target: Optional[str] = None) -> Tuple[int, str]:
+ test_description: Optional[str] = None) -> Tuple[int, str]:
"""
Creates performance alert on GitHub issues and returns GitHub issue
number and issue URL.
"""
description = github_issues_utils.get_issue_description(
- test_name=(
- test_name if not test_target else test_name + ':' + test_target),
+ test_name=test_name,
+ test_description=test_description,
metric_name=metric_name,
timestamps=timestamps,
metric_values=metric_values,
diff --git a/sdks/python/apache_beam/testing/analyzers/tests_config.yaml
b/sdks/python/apache_beam/testing/analyzers/tests_config.yaml
index 02e649c7586..bb8a1a04238 100644
--- a/sdks/python/apache_beam/testing/analyzers/tests_config.yaml
+++ b/sdks/python/apache_beam/testing/analyzers/tests_config.yaml
@@ -15,64 +15,89 @@
# limitations under the License.
#
-test_1:
- test_name: Pytorch image classification on 50k images of size 224 x 224 with
resnet 152
- test_target:
apache_beam.testing.benchmarks.inference.pytorch_image_classification_benchmarks
+# for the unique key to define a test, please use the following format:
+# {test_name}-{metric_name}
+
+pytorch_image_classification_benchmarks-resnet152-mean_inference_batch_latency_micro_secs:
+ test_description:
+ Pytorch image classification on 50k images of size 224 x 224 with resnet
152.
+ Test link -
https://github.com/apache/beam/blob/42d0a6e3564d8b9c5d912428a6de18fb22a13ac1/.test-infra/jenkins/job_InferenceBenchmarkTests_Python.groovy#L63
+ Test dashboard -
http://104.154.241.245/d/ZpS8Uf44z/python-ml-runinference-benchmarks?orgId=1&viewPanel=2
+ test_target:
metrics_dataset: beam_run_inference
metrics_table: torch_inference_imagenet_results_resnet152
project: apache-beam-testing
+ metric_name: mean_inference_batch_latency_micro_secs
+
+pytorch_image_classification_benchmarks-resnet101-mean_load_model_latency_milli_secs:
+ test_description:
+ Pytorch image classification on 50k images of size 224 x 224 with resnet
101.
+ Test link -
https://github.com/apache/beam/blob/42d0a6e3564d8b9c5d912428a6de18fb22a13ac1/.test-infra/jenkins/job_InferenceBenchmarkTests_Python.groovy#L34
+ Test dashboard -
http://104.154.241.245/d/ZpS8Uf44z/python-ml-runinference-benchmarks?orgId=1&viewPanel=7
+ test_target:
apache_beam.testing.benchmarks.inference.pytorch_image_classification_benchmarks
+ metrics_dataset: beam_run_inference
+ metrics_table: torch_inference_imagenet_results_resnet101
+ project: apache-beam-testing
metric_name: mean_load_model_latency_milli_secs
-test_2:
- test_name: Pytorch image classification on 50k images of size 224 x 224 with
resnet 152
+pytorch_image_classification_benchmarks-resnet101-mean_inference_batch_latency_micro_secs:
+ test_description:
+ Pytorch image classification on 50k images of size 224 x 224 with resnet
101.
+ Test link -
https://github.com/apache/beam/blob/42d0a6e3564d8b9c5d912428a6de18fb22a13ac1/.test-infra/jenkins/job_InferenceBenchmarkTests_Python.groovy#L34
+ Test dashboard -
http://104.154.241.245/d/ZpS8Uf44z/python-ml-runinference-benchmarks?orgId=1&viewPanel=2
test_target:
apache_beam.testing.benchmarks.inference.pytorch_image_classification_benchmarks
metrics_dataset: beam_run_inference
- metrics_table: torch_inference_imagenet_results_resnet152
+ metrics_table: torch_inference_imagenet_results_resnet101
project: apache-beam-testing
metric_name: mean_inference_batch_latency_micro_secs
-test_3:
- test_name: Pytorch image classification on 50k images of size 224 x 224 with
resnet 101
+pytorch_image_classification_benchmarks-resnet152-GPU-mean_inference_batch_latency_micro_secs:
+ test_description:
+ Pytorch image classification on 50k images of size 224 x 224 with resnet
152 with Tesla T4 GPU.
+ Test link -
https://github.com/apache/beam/blob/42d0a6e3564d8b9c5d912428a6de18fb22a13ac1/.test-infra/jenkins/job_InferenceBenchmarkTests_Python.groovy#L151
+ Test dashboard -
http://104.154.241.245/d/ZpS8Uf44z/python-ml-runinference-benchmarks?orgId=1&viewPanel=7
test_target:
apache_beam.testing.benchmarks.inference.pytorch_image_classification_benchmarks
metrics_dataset: beam_run_inference
metrics_table: torch_inference_imagenet_results_resnet101
project: apache-beam-testing
+ metric_name: mean_inference_batch_latency_micro_secs
+
+pytorch_image_classification_benchmarks-resnet152-GPU-mean_load_model_latency_milli_secs:
+ test_description:
+ Pytorch image classification on 50k images of size 224 x 224 with resnet
152 with Tesla T4 GPU.
+ Test link -
https://github.com/apache/beam/blob/42d0a6e3564d8b9c5d912428a6de18fb22a13ac1/.test-infra/jenkins/job_InferenceBenchmarkTests_Python.groovy#L151
+ Test dashboard -
http://104.154.241.245/d/ZpS8Uf44z/python-ml-runinference-benchmarks?orgId=1&viewPanel=7
+ test_target:
apache_beam.testing.benchmarks.inference.pytorch_image_classification_benchmarks
+ metrics_dataset: beam_run_inference
+ metrics_table: torch_inference_imagenet_results_resnet152_tesla_t4
+ project: apache-beam-testing
metric_name: mean_load_model_latency_milli_secs
-test_4:
- test_name: Pytorch image classification on 50k images of size 224 x 224 with
resnet 101
+pytorch_image_classification_benchmarks-resnet152-GPU-mean_inference_batch_latency_micro_secs:
+ test_description:
+ Pytorch image classification on 50k images of size 224 x 224 with resnet
152 with Tesla T4 GPU.
+ Test link -
https://github.com/apache/beam/blob/42d0a6e3564d8b9c5d912428a6de18fb22a13ac1/.test-infra/jenkins/job_InferenceBenchmarkTests_Python.groovy#L151).
+ Test dashboard -
http://104.154.241.245/d/ZpS8Uf44z/python-ml-runinference-benchmarks?from=now-90d&to=now&viewPanel=2
test_target:
apache_beam.testing.benchmarks.inference.pytorch_image_classification_benchmarks
metrics_dataset: beam_run_inference
- metrics_table: torch_inference_imagenet_results_resnet101
+ metrics_table: torch_inference_imagenet_results_resnet152_tesla_t4
project: apache-beam-testing
metric_name: mean_inference_batch_latency_micro_secs
-test_5:
- test_name: test_cloudml_benchmark_cirteo_no_shuffle_10GB
+test_cloudml_benchmark_cirteo_no_shuffle_10GB-runtime_sec:
+ test_description:
+ TFT Criteo test on 10 GB data with no Reshuffle.
+ Test link - [Test
link](https://github.com/apache/beam/blob/42d0a6e3564d8b9c5d912428a6de18fb22a13ac1/sdks/python/apache_beam/testing/benchmarks/cloudml/cloudml_benchmark_test.py#L82)
metrics_dataset: beam_cloudml
metrics_table: cloudml_benchmark_cirteo_no_shuffle_10GB
project: apache-beam-testing
metric_name: runtime_sec
-test_6:
- test_name: test_cloudml_benchmark_criteo_10GB
+test_cloudml_benchmark_criteo_10GB-runtime_sec:
+ test_description:
+ TFT Criteo test on 10 GB data.
+ Test link -
https://github.com/apache/beam/blob/42d0a6e3564d8b9c5d912428a6de18fb22a13ac1/sdks/python/apache_beam/testing/benchmarks/cloudml/cloudml_benchmark_test.py#LL104C7-L104C41
metrics_dataset: beam_cloudml
metrics_table: cloudml_benchmark_criteo_10GB
project: apache-beam-testing
metric_name: runtime_sec
-
-test_7:
- test_name: Pytorch image classification on 50k images of size 224 x 224 with
resnet 152 with Tesla T4 GPU
- test_target:
apache_beam.testing.benchmarks.inference.pytorch_image_classification_benchmarks
- metrics_dataset: beam_run_inference
- metrics_table: torch_inference_imagenet_results_resnet152_tesla_t4
- project: apache-beam-testing
- metric_name: mean_inference_batch_latency_micro_secs
-
-test_8:
- test_name: Pytorch image classification on 50k images of size 224 x 224 with
resnet 152 with Tesla T4 GPU
- test_target:
apache_beam.testing.benchmarks.inference.pytorch_image_classification_benchmarks
- metrics_dataset: beam_run_inference
- metrics_table: torch_inference_imagenet_results_resnet152_tesla_t4
- project: apache-beam-testing
- metric_name: mean_load_model_latency_milli_secs
