[
https://issues.apache.org/jira/browse/BEAM-4302?focusedWorklogId=116103&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-116103
]
ASF GitHub Bot logged work on BEAM-4302:
----------------------------------------
Author: ASF GitHub Bot
Created on: 26/Jun/18 18:48
Start Date: 26/Jun/18 18:48
Worklog Time Spent: 10m
Work Description: chamikaramj closed pull request #5406: [BEAM-4302] add
beam dependency checks
URL: https://github.com/apache/beam/pull/5406
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/.test-infra/jenkins/dependency_check/bigquery_client_utils.py
b/.test-infra/jenkins/dependency_check/bigquery_client_utils.py
new file mode 100644
index 00000000000..08571b7dce3
--- /dev/null
+++ b/.test-infra/jenkins/dependency_check/bigquery_client_utils.py
@@ -0,0 +1,148 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import datetime
+import logging
+from google.cloud import bigquery
+
+logging.getLogger().setLevel(logging.INFO)
+
+class BigQueryClientUtils:
+
+ def __init__(self, project_id, dataset_id, table_id):
+ self.project_id = project_id
+ self.dataset_id = dataset_id
+ self.table_id = table_id
+ self.bigquery_client = bigquery.Client(project_id)
+ self.table_ref = self.bigquery_client.dataset(dataset_id).table(table_id)
+ self.table = self.bigquery_client.get_table(self.table_ref)
+
+
+ def query_dep_info_by_version(self, dep, version):
+ """
+ Query for dependency information of a specific version
+ Args:
+ dep, version
+ Return:
+ release_date, is_currently_used
+ """
+ query = """SELECT release_date, is_currently_used
+ FROM `{0}.{1}.{2}`
+ WHERE package_name=\'{3}\' AND version=\'{4}\'""".format(self.project_id,
+ self.dataset_id,
+ self.table_id,
+ dep.strip(),
+ version.strip())
+
+ query_job = self.bigquery_client.query(query)
+ rows = list(query_job)
+ if len(rows) == 0:
+ logging.info("Did not find record of dependency {0} with version
{1}.".format(dep, version))
+ return None, False
+ assert len(rows) == 1
+ logging.info("""Found record of dependency {0} with version {1}:
+ release date: {2}; is_currently_used: {3}.""".format(dep, version,
rows[0]['release_date'], rows[0]['is_currently_used']))
+ return rows[0]['release_date'], rows[0]['is_currently_used']
+
+
+ def query_currently_used_dep_info_in_db(self, dep):
+ """
+ Query for the info of the currently used version of a specific dependency
+ Args:
+ dep
+ Return:
+ version, release_date
+ """
+ query = """SELECT version, release_date
+ FROM `{0}.{1}.{2}`
+ WHERE package_name=\'{3}\' AND
is_currently_used=True""".format(self.project_id,
+
self.dataset_id,
+
self.table_id,
+
dep.strip())
+
+ query_job = self.bigquery_client.query(query)
+ rows = list(query_job)
+ if len(rows) == 0:
+ return None, None
+ assert len(rows) == 1
+ return rows[0]['version'], rows[0]['release_date']
+
+
+ def insert_dep_to_table(self, dep, version, release_date,
is_currently_used=False):
+ """
+ Add a dependency with version and release date into bigquery table
+ Args:
+ dep, version, is_currently_used (default False)
+ """
+ query = """INSERT
+ `{0}.{1}.{2}` (package_name, version, release_date, is_currently_used)
+ VALUES (\'{3}\', \'{4}\', \'{5}\', {6})""".format(self.project_id,
+ self.dataset_id,
+ self.table_id,
+ dep.strip(),
+ version.strip(),
+ release_date,
+ is_currently_used)
+ logging.info("Inserting dep to table: \n {0}".format(query))
+ try:
+ query_job = self.bigquery_client.query(query)
+ if not query_job.done():
+ print query_job.result()
+ except:
+ raise
+
+
+ def delete_dep_from_table(self, dep, version):
+ """
+ Remove a dependency record from the table.
+ Args:
+ dep, version
+ """
+ query = """DELETE
+ FROM `{0}.{1}.{2}`
+ WHERE package_name=\'{3}\' AND version=\'{4}\'""".format(self.project_id,
+ self.dataset_id,
+ self.table_id,
+ dep.strip(),
+ version.strip())
+ logging.info("Deleting dep from table: \n {0}".format(query))
+ try:
+ query_job = self.bigquery_client.query(query)
+ if not query_job.done():
+ print query_job.result()
+ except:
+ raise
+
+
+ def clean_stale_records_from_table(self):
+ """
+ Remove stale records from the table. A record is stale if it is not
currently used and the release date is behind 3
+ years.
+ """
+ query = """DELETE
+ FROM `{0}.{1}.{2}`
+ WHERE release_date < '{3}'""".format(self.project_id,
+ self.dataset_id,
+ self.table_id,
+
datetime.datetime.today().date() - datetime.timedelta(3*365))
+ logging.info("Clean Up Starts")
+ try:
+ query_job = self.bigquery_client.query(query)
+ if not query_job.done():
+ logging.error(query_job.result())
+ except:
+ raise
diff --git
a/.test-infra/jenkins/dependency_check/dependency_check_report_generator.py
b/.test-infra/jenkins/dependency_check/dependency_check_report_generator.py
new file mode 100644
index 00000000000..1703375e695
--- /dev/null
+++ b/.test-infra/jenkins/dependency_check/dependency_check_report_generator.py
@@ -0,0 +1,289 @@
+#!/usr/bin/env python
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import sys
+import os.path
+import re
+import traceback
+import logging
+from datetime import datetime
+from bigquery_client_utils import BigQueryClientUtils
+
+
+_MAX_STALE_DAYS = 360
+_MAX_MINOR_VERSION_DIFF = 3
+_PYPI_URL = "https://pypi.org/project/";
+_MAVEN_CENTRAL_URL = "http://search.maven.org/#search|gav|1|"
+
+logging.getLogger().setLevel(logging.INFO)
+
+class InvalidFormatError(Exception):
+ def __init__(self, message):
+ super(InvalidFormatError, self).__init__(message)
+
+
+def extract_results(file_path):
+ """
+ Extract the Java/Python dependency reports and return a collection of
out-of-date dependencies.
+ Args:
+ file_path: the path of the raw reports
+ Return:
+ outdated_deps: a collection of dependencies that has updates
+ """
+ outdated_deps = []
+ try:
+ with open(file_path) as raw_report:
+ see_oudated_deps = False
+ for line in raw_report:
+ if see_oudated_deps:
+ outdated_deps.append(line)
+ if line.startswith('The following dependencies have later '):
+ see_oudated_deps = True
+ raw_report.close()
+ return outdated_deps
+ except Exception, e:
+ raise
+
+
+def extract_single_dep(dep):
+ """
+ Extract a single dependency check record from Java and Python reports.
+ Args:
+ dep: e.g "- org.assertj:assertj-core [2.5.0 -> 3.10.0]".
+ Return:
+ dependency name, current version, latest version.
+ """
+ pattern = " - ([\s\S]*)\[([\s\S]*) -> ([\s\S]*)\]"
+ match = re.match(pattern, dep)
+ if match is None:
+ raise InvalidFormatError("Failed to extract the dependency information:
{}".format(dep))
+ return match.group(1).strip(), match.group(2).strip(), match.group(3).strip()
+
+
+def prioritize_dependencies(deps, sdk_type, project_id, dataset_id, table_id):
+ """
+ Extracts and analyze dependency versions and release dates.
+ Returns a collection of dependencies which is "high priority" in html format:
+ 1. dependency has major release. e.g org.assertj:assertj-core [2.5.0 ->
3.10.0]
+ 2. dependency is 3 sub-versions behind the newest one. e.g org.tukaani:xz
[1.5 -> 1.8]
+ 3. dependency has not been updated for more than 6 months.
+
+ Args:
+ deps: A collection of outdated dependencies.
+ Return:
+ high_priority_deps: A collection of dependencies which need to be taken
care of before next release.
+ """
+ high_priority_deps = []
+ bigquery_client = BigQueryClientUtils(project_id, dataset_id, table_id)
+
+ for dep in deps:
+ try:
+ logging.info("Start processing: %s", dep)
+ dep_name, curr_ver, latest_ver = extract_single_dep(dep)
+ curr_release_date, latest_release_date =
query_dependency_release_dates(bigquery_client,
+
dep_name,
+
curr_ver,
+
latest_ver)
+ if sdk_type == 'Java':
+ # extract the groupid and artifactid
+ group_id, artifact_id = dep_name.split(":")
+ dep_details_url = "{0}g:\"{1}\" AND
a:\"{2}\"".format(_MAVEN_CENTRAL_URL, group_id, artifact_id)
+ else:
+ dep_details_url = _PYPI_URL + dep_name
+
+ dep_info = """<tr>
+ <td><a href=\'{0}\'>{1}</a></td>
+ <td>{2}</td>
+ <td>{3}</td>
+ <td>{4}</td>
+ <td>{5}</td>
+ </tr>\n""".format(dep_details_url,
+ dep_name,
+ curr_ver,
+ latest_ver,
+ curr_release_date,
+ latest_release_date)
+ if compare_dependency_versions(curr_ver, latest_ver):
+ high_priority_deps.append(dep_info)
+ elif compare_dependency_release_dates(curr_release_date,
latest_release_date):
+ high_priority_deps.append(dep_info)
+ except:
+ traceback.print_exc()
+ continue
+
+ bigquery_client.clean_stale_records_from_table()
+ return high_priority_deps
+
+
+def compare_dependency_versions(curr_ver, latest_ver):
+ """
+ Compare the current using version and the latest version.
+ Return true if a major version change was found, or 3 minor versions that
the current version is behind.
+ Args:
+ curr_ver
+ latest_ver
+ Return:
+ boolean
+ """
+ if curr_ver is None or latest_ver is None:
+ return True
+ else:
+ curr_ver_splitted = curr_ver.split('.')
+ latest_ver_splitted = latest_ver.split('.')
+ curr_major_ver = curr_ver_splitted[0]
+ latest_major_ver = latest_ver_splitted[0]
+ # compare major versions
+ if curr_major_ver != latest_major_ver:
+ return True
+ # compare minor versions
+ else:
+ curr_minor_ver = curr_ver_splitted[1] if len(curr_ver_splitted) > 1 else
None
+ latest_minor_ver = latest_ver_splitted[1] if len(latest_ver_splitted) >
1 else None
+ if curr_minor_ver is not None and latest_minor_ver is not None:
+ if (not curr_minor_ver.isdigit() or not latest_minor_ver.isdigit())
and curr_minor_ver != latest_minor_ver:
+ return True
+ elif int(curr_minor_ver) + _MAX_MINOR_VERSION_DIFF <=
int(latest_minor_ver):
+ return True
+ # TODO: Comparing patch versions if needed.
+ return False
+
+
+def query_dependency_release_dates(bigquery_client, dep_name,
curr_ver_in_beam, latest_ver):
+ """
+ Query release dates of current version and the latest version from BQ tables.
+ Args:
+ bigquery_client: a bq client object that bundle configurations for API
requests
+ dep_name: dependency name
+ curr_ver_in_beam: the current version used in beam
+ latest_ver: the later version
+ Return:
+ A tuple that contains `curr_release_date` and `latest_release_date`.
+ """
+ try:
+ curr_release_date, is_currently_used_bool =
bigquery_client.query_dep_info_by_version(dep_name, curr_ver_in_beam)
+ latest_release_date, _ =
bigquery_client.query_dep_info_by_version(dep_name, latest_ver)
+ date_today = datetime.today().date()
+
+ # sync to the bigquery table on the dependency status of the currently
used version.
+ if not is_currently_used_bool:
+ currently_used_version_in_db, currently_used_release_date_in_db =
bigquery_client.query_currently_used_dep_info_in_db(dep_name)
+ if currently_used_version_in_db is not None:
+ bigquery_client.delete_dep_from_table(dep_name,
currently_used_version_in_db)
+ bigquery_client.insert_dep_to_table(dep_name,
currently_used_version_in_db, currently_used_release_date_in_db,
is_currently_used=False)
+ if curr_release_date is None:
+ bigquery_client.insert_dep_to_table(dep_name, curr_ver_in_beam,
date_today, is_currently_used=True)
+ else:
+ bigquery_client.delete_dep_from_table(dep_name, curr_ver_in_beam)
+ bigquery_client.insert_dep_to_table(dep_name, curr_ver_in_beam,
curr_release_date, is_currently_used=True)
+ # sync to the bigquery table on the dependency status of the latest
version.
+ if latest_release_date is None:
+ bigquery_client.insert_dep_to_table(dep_name, latest_ver, date_today,
is_currently_used=False)
+ latest_release_date = date_today
+ except Exception:
+ raise
+ return curr_release_date, latest_release_date
+
+
+def compare_dependency_release_dates(curr_release_date, latest_release_date):
+ """
+ Compare release dates of current using version and the latest version.
+ Return true if the current version is behind over 60 days.
+ Args:
+ curr_release_date
+ latest_release_date
+ Return:
+ boolean
+ """
+ if curr_release_date is None or latest_release_date is None:
+ return True
+ else:
+ if (latest_release_date - curr_release_date).days >= _MAX_STALE_DAYS:
+ return True
+ return False
+
+
+def generate_report(file_path, sdk_type, project_id, dataset_id, table_id):
+ """
+ Write SDK dependency check results into a html report.
+ Args:
+ file_path: the path that report will be write into.
+ sdk_type: String [Java, Python, TODO: Go]
+ project_id: the gcloud project ID that is used for BigQuery API requests.
+ dataset_id: the BigQuery dataset ID.
+ table_id: the BigQuery table ID.
+ """
+ report_name = 'build/dependencyUpdates/beam-dependency-check-report.html'
+
+ if os.path.exists(report_name):
+ append_write = 'a'
+ else:
+ append_write = 'w'
+
+ try:
+ # Extract dependency check results from build/dependencyUpdate
+ report = open(report_name, append_write)
+ if os.path.isfile(file_path):
+ outdated_deps = extract_results(file_path)
+ else:
+ report.write("Did not find the raw report of dependency check:
{}".format(file_path))
+ report.close()
+ return
+
+ # Prioritize dependencies by comparing versions and release dates.
+ high_priority_deps = prioritize_dependencies(outdated_deps, sdk_type,
project_id, dataset_id, table_id)
+
+ # Write results to a report
+ subtitle = "<h2>High Priority Dependency Updates Of Beam {}
SDK:</h2>\n".format(sdk_type)
+ table_fields = """<tr>
+ <td><b>{0}</b></td>
+ <td><b>{1}</b></td>
+ <td><b>{2}</b></td>
+ <td><b>{3}</b></td>
+ <td><b>{4}</b></td>
+ </tr>""".format("Dependency Name",
+ "Current Version",
+ "Latest Version",
+ "Release Date Of the Current Used Version",
+ "Release Date Of The Latest Release")
+ report.write(subtitle)
+ report.write("<table>\n")
+ report.write(table_fields)
+ for dep in high_priority_deps:
+ report.write("%s" % dep)
+ report.write("</table>\n")
+ except Exception, e:
+ report.write('<p> {0} </p>'.format(str(e)))
+
+ report.close()
+ logging.info("Dependency check on {0} SDK complete. The report is
created.".format(sdk_type))
+
+def main(args):
+ """
+ Main method.
+ Args:
+ args[0]: path of the raw report generated by Java/Python dependency check.
Typically in build/dependencyUpdates
+ args[1]: type of the check [Java, Python]
+ args[2]: google cloud project id
+ args[3]: BQ dataset id
+ args[4]: BQ table id
+ """
+ generate_report(args[0], args[1], args[2], args[3], args[4])
+
+
+if __name__ == '__main__':
+ main(sys.argv[1:])
diff --git
a/.test-infra/jenkins/dependency_check/dependency_check_report_generator_test.py
b/.test-infra/jenkins/dependency_check/dependency_check_report_generator_test.py
new file mode 100644
index 00000000000..3a48f2a7230
--- /dev/null
+++
b/.test-infra/jenkins/dependency_check/dependency_check_report_generator_test.py
@@ -0,0 +1,132 @@
+#!/usr/bin/env python
+#
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# This script performs testing of scenarios from
verify_performance_test_results.py
+#
+
+import unittest, mock
+from mock import patch
+from datetime import datetime
+from dependency_check_report_generator import prioritize_dependencies
+
+
+_PROJECT_ID = 'mock-apache-beam-testing'
+_DATASET_ID = 'mock-beam_dependency_states'
+_TABLE_ID = 'mock-java_dependency_states'
+_SDK_TYPE = 'JAVA'
+
+# initialize current/latest version release dates for low-priority (LP) and
high-priority (HP) dependencies
+_LP_CURR_VERSION_DATE = datetime.strptime('2000-01-01', '%Y-%m-%d')
+_LATEST_VERSION_DATE = datetime.strptime('2000-01-02', '%Y-%m-%d')
+_HP_CURR_VERSION_DATE = datetime.strptime('1999-01-01', '%Y-%m-%d')
+
+class DependencyCheckReportGeneratorTest(unittest.TestCase):
+ """Tests for `dependency_check_report_generator.py`."""
+
+ def setUp(self):
+ print "Test name:", self._testMethodName
+
+
+ @patch('google.cloud.bigquery.Client')
+ @patch('bigquery_client_utils.BigQueryClientUtils')
+ def test_empty_dep_input(self, *args):
+ """
+ Test on empty outdated dependencies.
+ Except: empty report
+ """
+ report = prioritize_dependencies([], _SDK_TYPE, _PROJECT_ID, _DATASET_ID,
_TABLE_ID)
+ self.assertEqual(len(report), 0)
+
+
+ @patch('google.cloud.bigquery.Client')
+ @patch('bigquery_client_utils.BigQueryClientUtils.query_dep_info_by_version',
+ side_effect = [(_LP_CURR_VERSION_DATE, True), (_LATEST_VERSION_DATE,
False),
+ (_LP_CURR_VERSION_DATE, True), (_LATEST_VERSION_DATE,
False),
+ (_HP_CURR_VERSION_DATE, True), (_LATEST_VERSION_DATE,
False),
+ (_LP_CURR_VERSION_DATE, True), (_LATEST_VERSION_DATE,
False),])
+ def test_normal_dep_input(self, *args):
+ """
+ Test on a normal outdated dependencies set.
+ Except: group1:artifact1, group2:artifact2, and group3:artifact3
+ """
+ deps = [
+ " - group1:artifact1 [1.0.0 -> 3.0.0]",
+ " - group2:artifact2 [1.0.0 -> 1.3.0]",
+ " - group3:artifact3 [1.0.0 -> 1.1.0]",
+ " - group4:artifact4 [1.0.0 -> 1.1.0]"
+ ]
+ report = prioritize_dependencies(deps, _SDK_TYPE, _PROJECT_ID,
_DATASET_ID, _TABLE_ID)
+ self.assertEqual(len(report), 3)
+ self.assertIn('group1:artifact1', report[0])
+ self.assertIn('group2:artifact2', report[1])
+ self.assertIn('group3:artifact3', report[2])
+
+
+ @patch('google.cloud.bigquery.Client')
+ @patch('bigquery_client_utils.BigQueryClientUtils.query_dep_info_by_version',
+ side_effect = [(_LP_CURR_VERSION_DATE, True),
+ (_LATEST_VERSION_DATE, False),])
+ def test_dep_with_nondigit_major_versions(self, *args):
+ """
+ Test on a outdated dependency with non-digit major number.
+ Except: group1:artifact1
+ """
+ deps = [" - group1:artifact1 [Release1-123 -> Release2-456]"]
+ report = prioritize_dependencies(deps, _SDK_TYPE, _PROJECT_ID,
_DATASET_ID, _TABLE_ID)
+ self.assertEqual(len(report), 1)
+ self.assertIn('group1:artifact1', report[0])
+
+
+ @patch('google.cloud.bigquery.Client')
+ @patch('bigquery_client_utils.BigQueryClientUtils.query_dep_info_by_version',
+ side_effect = [(_LP_CURR_VERSION_DATE, True),
+ (_LATEST_VERSION_DATE, False),])
+ def test_dep_with_nondigit_minor_versions(self, *args):
+ """
+ Test on a outdated dependency with non-digit minor number.
+ Except: group1:artifact1
+ """
+ deps = [" - group1:artifact1 [0.rc1.0 -> 0.rc2.0]"]
+ report = prioritize_dependencies(deps, _SDK_TYPE, _PROJECT_ID,
_DATASET_ID, _TABLE_ID)
+ self.assertEqual(len(report), 1)
+ self.assertIn('group1:artifact1', report[0])
+
+
+ @patch('google.cloud.bigquery.Client')
+ @patch('bigquery_client_utils.BigQueryClientUtils.insert_dep_to_table')
+ @patch('bigquery_client_utils.BigQueryClientUtils.delete_dep_from_table')
+
@patch('bigquery_client_utils.BigQueryClientUtils.query_currently_used_dep_info_in_db',
side_effect = [(None, None)])
+ @patch('bigquery_client_utils.BigQueryClientUtils.query_dep_info_by_version',
+ side_effect = [(_HP_CURR_VERSION_DATE, True), (_LATEST_VERSION_DATE,
False),])
+ def test_invalid_dep_input(self, *args):
+ """
+ Test on a invalid outdated dependencies format.
+ Except: Exception through out. And group2:artifact2 is picked.
+ """
+ deps = [
+ "- group1:artifact1 (1.0.0, 2.0.0)",
+ " - group2:artifact2 [1.0.0 -> 2.0.0]"
+ ]
+ report = prioritize_dependencies(deps, _SDK_TYPE, _PROJECT_ID,
_DATASET_ID, _TABLE_ID)
+ self.assertEqual(len(report), 1)
+ self.assertIn('group2:artifact2', report[0])
+
+
+if __name__ == '__main__':
+ unittest.main()
+
\ No newline at end of file
diff --git a/.test-infra/jenkins/dependency_check/generate_report.sh
b/.test-infra/jenkins/dependency_check/generate_report.sh
new file mode 100755
index 00000000000..7e7ff2f4dce
--- /dev/null
+++ b/.test-infra/jenkins/dependency_check/generate_report.sh
@@ -0,0 +1,72 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# This script will be run by Jenkins as a Python dependency test.
+
+set -e
+set -v
+
+PROJECT_ID='apache-beam-testing'
+DATASET_ID='beam_dependency_states'
+PYTHON_DEP_TABLE_ID='python_dependency_states'
+JAVA_DEP_TABLE_ID='java_dependency_states'
+REPORT_DESCRIPTION="
+<h4> A dependency update is high priority if it satisfies one of following
criteria: </h4>
+<ul>
+<li> It has major versions update available, e.g. org.assertj:assertj-core
2.5.0 -> 3.10.0; </li>
+</ul>
+<ul>
+<li> It is over 3 minor versions behind the latest version, e.g.
org.tukaani:xz 1.5 -> 1.8; </li>
+</ul>
+<ul>
+<li> The current version is behind the later version for over 180 days, e.g.
com.google.auto.service:auto-service 2014-10-24 -> 2017-12-11. </li>
+</ul>
+<h4> In Beam, we make a best-effort attempt at keeping all dependencies
up-to-date.
+ In the future, issues will be filed and tracked for these automatically,
+ but in the meantime you can search for existing issues or open a new one.
+</h4>
+<h4> For more information: <a
href=\"https://docs.google.com/document/d/15m1MziZ5TNd9rh_XN0YYBJfYkt0Oj-Ou9g0KFDPL2aA/edit#\";>
Beam Dependency Update Policy </a></h4>"
+
+
+# Virtualenv for the rest of the script to run setup
+/usr/bin/virtualenv dependency/check
+. dependency/check/bin/activate
+pip install --upgrade google-cloud-bigquery
+
+# Run the unit tests of the report generator
+pip install mock
+python
$WORKSPACE/src/.test-infra/jenkins/dependency_check/dependency_check_report_generator_test.py
\
+
+rm -f build/dependencyUpdates/beam-dependency-check-report.txt
+
+echo "<html><body>" >
$WORKSPACE/src/build/dependencyUpdates/beam-dependency-check-report.html
+
+python
$WORKSPACE/src/.test-infra/jenkins/dependency_check/dependency_check_report_generator.py
\
+build/dependencyUpdates/python_dependency_report.txt \
+Python \
+$PROJECT_ID \
+$DATASET_ID \
+$PYTHON_DEP_TABLE_ID
+
+python
$WORKSPACE/src/.test-infra/jenkins/dependency_check/dependency_check_report_generator.py
\
+build/dependencyUpdates/report.txt \
+Java \
+$PROJECT_ID \
+$DATASET_ID \
+$JAVA_DEP_TABLE_ID
+
+echo "$REPORT_DESCRIPTION </body></html>" >>
$WORKSPACE/src/build/dependencyUpdates/beam-dependency-check-report.html
diff --git a/.test-infra/jenkins/job_Dependency_Check.groovy
b/.test-infra/jenkins/job_Dependency_Check.groovy
new file mode 100644
index 00000000000..bdd0bdd24d3
--- /dev/null
+++ b/.test-infra/jenkins/job_Dependency_Check.groovy
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import common_job_properties
+
+job('beam_Dependency_Check') {
+ description('Runs Beam dependency check.')
+
+ // Set common parameters.
+ common_job_properties.setTopLevelMainJobProperties(delegate)
+
+ // Allows triggering this build against pull requests.
+ common_job_properties.enablePhraseTriggeringFromPullRequest(
+ delegate,
+ 'Beam Dependency Check',
+ 'Run Dependency Check')
+
+ // This is a job that runs weekly.
+ common_job_properties.setPostCommit(
+ delegate,
+ '0 12 * * 1',
+ false)
+
+ steps {
+ gradle {
+ rootBuildScriptDir(common_job_properties.checkoutDir)
+ tasks(':runBeamDependencyCheck')
+ common_job_properties.setGradleSwitches(delegate)
+ switches('-Drevision=release')
+ }
+
+ shell('cd ' + common_job_properties.checkoutDir +
+ ' && bash .test-infra/jenkins/dependency_check/generate_report.sh')
+ }
+
+ def date = new Date().format('yyyy-MM-dd')
+ publishers {
+ extendedEmail {
+ triggers {
+ always {
+ recipientList('[email protected]')
+ contentType('text/html')
+ subject("Beam Dependency Check Report (${date})")
+ content('''${FILE,
path="src/build/dependencyUpdates/beam-dependency-check-report.html"}''')
+ }
+ }
+ }
+ archiveArtifacts {
+ pattern('src/build/dependencyUpdates/beam-dependency-check-report.html')
+ onlyIfSuccessful()
+ }
+ }
+}
diff --git a/build.gradle b/build.gradle
index 5fc2ee5b731..c4713c15fab 100644
--- a/build.gradle
+++ b/build.gradle
@@ -64,6 +64,7 @@ buildscript {
classpath "ca.coglinc:javacc-gradle-plugin:2.4.0"
// Enable the JavaCC parser generator
classpath
"gradle.plugin.io.pry.gradle.offline_dependencies:gradle-offline-dependencies-plugin:0.3"
// Enable creating an offline repository
classpath "net.ltgt.gradle:gradle-errorprone-plugin:0.0.13"
// Enable errorprone Java static analysis
+ classpath "com.github.ben-manes:gradle-versions-plugin:0.17.0"
// Enable dependency checks
}
}
@@ -73,6 +74,7 @@ buildscript {
apply plugin: "base"
apply plugin: "com.gradle.build-scan"
+apply plugin: 'com.github.ben-manes.versions'
// JENKINS_HOME and BUILD_ID set automatically during Jenkins execution
def isCIBuild = ['JENKINS_HOME', 'BUILD_ID'].every System.&getenv
if (isCIBuild) {
@@ -192,6 +194,11 @@ task pythonPostCommit() {
dependsOn ":beam-sdks-python:postCommit"
}
+task runBeamDependencyCheck() {
+ dependsOn ":dependencyUpdates"
+ dependsOn ":beam-sdks-python:dependencyUpdates"
+}
+
apply plugin: 'net.researchgate.release'
release {
revertOnFail = true
diff --git a/sdks/python/build.gradle b/sdks/python/build.gradle
index ff7e6dd0491..240de5321b6 100644
--- a/sdks/python/build.gradle
+++ b/sdks/python/build.gradle
@@ -230,3 +230,12 @@ task postCommit() {
dependsOn "hdfsIntegrationTest"
dependsOn "postCommitVRTests"
}
+
+task dependencyUpdates(dependsOn: ':dependencyUpdates') {
+ doLast {
+ exec {
+ executable 'sh'
+ args '-c', "./scripts/run_dependency_check.sh"
+ }
+ }
+}
diff --git a/sdks/python/scripts/run_dependency_check.sh
b/sdks/python/scripts/run_dependency_check.sh
new file mode 100755
index 00000000000..6fc87663cf2
--- /dev/null
+++ b/sdks/python/scripts/run_dependency_check.sh
@@ -0,0 +1,40 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# This script will be run by Jenkins as a Python dependency test.
+
+set -e
+set -v
+
+# Virtualenv for the rest of the script to run setup
+/usr/bin/virtualenv sdks/python
+. sdks/python/bin/activate
+pip install -e .[gcp,test,docs]
+
+mkdir -p $WORKSPACE/src/build/dependencyUpdates
+rm -f $WORKSPACE/src/build/dependencyUpdates/python_dependency_report.txt
+
+# List all outdated dependencies and write results in pythonDependencyReport
+echo "The following dependencies have later release versions:" >
$WORKSPACE/src/build/dependencyUpdates/python_dependency_report.txt
+pip list --outdated | sed -n '1,2!p' | while IFS= read -r line
+do
+ echo $line | while IFS=' ' read dep curr_ver new_ver type
+ do
+ echo $line
+ echo " - $dep [$curr_ver -> $new_ver]" >>
$WORKSPACE/src/build/dependencyUpdates/python_dependency_report.txt
+ done
+done
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
Issue Time Tracking
-------------------
Worklog Id: (was: 116103)
Time Spent: 87h 40m (was: 87.5h)
> Fix to dependency hell
> ----------------------
>
> Key: BEAM-4302
> URL: https://issues.apache.org/jira/browse/BEAM-4302
> Project: Beam
> Issue Type: New Feature
> Components: testing
> Reporter: yifan zou
> Assignee: yifan zou
> Priority: Major
> Time Spent: 87h 40m
> Remaining Estimate: 0h
>
> # For Java, a daily Jenkins test to compare version of all Beam dependencies
> to the latest version available in Maven Central.
> # For Python, a daily Jenkins test to compare versions of all Beam
> dependencies to the latest version available in PyPI.
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
