Abacn commented on code in PR #30262: URL: https://github.com/apache/beam/pull/30262#discussion_r1484464994
########## scripts/tools/gcpbomupgrader.py: ########## @@ -0,0 +1,212 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import errno +import logging +import os +import re +import subprocess +import sys +""" +This Python script is used for upgrading the GCP-BOM in BeamModulePlugin. +Specifically, it + +1. preprocessing BeamModulePlugin.groovy to decide the dependencies need to sync +2. generate an empty Maven project to fetch the exact target versions to change +3. Write back to BeamModulePlugin.groovy + +There are few reasons we need to declare the version numbers: +1. Sync the dependency that not included in GCP-BOM with those included with BOM + For example, "com.google.cloud:google-cloud-spanner" does while "com.google.cloud:google-cloud-spanner:():test" doesn't +2. There are Beam artifacts not depending on GCP-BOM but used dependency managed + by GCP-BOM. +""" + + +class BeamModulePluginProcessor: + # Known dependencies managed by GCP BOM and also used by Beam. + # We only need to have one dependency for each project to figure out the target version + KNOWN_DEPS = { + "arrow": "org.apache.arrow:arrow-memory-core", + "gax": "com.google.api:gax", + "google_cloud_spanner": "com.google.cloud:google-cloud-spanner", + "grpc": + "io.grpc:grpc-netty", # use "grpc-netty" to pick up proper netty version + "netty": "io.netty:netty-transport", + "protobuf": "com.google.protobuf:protobuf-java" + } + # dependencies managed by GCP-BOM that used the dependencies in KNOWN_DEPS + # So we need to add it to the example project to get the version to sync + OTHER_CONSTRANTS = [ + "com.google.cloud:google-cloud-bigquery" # uses arrow + ] + + # e.g. // Try to keep grpc_version consistent with gRPC version in google_cloud_platform_libraries_bom + ANCHOR = re.compile( + r'^\s*// Try to keep .+ consistent .+ google_cloud_platform_libraries_bom\s*$' + ) + # e.g. def grpc_version = "1.61.0" + VERSION_STRING = re.compile( + r'^\s*def (\w+)_version\s*=\s*[\'"](\S+)[\'"]') + BOM_VERSION_STRING = re.compile( + r'\s*google_cloud_platform_libraries_bom\s*:\s*[\'"]com\.google\.cloud:libraries-bom:([0-9\.]+)[\'"],?' + ) + BUILD_DIR = 'build/dependencyResolver' + GRADLE_TEMPLATE = """ +plugins { id 'java' } +repositories { mavenCentral() } +dependencies { +implementation platform('com.google.cloud:libraries-bom:%s') +%s +} +configurations.implementation.canBeResolved = true +""" + + def __init__( + self, + bom_version, + filepath='buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy', + runnable=None): + self.bom_version = bom_version + self.filepath = filepath + self.runnable = runnable or os.path.abspath('gradlew') + logging.info('-----Read BeamModulePlugin-----') + with open(filepath, 'r') as fin: + self.original_lines = fin.readlines() + # e.g. {"io.grpc:grpc-netty", "1.61.0"} + self.dep_versions = {} + self.dep_versions_current = {} + + def check_dependencies(self): + """Check dependencies in KNOWN_DEPS are found in BeamModulePlugin, and vice versa.""" + logging.info("-----check dependency defs in BeamModulePlugin-----") + found_deps = {} + for idx, line in enumerate(self.original_lines): + m = self.ANCHOR.match(line) + if m: + n = self.VERSION_STRING.search(self.original_lines[idx + 1]) + if not n: + raise RuntimeError( + "Version definition not found after anchor comment. Try standardize it." + ) + found_deps[n.group(1)] = n.group(2) + assert sorted(self.KNOWN_DEPS.keys()) == sorted(found_deps.keys()) + self.dep_versions_current = { + self.KNOWN_DEPS[k]: v for k, v in found_deps.items() + } + + def prepare_gradle(self, bom_version): + logging.info("-----prepare build.gradle for example project-----") + try: + os.makedirs(self.BUILD_DIR) + except OSError as e: + if e.errno != errno.EEXIST: + raise + + deps = [] + for dep in list(self.KNOWN_DEPS.values()) + self.OTHER_CONSTRANTS: + deps.append(f"implementation '{dep}'") + gradle_file = self.GRADLE_TEMPLATE % (bom_version, "\n".join(deps)) + with open(os.path.join(self.BUILD_DIR, 'build.gradle'), 'w') as fout: + fout.write(gradle_file) + # we need a settings.gradle + with open(os.path.join(self.BUILD_DIR, 'settings.gradle'), 'w') as fout: + fout.write('\n') + + def resolve(self): + logging.info("-----resolve dependency-----") + subp = subprocess.run([ + self.runnable, + *('-q dependencies --configuration implementation --console=plain' + .split()) + ], + cwd=self.BUILD_DIR, Review Comment: That was the result that I have run yapf on the script. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
