This is an automated email from the ASF dual-hosted git repository. mbalassi pushed a commit to branch release-1.15.2.4-acs in repository https://gitbox.apache.org/repos/asf/flink.git
commit 3d90bcd10e5c5a9f87ffa1a31a399075057477e6 Author: Gabor Somogyi <[email protected]> AuthorDate: Thu Oct 13 19:07:43 2022 +0200 [apple] Add binary compatibility check --- apple/README.md | 11 ++++ apple/create_release_branch.sh | 23 +++++-- apple/release_rio_bump.patch.template | 72 -------------------- apple/update_rio_for_release.py | 77 ++++++++++++++++++++++ .../flink-sql-connector-hive-3.1.3.0-apple/pom.xml | 7 ++ pom.xml | 6 +- rio.yml | 2 +- 7 files changed, 118 insertions(+), 80 deletions(-) diff --git a/apple/README.md b/apple/README.md index f4de683046c..834f63c0ec9 100644 --- a/apple/README.md +++ b/apple/README.md @@ -18,6 +18,17 @@ MAJOR_VERSION=1 MINOR_VERSION=15 PATCH_VERSION=1 ACS_VERSION=0 +# BCC = binary compatibility check +# BCC version must be the previous patch version of a release +BCC_MAJOR_VERSION=1 +BCC_MINOR_VERSION=15 +BCC_PATCH_VERSION=0 +BCC_ACS_VERSION=0 ``` The script assumes you are currently on the branch `release-$MAJOR_VERSION.$MINOR_VERSION-acs` and the current snapshot version is `$MAJOR_VERSION.$MINOR_VERSION.$PATCH_VERSION-acs-SNAPSHOT`. + +Binary compatibility is enforced between patch releases from version `1.15.2.2-acs`. +This effectively means binary compatibility will be checked between the following versions: +* $MAJOR_VERSION.$MINOR_VERSION.$PATCH_VERSION.$ACS_VERSION-acs +* $BCC_MAJOR_VERSION.$BCC_MINOR_VERSION.$BCC_PATCH_VERSION.$BCC_ACS_VERSION-acs diff --git a/apple/create_release_branch.sh b/apple/create_release_branch.sh index c7aee6dc08a..3c31a30591a 100755 --- a/apple/create_release_branch.sh +++ b/apple/create_release_branch.sh @@ -20,14 +20,20 @@ set -e MAJOR_VERSION=1 MINOR_VERSION=15 -PATCH_VERSION=1 -ACS_VERSION=0 +PATCH_VERSION=2 +ACS_VERSION=2 +# BCC = binary compatibility check +# BCC version must be the previous patch version of a release +BCC_MAJOR_VERSION=1 +BCC_MINOR_VERSION=15 +BCC_PATCH_VERSION=2 +BCC_ACS_VERSION=1 OLD="$MAJOR_VERSION.$MINOR_VERSION.$PATCH_VERSION-acs-SNAPSHOT" RELEASE_VERSION="$MAJOR_VERSION.$MINOR_VERSION.$PATCH_VERSION.$ACS_VERSION-acs" -BRANCH_NAME="release-$MAJOR_VERSION.$MINOR_VERSION-acs" +BCC_RELEASE_VERSION="$BCC_MAJOR_VERSION.$BCC_MINOR_VERSION.$BCC_PATCH_VERSION.$BCC_ACS_VERSION-acs" -HERE=` basename "$PWD"` +HERE=$(basename "$PWD") if [[ "$HERE" != "apple" ]]; then echo "Please only execute in the apple/ directory"; exit 1; @@ -41,10 +47,15 @@ find .. -name 'pom.xml' -type f -exec perl -pi -e 's#<version>'"$OLD"'</version> # change version of the quickstart property find .. -name 'pom.xml' -type f -exec perl -pi -e 's#<flink.version>'"$OLD"'</flink.version>#<flink.version>'"$RELEASE_VERSION"'</flink.version>#' {} \; -git commit -am"[apple][release] Set release version to $RELEASE_VERSION" +git commit -am"[apple][release] Set release version to $RELEASE_VERSION with binary compatibility to $BCC_RELEASE_VERSION" cd .. -sed -e "s/\${MINOR_VERSION}/${MINOR_VERSION}/" -e "s/\${PATCH_VERSION}/${PATCH_VERSION}/" -e "s/\${ACS_VERSION}/${ACS_VERSION}/" -e "s/\${RELEASE_VERSION}/${RELEASE_VERSION}/" -e "s/\${BRANCH_NAME}/${BRANCH_NAME}/" apple/release_rio_bump.patch.template | git apply + +# change binary compatibility reference version in root pom file +sed -i -e 's#<japicmp.referenceVersion>.*</japicmp.referenceVersion>#<japicmp.referenceVersion>'"$BCC_RELEASE_VERSION"'</japicmp.referenceVersion>#' pom.xml +rm -f pom.xml-e + +apple/update_rio_for_release.py "${RELEASE_VERSION}" git commit -am"[apple][release] Update rio.yaml for $RELEASE_VERSION" cd apple diff --git a/apple/release_rio_bump.patch.template b/apple/release_rio_bump.patch.template deleted file mode 100644 index fadf29657ff..00000000000 --- a/apple/release_rio_bump.patch.template +++ /dev/null @@ -1,72 +0,0 @@ -diff --git a/rio.yml b/rio.yml -index c5d8c8e6adf..e647cf3949f 100644 ---- a/rio.yml -+++ b/rio.yml -@@ -3,8 +3,8 @@ timeout: 120 - - pipelines: - # Pipeline to build Flink from Apple branch. -- - name: publish-${BRANCH_NAME} -- branchName: ${BRANCH_NAME} -+ - name: publish-release-${RELEASE_VERSION} -+ branchName: release-${RELEASE_VERSION} - machine: - baseImage: docker.apple.com/thomas-weise/flink-ci:latest - env: -@@ -23,7 +23,7 @@ pipelines: - - pushd /workspace/apple/docker; wget https://archive.apache.org/dist/hadoop/common/hadoop-3.3.1/hadoop-3.3.1.tar.gz; popd - package: - # enable for release (non -SNAPSHOT) builds -- # release: true -+ release: true - freeform: - - publish: - - repo: m2:oss-patched -@@ -31,22 +31,20 @@ pipelines: - - dockerfilePath: apple/docker/Dockerfile - target: base - context: apple/docker/ - perApplication: false -- version: "${PATCH_VERSION}-r${RIO_BUILD_NUMBER}" -- extraTags: [ "latest", "${PATCH_VERSION}-latest" ] -+ version: "${PATCH_VERSION}.${ACS_VERSION}" - publish: - - repo: docker.apple.com/acs-flink/flink-1.${MINOR_VERSION}-acs - - dockerfilePath: apple/docker/Dockerfile - target: hadoop - context: apple/docker/ - perApplication: false -- version: "${PATCH_VERSION}-r${RIO_BUILD_NUMBER}" -- extraTags: [ "latest", "${PATCH_VERSION}-latest" ] -+ version: "${PATCH_VERSION}.${ACS_VERSION}" - publish: - - repo: docker.apple.com/acs-flink/flink-1.${MINOR_VERSION}-acs-hadoop - -- - name: binary-${BRANCH_NAME} -- branchName: ${BRANCH_NAME} -+ - name: binary-release-${RELEASE_VERSION} -+ branchName: release-${RELEASE_VERSION} - machine: - baseImage: docker.apple.com/thomas-weise/flink-ci:latest - build: -@@ -64,19 +62,7 @@ pipelines: - - ci stage-lib "flink-$TIMED_VERSION.tgz,snapshot/org/apache/flink/flink-dist/$VERSION/" - package: - # enable for release (non -SNAPSHOT) builds -- # release: true -+ release: true - freeform: - - publish: - - repo: oss-patched-binaries-local -- -- - name: pull-request-${BRANCH_NAME} -- branchName: ${BRANCH_NAME} -- timeout: 600 -- machine: -- baseImage: docker.apple.com/thomas-weise/flink-ci:latest -- build: -- template: freestyle:v4:prb -- steps: -- - cd /workspace -- - rm -rf .staged-artifacts -- - mvn clean package -Phive-3.1.3.0-apple,s3-plugin -Dscala-2.12 -Dmaven.test.failure.ignore=true diff --git a/apple/update_rio_for_release.py b/apple/update_rio_for_release.py new file mode 100755 index 00000000000..66669f376cd --- /dev/null +++ b/apple/update_rio_for_release.py @@ -0,0 +1,77 @@ +#!/usr/bin/env python3 +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ +import sys +import ruamel.yaml + +if len(sys.argv) != 2: + print(f'usage: {sys.argv[0]} release_version') + sys.exit(1) + +release_version = sys.argv[1] + +yaml_file_name = 'rio.yml' +yaml = ruamel.yaml.YAML() + +with open(yaml_file_name) as istream: + print(f'Loading yaml file: {yaml_file_name}\n') + ymldoc = yaml.load(istream) + + pipelines = ymldoc['pipelines'] + + # Publish pipeline + print('Searching for publish pipeline') + publish_pipeline = next(filter(lambda p: p['name'].startswith('publish'), pipelines), None) + if publish_pipeline is None: + print('Publish pipeline not found') + sys.exit(1) + publish_pipeline['name'] = f'publish-release-{release_version}' + publish_pipeline['branchName'] = f'release-{release_version}' + publish_pipeline['package']['release'] = True + for d in publish_pipeline['package']['dockerfile']: + d['version'] = release_version + del d['extraTags'] + print('Publish pipeline updated\n') + + # Binary pipeline + print('Searching for binary pipeline') + binary_pipeline = next(filter(lambda p: p['name'].startswith('binary'), pipelines), None) + if binary_pipeline is None: + print("Binary pipeline not found") + sys.exit(1) + binary_pipeline['name'] = f'binary-release-{release_version}' + binary_pipeline['branchName'] = f'release-{release_version}' + binary_pipeline['package']['release'] = True + print('Binary pipeline updated\n') + + # Pull request pipeline + print("Searching pull request pipeline") + pull_request_removed = False + for k, v in enumerate(pipelines): + if v['name'].startswith('pull-request'): + pipelines.pop(k) + pull_request_removed = True + if pull_request_removed is False: + print('Pull request pipeline not found') + sys.exit(1) + print('Pull request pipeline removed\n') + +with open(yaml_file_name, 'w') as ostream: + print(f'Saving yaml file: {yaml_file_name}') + yaml.width = 4096 + yaml.dump(ymldoc, ostream) diff --git a/flink-connectors/flink-sql-connector-hive-3.1.3.0-apple/pom.xml b/flink-connectors/flink-sql-connector-hive-3.1.3.0-apple/pom.xml index 41697239abc..eb474217690 100644 --- a/flink-connectors/flink-sql-connector-hive-3.1.3.0-apple/pom.xml +++ b/flink-connectors/flink-sql-connector-hive-3.1.3.0-apple/pom.xml @@ -102,6 +102,13 @@ under the License. </execution> </executions> </plugin> + <plugin> + <groupId>io.github.zentol.japicmp</groupId> + <artifactId>japicmp-maven-plugin</artifactId> + <configuration> + <skip>true</skip> + </configuration> + </plugin> </plugins> </build> </project> diff --git a/pom.xml b/pom.xml index fa0511340ef..d5ebb8c86bc 100644 --- a/pom.xml +++ b/pom.xml @@ -185,7 +185,7 @@ under the License. For Hadoop 2.7, the minor Hadoop version supported for flink-shaded-hadoop-2-uber is 2.7.5 --> <hivemetastore.hadoop.version>2.7.5</hivemetastore.hadoop.version> - <japicmp.referenceVersion>1.15.2</japicmp.referenceVersion> + <japicmp.referenceVersion>1.15.2.1-acs</japicmp.referenceVersion> <japicmp.outputDir>tools/japicmp-output</japicmp.outputDir> <spotless.version>2.13.0</spotless.version> <spotless.scalafmt.version>3.4.3</spotless.scalafmt.version> @@ -1605,6 +1605,10 @@ under the License. <!-- AWS SDK config that does not support license headers --> <exclude>**/awssdk/global/handlers/execution.interceptors</exclude> + + <!-- Apple Internal --> + <exclude>apple/**</exclude> + <exclude>rio.yml</exclude> </excludes> </configuration> </plugin> diff --git a/rio.yml b/rio.yml index bf982e6113d..4e8326d7154 100644 --- a/rio.yml +++ b/rio.yml @@ -14,7 +14,7 @@ pipelines: steps: - cd /workspace - rm -rf .staged-artifacts - - mvn clean deploy -Dscala-2.12 -Phive-3.1.3.0-apple,s3-plugin,docs-and-source -DskipTests -Dfast -DaltDeploymentRepository=local-staging::default::file://`pwd`/.staged-artifacts + - mvn clean deploy -Dscala-2.12 -Phive-3.1.3.0-apple,s3-plugin,docs-and-source -DskipTests -DaltDeploymentRepository=local-staging::default::file://`pwd`/.staged-artifacts # see https://docs.aci.apple.com/rio/guide-to-rio/package-and-publish/freeform.html # must exclude maven-metadata.xml - ci stage-lib --exclude maven-metadata.xml --exclude *.md5 --exclude *.sha1 --many-many-artifacts ./.staged-artifacts/**
