This is an automated email from the ASF dual-hosted git repository.
onichols pushed a commit to branch develop
in repository https://gitbox.apache.org/repos/asf/geode.git
The following commit(s) were added to refs/heads/develop by this push:
new 81ae780 GEODE-9447: fix release scripts (#6824)
81ae780 is described below
commit 81ae7806cb8d336144a0a7760c7a1444702bdd41
Author: Owen Nichols <[email protected]>
AuthorDate: Thu Sep 2 23:13:57 2021 -0700
GEODE-9447: fix release scripts (#6824)
Release script fixes:
* don't fail rc pipeline on apt-get update, only on the subsequent apt-get
install
* actually commit the changes to cache_dependencies.sh
* remind about publishing the cms REST api at the end of promote_rc for a
new minor
* don't allow closer.lua in the huntgroup first since it could return an
error page as a success response and fail checksum. instead of trying the next
url
* consider that someone may alread be on top of dependency-bumping
* retry docker build a few times if at first there is a failure
* improve license checker exit status correctness for rc pipeline
License fixes:
* fix the version of Checker Qual in the license
* update ASM version in license after a json-smart bump pulled in a new
version
* ensure binary license is a superset of source license
CMS REST API generator script fixes:
* autodetect python3 in api generator script
* add 1.15
Bump-dependencies script fixes:
* remind about adding windows label in bump -l
* remind to bump gradle plugins too
* remind to run license review script
Pipeline fixes:
* align call stack timeout with task timeout for stress new
* print build JDK same as we print test JDK in test jobs
---
LICENSE | 4 ++--
ci/pipelines/shared/jinja.variables.yml | 2 +-
ci/scripts/execute_build.sh | 7 ++++--
dev-tools/dependencies/bump.sh | 3 +++
dev-tools/release/deploy_rc_pipeline.sh | 6 +++---
dev-tools/release/license_review.sh | 7 ++++--
dev-tools/release/promote_rc.sh | 25 +++++++++++++++++++---
dev-tools/release/set_versions.sh | 2 +-
geode-assembly/src/main/dist/LICENSE | 6 ++++--
.../src/test/script/update-management-wiki.sh | 25 +++++++++++++++++-----
10 files changed, 66 insertions(+), 21 deletions(-)
diff --git a/LICENSE b/LICENSE
index a525dd1..86ccd7d 100644
--- a/LICENSE
+++ b/LICENSE
@@ -280,6 +280,8 @@ Apache Geode bundles the following files under the MIT
License:
Kelvin Luck
- MooTools (http://mootools.net), Copyright (c) 2006-2015 Valerio
Proietti, <http://mad4milk.net/>
+ - OrderStatisticTree (https://github.com/coderodde/OrderStatisticTree),
+ Copyright (c) 2021 Rodion Efremov
- Sizzle.js (http://sizzlejs.com/), Copyright (c) 2011, The Dojo Foundation
- Split.js (https://github.com/nathancahill/Split.js), Copyright (c)
2015 Nathan Cahill
@@ -289,8 +291,6 @@ Apache Geode bundles the following files under the MIT
License:
- Timeago v1.3.0 (http://timeago.yarp.com), Copyright (c) 2008-2015 Ryan
McGeary
- zTree v3.5.02 (http://zTree.me/), Copyright (c) 2010 Hunter.z
- - OrderStatisticTree (https://github.com/coderodde/OrderStatisticTree),
- Copyright (c) 2021 Rodion Efremov
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
diff --git a/ci/pipelines/shared/jinja.variables.yml
b/ci/pipelines/shared/jinja.variables.yml
index 60e2f9d..7c22658 100644
--- a/ci/pipelines/shared/jinja.variables.yml
+++ b/ci/pipelines/shared/jinja.variables.yml
@@ -175,7 +175,7 @@ tests:
RAM: '8'
name: redis
- ARTIFACT_SLUG: stressnewtestfiles
- CALL_STACK_TIMEOUT: '20700'
+ CALL_STACK_TIMEOUT: '35100'
CPUS: '96'
DISK: '300GB'
DUNIT_PARALLEL_FORKS: '24'
diff --git a/ci/scripts/execute_build.sh b/ci/scripts/execute_build.sh
index e97314a..b090fa3 100755
--- a/ci/scripts/execute_build.sh
+++ b/ci/scripts/execute_build.sh
@@ -85,7 +85,8 @@ else
fi
-SET_JAVA_HOME="export
JAVA_HOME=/usr/lib/jvm/bellsoft-java${JAVA_BUILD_VERSION}-amd64"
+JAVA_BUILD_PATH="/usr/lib/jvm/bellsoft-java${JAVA_BUILD_VERSION}-amd64"
+SET_JAVA_HOME="export JAVA_HOME=${JAVA_BUILD_PATH}"
if [ -v CALL_STACK_TIMEOUT ]; then
ssh ${SSH_OPTIONS} geode@${INSTANCE_IP_ADDRESS} "${SET_JAVA_HOME} && tmux
new-session -d -s callstacks; tmux send-keys ~/capture-call-stacks.sh\
${PARALLEL_DUNIT}\ ${CALL_STACK_TIMEOUT} C-m"
@@ -103,7 +104,9 @@ GRADLE_ARGS="\
-PbuildId=${BUILD_ID} \
build install javadoc spotlessCheck rat checkPom resolveDependencies
pmdMain -x test"
-EXEC_COMMAND="mkdir -p tmp \
+EXEC_COMMAND="echo Building with: \
+ && ${JAVA_BUILD_PATH}/bin/java -version \
+ && mkdir -p /tmp \
&& cp geode/ci/scripts/attach_sha_to_branch.sh /tmp/ \
&& /tmp/attach_sha_to_branch.sh geode ${BUILD_BRANCH} \
&& cd geode \
diff --git a/dev-tools/dependencies/bump.sh b/dev-tools/dependencies/bump.sh
index 8b0f853..4205b7e 100755
--- a/dev-tools/dependencies/bump.sh
+++ b/dev-tools/dependencies/bump.sh
@@ -26,6 +26,9 @@ if [ "$2" = "-l" ] ; then
./gradlew dependencyUpdates -Drevision=release ; find . | grep
build/dependencyUpdates/report.txt | xargs cat \
| grep ' -> ' | egrep -v
'(Gradle|antlr|lucene|JUnitParams|docker-compose-rule|javax.servlet-api|springfox|derby|selenium|jgroups|jmh|\[6.0.37|commons-collections|jaxb|testcontainers|gradle-tooling-api|slf4j|archunit)'
\
| sort -u | tr -d '][' | sed -e 's/ -> / /' -e 's#.*:#'"$0 $1"' #'
+ echo "cd .. ; geode/dev-tools/release/license_review.sh -v HEAD && echo ':)'
|| echo 'ERROR(S) WERE FOUND, SEE ABOVE' ; cd $(pwd)"
+ echo "#Also: manually check for newer version of plugins listed in
build.gradle (search on https://plugins.gradle.org/plugin)"
+ echo "#Tip: prepend SKIP=true to some lines to bump a few dependencies at
once between validation checks. Push in small batches. Add Windows label to
PR before pushing final batch."
exit 0
fi
diff --git a/dev-tools/release/deploy_rc_pipeline.sh
b/dev-tools/release/deploy_rc_pipeline.sh
index 9ae1905..77c49d9 100755
--- a/dev-tools/release/deploy_rc_pipeline.sh
+++ b/dev-tools/release/deploy_rc_pipeline.sh
@@ -303,7 +303,7 @@ jobs:
echo 'deb http://ftp.de.debian.org/debian/ stable main
contrib non-free' >> /etc/apt/sources.list.d/stable.list
echo 'deb-src http://ftp.de.debian.org/debian/ stable main
contrib non-free' >> /etc/apt/sources.list.d/stable.list
echo 'deb http://security.debian.org/ stable/updates
main contrib non-free' >> /etc/apt/sources.list.d/stable.list
- apt-get update
+ apt-get update || true
DEBIAN_FRONTEND=noninteractive apt-get install
--no-install-recommends -y cmake openssl doxygen build-essential libssl-dev
zlib1g-dev
cd geode-native
mkdir build
@@ -352,7 +352,7 @@ jobs:
echo 'deb http://ftp.de.debian.org/debian/ stable main
contrib non-free' >> /etc/apt/sources.list.d/stable.list
echo 'deb-src http://ftp.de.debian.org/debian/ stable main
contrib non-free' >> /etc/apt/sources.list.d/stable.list
echo 'deb http://security.debian.org/ stable/updates
main contrib non-free' >> /etc/apt/sources.list.d/stable.list
- apt-get update
+ apt-get update || true
DEBIAN_FRONTEND=noninteractive apt-get install
--no-install-recommends -y cmake openssl doxygen build-essential libssl-dev
zlib1g-dev
curl -L -s
https://dist.apache.org/repos/dist/dev/geode/${FULL_VERSION}/apache-geode-native-${VERSION}-src.tgz
> src.tgz
tar xzf src.tgz
@@ -435,7 +435,7 @@ jobs:
mkdir -p ~/.ssh
ssh-keygen -m PEM -b 2048 -t rsa -f ~/.ssh/id_rsa -q -N ""
cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys
- apt-get update
+ apt-get update || true
apt-get install openssh-server --no-install-recommends -y
echo "StrictHostKeyChecking no" >> /etc/ssh/ssh_config
service ssh start
diff --git a/dev-tools/release/license_review.sh
b/dev-tools/release/license_review.sh
index 7751ecc..ab8dd7c 100755
--- a/dev-tools/release/license_review.sh
+++ b/dev-tools/release/license_review.sh
@@ -339,6 +339,7 @@ if [ "${licFromWs}" = "true" ] ; then
echo "Incorrect LICENSE in binary distribution"
echo "Expected:" $(wc -c ${BLICENSE})
echo "Actual:" $(wc -c ${NEW_DIR}/LICENSE)
+ result=1
fi
if ! [ "$SKIP_SRC_LICENSE" = "true" ] ; then
@@ -349,13 +350,15 @@ if [ "${licFromWs}" = "true" ] ; then
echo "Incorrect LICENSE in source distribution"
echo "Expected:" $(wc -c ${SLICENSE})
echo "Actual:" $(wc -c ${NEW_SRC_DIR}/LICENSE)
+ result=1
fi
banner "Checking references in source license"
- cat $SLICENSE | checkMissing $SLICENSE
+ cat $SLICENSE | checkMissing $SLICENSE || result=1
+
banner "Checking references in binary license"
- cat $SLICENSE $SLICENSE $BLICENSE | sort | uniq -u | checkMissing $BLICENSE
+ cat $SLICENSE $SLICENSE $BLICENSE | sort | uniq -u | checkMissing
$BLICENSE || result=1
fi
fi
diff --git a/dev-tools/release/promote_rc.sh b/dev-tools/release/promote_rc.sh
index 0617e21..caaa431 100755
--- a/dev-tools/release/promote_rc.sh
+++ b/dev-tools/release/promote_rc.sh
@@ -259,7 +259,25 @@ echo "Building Geode docker image"
echo "============================================================"
set -x
cd ${GEODE}/docker
-docker build .
+sed -e '/www.apache.org.dyn.closer/d' -i.backup Dockerfile
+if ! docker build . ; then
+ echo retrying in 1 minute...
+ sleep 60
+ if ! docker build . ; then
+ echo retrying in 4 minutes...
+ sleep 240
+ if ! docker build . ; then
+ echo retrying in 7 minutes...
+ sleep 420
+ if ! docker build . ; then
+ echo "Hmm, the geode Dockerfile doesn't seem to want to build..."
+ echo "Try instrumenting it with some echo's to track down where it's
failing..."
+ exit 7
+ fi
+ fi
+ fi
+fi
+mv Dockerfile.backup Dockerfile
docker build -t apachegeode/geode:${VERSION} .
[ -n "$LATER" ] || docker build -t apachegeode/geode:latest .
set +x
@@ -271,7 +289,7 @@ echo "Building Native docker image"
echo "============================================================"
set -x
cd ${GEODE_NATIVE}/docker
-docker build .
+docker build . || docker build . || docker build .
docker build -t apachegeode/geode-native-build:${VERSION} .
[ -n "$LATER" ] || docker build -t apachegeode/geode-native-build:latest .
set +x
@@ -504,10 +522,11 @@ echo "9. Check that ${VERSION} download info has been
published to https://geode
MAJOR="${VERSION_MM%.*}"
MINOR="${VERSION_MM#*.}"
PATCH="${VERSION##*.}"
-[ "${PATCH}" -ne 0 ] || echo "10. Ask on the dev list for a volunteer to begin
the chore of updating 3rd-party dependency versions on develop (see
dev-tools/dependencies/README.md)"
+[ "${PATCH}" -ne 0 ] || echo "10. If 3rd-party dependencies haven't been
bumped in awhile, ask on the dev list for a volunteer (details in
dev-tools/dependencies/README.md)"
[ "${PATCH}" -ne 0 ] || [ "${MINOR}" -lt 15 ] || echo "11. In accordance with
Geode's N-2 support policy, the time has come to ${0%/*}/end_of_support.sh -v
${MAJOR}.$((MINOR - 3))"
[ "${PATCH}" -ne 0 ] || [ -n "$LATER" ] || echo "12. Log in to
https://hub.docker.com/repository/docker/apachegeode/geode and update the
latest Dockerfile linktext and url to ${VERSION_MM}"
echo "If there are any support branches between ${VERSION_MM} and develop,
manually cherry-pick '${VERSION}' bumps to those branches of geode and
geode-native."
echo "Bump support pipeline to ${VERSION_MM}.$(( PATCH + 1 )) by plussing
BumpPatch in
https://concourse.apachegeode-ci.info/teams/main/pipelines/apache-support-${VERSION_MM//./-}-main?group=Semver%20Management";
echo "Run ${0%/*}/set_versions.sh -v ${VERSION_MM}.$(( PATCH + 1 )) -s"
+[ "${PATCH}" -ne 0 ] || echo "Run cd ${GEODE} &&
geode-management/src/test/script/update-management-wiki.sh"
echo 'Finally, send announce email!'
diff --git a/dev-tools/release/set_versions.sh
b/dev-tools/release/set_versions.sh
index 33d17d5..e0a8820 100755
--- a/dev-tools/release/set_versions.sh
+++ b/dev-tools/release/set_versions.sh
@@ -134,7 +134,7 @@ sed -e "s#clone -b [ds][evlopurt/0-9.]*#clone -b
support/${VERSION_MM}#" \
rm -f gradle.properties.bak geode-book/config.yml.bak
ci/docker/cache_dependencies.sh.bak
ci/images/google-geode-builder/scripts/cache_dependencies.sh.bak
set -x
-git add gradle.properties geode-book/config.yml
+git add gradle.properties geode-book/config.yml
ci/docker/cache_dependencies.sh
ci/images/google-geode-builder/scripts/cache_dependencies.sh
if [ $(git diff --staged | wc -l) -gt 0 ] ; then
git diff --staged --color | cat
git commit -m "Bumping version to ${VERSION}"
diff --git a/geode-assembly/src/main/dist/LICENSE
b/geode-assembly/src/main/dist/LICENSE
index 4335248..a3b3aed 100644
--- a/geode-assembly/src/main/dist/LICENSE
+++ b/geode-assembly/src/main/dist/LICENSE
@@ -221,7 +221,7 @@ Apache Geode bundles the following files under the BSD
3-Clause License:
Copyright (c) 2002-2007 Marc Prud'hommeaux.
- Antlr v2.7.7 (http://www.antlr.org), Copyright (c) 2012 Terrence Parr
and Sam Harwell
- - ASM v5.0.4 (https://asm.ow2.io) Copyright (c) 2000-2011 INRIA, France
+ - ASM v9.1 (https://asm.ow2.io) Copyright (c) 2000-2011 INRIA, France
Telecom
- JLine v2.12 (http://jline.sourceforge.net), Copyright (c) 2002-2006,
Marc Prud'hommeaux <[email protected]>
@@ -1062,7 +1062,7 @@ The MIT License
(http://opensource.org/licenses/mit-license.html)
Apache Geode bundles the following files under the MIT License:
- - Checker Qual v2.10.0 (https://checkerframework.org), Copyright (c)
+ - Checker Qual v3.8.0 (https://checkerframework.org), Copyright (c)
2004-present by the Checker Framework developers
- ClassGraph v4.8.115 (https://github.com/classgraph/classgraph), Copyright
(c) 2019 Luke Hutchison
@@ -1095,6 +1095,8 @@ Apache Geode bundles the following files under the MIT
License:
Proietti, <http://mad4milk.net/>
- Normalize.css v2.1.0 (https://necolas.github.io/normalize.css/),
Copyright (c) Nicolas Gallagher and Jonathan Neal
+ - OrderStatisticTree (https://github.com/coderodde/OrderStatisticTree),
+ Copyright (c) 2021 Rodion Efremov
- Sizzle.js (http://sizzlejs.com/), Copyright (c) 2011, The Dojo Foundation
- SLF4J API v1.7.30 (http://www.slf4j.org), Copyright (c) 2004-2017 QOS.ch
- Split.js (https://github.com/nathancahill/Split.js), Copyright (c)
diff --git a/geode-management/src/test/script/update-management-wiki.sh
b/geode-management/src/test/script/update-management-wiki.sh
index b1ad84b..6ae4db7 100755
--- a/geode-management/src/test/script/update-management-wiki.sh
+++ b/geode-management/src/test/script/update-management-wiki.sh
@@ -53,8 +53,22 @@ if [[ ${BRANCH} != "develop" ]] && [[ ${BRANCH} != "master"
]] && [[ ${BRANCH%/*
fi
SUDO=
-PIP=pip
-PYTHON=python
+if python --version | grep 'Python 3' ; then
+ PYTHON=python
+elif python3 --version | grep 'Python 3' ; then
+ PYTHON=python3
+else
+ echo "please install python 3"
+ exit 1
+fi
+if pip --version | grep 'python 3' ; then
+ PIP=pip
+elif pip3 --version | grep 'python 3' ; then
+ PIP=pip3
+else
+ echo "please install pip 3"
+ exit 1
+fi
echo ""
echo "============================================================"
@@ -80,9 +94,10 @@ GEODE_VERSION=$("$GEODE"/bin/gfsh version | sed -e
's/-SNAPSHOT//' -e 's/-build.
[[ "${GEODE_VERSION%.*}" == "1.12" ]] && PAGE_ID=132322415
[[ "${GEODE_VERSION%.*}" == "1.13" ]] && PAGE_ID=147426059
[[ "${GEODE_VERSION%.*}" == "1.14" ]] && PAGE_ID=153817491
+[[ "${GEODE_VERSION%.*}" == "1.15" ]] && PAGE_ID=188744355
if [[ -z "${PAGE_ID}" ]] ; then
- echo "Please create a new wiki page for $GEODE_VERSION and add its page ID
to $0 near line 83"
+ echo "Please create a new blank wiki page for $GEODE_VERSION under
https://cwiki.apache.org/confluence/display/GEODE/Cluster+Management+Service+Rest+API
and add its page ID to $0 near line 98"
exit 1
fi
@@ -159,7 +174,7 @@ cat static/index.html |
# clean up a few things premailer will otherwise choke on
grep -v doctype | sed -e 's/—/--/g' |
# convert css style block to inline css (that is the only way confluence
accepts styling)
-$SUDO $PYTHON -m premailer --method xml --encoding ascii --pretty |
+(set -o pipefail && $SUDO $PYTHON -m premailer --method xml --encoding ascii
--pretty |
# strip off the document envelope (otherwise confluence will not accept it) by
keeping only lines between the body tags
awk '
/<\/body>/ {inbody=0}
@@ -234,7 +249,7 @@ sed -e 's/«/_/g' -e 's/»//g' |
sed -e 's/class="method" style="margin-left:20p/class="method"
style="margin-left:0p/' |
# add more information at the top
awk '/More information:/{sub(/More information:/,"Swagger:
http://locator:7070/management/docs (requires access to a running
locator)<br/>Codegen: <code><small>brew install swagger-codegen;
swagger-codegen generate -i
http://locator:7070/management'${URI_VERSION}'/api-docs</small></code><br/>More
information:")}{print}' |
-cat > static/index-xhtml.html
+cat > static/index-xhtml.html)
# if file is empty due to some error, abort!
! [ -z "$(cat static/index-xhtml.html)" ]
