This is an automated email from the ASF dual-hosted git repository.
janardhan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/systemds.git
The following commit(s) were added to refs/heads/master by this push:
new 8dd1394 [SYSTEMDS-2958] Add release automation scripts (#1404)
8dd1394 is described below
commit 8dd1394e9b3b7c5363b34ddd5a6503a1fbcddb6f
Author: Janardhan Pulivarthi <[email protected]>
AuthorDate: Mon Oct 11 19:15:25 2021 +0530
[SYSTEMDS-2958] Add release automation scripts (#1404)
Scripts files:
1. do-release.sh - Triggers the release workflow
2. create-tag.sh - Release preparation with Maven Release Plugin
3. release-build.sh - Create Release artifacts
4. release-utils.sh - Helper utilities for the release workflow
Script procedure:
1. It prompts for release related info such as branch, release version, etc.
2. Create a release tag with maven-release-plugin [1], and update the
pom.xml file
with new tag. And push the changes to the official repo.
3. Next, build release artifacts with maven-deploy-plugin [2]. At this
points release
artifacts will be signed.
4. Once the artifacts are built they will be uploaded to nexus repo [3]
(acts as a
staging repo for maven central) and svn staging repo [4].
--
[1]
https://maven.apache.org/maven-release/maven-release-plugin/prepare-mojo.html
[2] https://maven.apache.org/plugins/maven-deploy-plugin/deploy-mojo.html
[3] https://repository.apache.org/
[4] https://dist.apache.org/repos/dist/dev/systemds/
---
dev/release/README.md | 45 +++++
dev/release/create-tag.sh | 138 +++++++++++++
dev/release/damslab-pubkey.asc | 78 --------
dev/release/do-release.sh | 72 +++++++
dev/release/{ => old}/deploy.sh | 0
dev/release/{ => old}/old-release-build.sh | 0
dev/release/{ => old}/publish.sh | 0
dev/release/{ => old}/simple-release-build.sh | 0
dev/release/{ => old}/svn_dev_upload.sh | 0
dev/release/release-architecture.svg | 3 +
dev/release/release-build.sh | 259 ++++++++++++++++++++++++
dev/release/release-utils.sh | 278 ++++++++++++++++++++++++++
pom.xml | 1 -
13 files changed, 795 insertions(+), 79 deletions(-)
diff --git a/dev/release/README.md b/dev/release/README.md
new file mode 100644
index 0000000..72e5307
--- /dev/null
+++ b/dev/release/README.md
@@ -0,0 +1,45 @@
+<!--
+{% comment %}
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to you under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+{% end comment %}
+-->
+
+### Usage
+
+Set gpg home as
+
+```bash
+export GNUPGHOME="$HOME/.gnupg"
+```
+
+Dry run:
+
+```sh
+./dev/release/do-release.sh -n
+```
+
+Release: (irreversible operation)
+
+```sh
+./dev/release/do-release.sh
+```
+
+### Architecture of the release pipeline
+
+The following diagram illustrates the steps building and publishing release.
+
+
+
diff --git a/dev/release/create-tag.sh b/dev/release/create-tag.sh
new file mode 100755
index 0000000..21c4d3d
--- /dev/null
+++ b/dev/release/create-tag.sh
@@ -0,0 +1,138 @@
+#!/usr/bin/env bash
+#-------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------
+
+################################################################################
+## File: create-tag.sh
+## Desc: Release preparation with Maven Release Plugin
+################################################################################
+
+# https://stackoverflow.com/q/59895
+SELF=$(cd $(dirname $0) && pwd)
+. "$SELF/release-utils.sh"
+
+exit_with_usage() {
+ local NAME=$(basename $0)
+ cat << EOF
+usage: $NAME
+
+Tags a SystemDS release on a particular branch.
+
+Inputs are specified with the following environment variables:
+
+ASF_USERNAME - Apache Username
+ASF_PASSWORD - Apache Password
+GIT_NAME - Name to use with git
+GIT_EMAIL - E-mail address to use with git
+GIT_BRANCH - Git branch from which to make release
+RELEASE_VERSION - Version used in pom files for release
+RELEASE_TAG - Name of release tag
+NEXT_VERSION - Development version after release
+EOF
+
+ exit 1
+}
+
+set -e
+set -o pipefail
+
+if [[ $@ == *"help"* ]]; then
+ exit_with_usage
+fi
+
+# docs related to stty
+# https://www.ibm.com/docs/en/aix/7.2?topic=s-stty-command
+if [[ -z "$ASF_PASSWORD" ]]; then
+ echo 'The environment variable ASF_PASSWORD is not set. Enter the password.'
+ echo
+ stty -echo && printf "ASF password: " && read ASF_PASSWORD && printf '\n' &&
stty echo
+fi
+
+for env in ASF_USERNAME ASF_PASSWORD RELEASE_VERSION RELEASE_TAG NEXT_VERSION
GIT_EMAIL GIT_NAME GIT_BRANCH; do
+ if [ -z "${!env}" ]; then
+ echo "$env must be set to run this script"
+ exit 1
+ fi
+done
+
+uriencode() { jq -nSRr --arg v "$1" '$v|@uri'; }
+
+declare -r ENCODED_ASF_PASSWORD=$(uriencode "$ASF_PASSWORD")
+
+# git configuration
+git config user.name "$GIT_NAME"
+git config user.email "$GIT_EMAIL"
+
+printf "$RELEASE_TAG \n"
+printf "$RELEASE_VERSION\n"
+printf "$NEXT_VERSION"
+
+# options available at
https://maven.apache.org/plugins/maven-gpg-plugin/sign-mojo.html
+GPG_OPTS="-Dgpg.homedir=$GNUPGHOME -Dgpg.keyname=$GPG_KEY
-Dgpg.passphrase=$GPG_PASSPHRASE"
+
+printf "\n -Dgpg.homedir=$GNUPGHOME -Dgpg.keyname=$GPG_KEY
-Dgpg.passphrase=$GPG_PASSPHRASE \n"
+
+# Tag release version before `mvn release:prepare`
+# tag python build
+# PySpark version info we use dev0 instead of SNAPSHOT to be closer
+# to PEP440.
+# sed -i".tmp" 's/__version__ = .*$/__version__ = "'"$NEXT_VERSION.dev0"'"/'
python/systemds/version.py
+
+# change tags in docs
+# docs/_config.yml
+# update SYSTEMDS_VERSION
+# sed -i 's/SYSTEMDS_VERSION:.*$/SYSTEMDS_VERSION: '"$RELEASE_VERSION"'/g'
docs/_config.yml
+# and run docs/updateAPI.sh to update version in api docs
+
+
+# NOTE:
+#
+# When using gpg-plugin in conjunction with release plugin use
+#
+# $ mvn release:perform -Darguments=-Dgpg.passphrase=thephrase
+#
+# since the system properties of the current maven session are
+# not propagated to the forked session automatically.
+#
+
+if is_dry_run; then
+ dry_run=true
+fi
+
+
+CMD="mvn --batch-mode -DdryRun=${dry_run} -Dtag=$RELEASE_TAG \
+ -Dresume=false \
+ -DreleaseVersion=$RELEASE_VERSION \
+ -DdevelopmentVersion=$NEXT_VERSION \
+ -Dgpg.keyname=${GPG_KEY} -Dgpg.passphrase=${GPG_PASSPHRASE} \
+ -Darguments=${GPG_OPTS} \
+ release:prepare"
+
+printf "\n #### Executing command: #### \n"
+printf "\n $(bold $(greencolor $CMD)) \n\n"
+
+$CMD
+
+# tag snapshot version after `mvn release:prepare`
+
+# Change docs to dev snapshot tag
+# sed -i".tmp1" 's/SYSTEMDS_VERSION:.*$/SYSTEMDS_VERSION: '"$NEXT_VERSION"'/g'
docs/_config.yml
+# and run docs/updateAPI.sh to update version in api docs
diff --git a/dev/release/damslab-pubkey.asc b/dev/release/damslab-pubkey.asc
deleted file mode 100644
index 4ab1c22..0000000
--- a/dev/release/damslab-pubkey.asc
+++ /dev/null
@@ -1,78 +0,0 @@
------BEGIN PGP PUBLIC KEY BLOCK-----
-
-mQINBF1pQWIBEACkUyhORiP/RjxYHiX1Mtkp8HTjzpS4T3E2Wg9Ey6V7rpKhPtKm
-qwJW4rPU7IK/CRVbT7nsR7OQaDQaMY9ptATloWpUNSbmH+fKq9JsQll1kJOcLgUe
-0QPZ5KJK3ZNMdlFva5k13nMjGIHd6ThTilfq1XuwD5+qQIXxUuDExflphSCR84V4
-UrUUyqJTtMyeqWYNucCm8BOcU79hKNbGXCnLQTW7cEnRy5rg5wXc0ZI7QUM8bdWY
-3AylaKP2rdlycjAPBZSy2cunKl/pUgPiRa8uuqOpnw9TIfCTjmqwXMrM3FJkM7Jg
-Ul0oeIaUa/M3bsJECflwBMP8Z0nGHg/9H1geu6iSIi5xNIVX/H5A+DkZKDS9vvAF
-DoK366N39MF6UkwHt0Sio98QCUOiOG9Lxf70DYI+rPFSf2HhKhTa3/VmJ2hpU5AV
-/Lnt1VDSmUbMRvBiGn4IL38F8dWE6wythJgmCi+obBJaE1d9GsPkYxwrdBqNEJig
-lOTUULYuYyBe2oNEmq6k99pNChlOQ0bLlXBAbzdz6fZ9pvAvjjgm+8DtkxnmDqJx
-hHavYfameHgGBqJBEbato93JtmxhxJ3iigq+qPIyV7VcLt3C0Hm+hL2Gv/mYiA1X
-uDCKwFMavM+oZHkyfVZb5Enqkq2KvFWNVJK9PyOwzStweAgSB+HJ8YoZzwARAQAB
-tC9TeXN0ZW1EUyAocmVsZWFzZSBzaWduaW5nIGtleSkgPG1hcmtAZG9rdGVyLmNj
-PokCVAQwAQoAPhYhBI7TggNO5z9SeN7qplE80Mx9Yvt3BQJeeTtnIB0AdHJhbnNm
-ZXIgZnJvbSBjcmVhdG9yIHRvIGdyb3VwAAoJEFE80Mx9Yvt3OSMQAIS11onjwH0h
-/sAfBJRg5j18xBqCflq79Cdb4amz3HXV8OqRVY01XxTNGzCS/QZwVRLPbh7LvO6o
-Z3lr3sJ3WOS79Wy1ABoV0WVPnfnvgLNz+xF1Tb4/R1NyNODoN40T+gxM+CXHkGUJ
-tSXiqQ48oexeQKgBleVof91WjCHlXgi5K6cMSWhexI/tIov5aFNEhZyk7xOA0tpu
-+hKbrXl/emWOzOiSxp3JZ9VP7KZ3KvnpdnJNqbJdajMlsNvcihHqABuRwxM8gLT0
-z1ATYLF+bSuK7McCrmPU2axNIwfpolASheTK7qK7Yo7PgWNuLcKSmzE4ua5xm/P9
-TyNoVlKGpvFHkM2XYBU41FJGAKz8ypQuQFByeFPTgg46YWCQ+5J7lN73iinZQGAe
-C5CfcKSCkglwreihNQ2sjXSQSsm/IhgnR3Gg5ZlzHiKAM96LC9quManWGMktG8y0
-vRqd5I1s08LrmAQjfqxVheEwYAiVqkLqS6JbzkmpqPiAfp/bx7NuyrCu+LY81kRt
-lI3qQ9dKpHQuQAiEfLq+I6rHkkPJIGVaIgOAh7PZyNh9n4lNVqhMICo7asbDbtj8
-MaH18B3ZKl+LuYxIlyFfBAOqfZjl8LVxkc3AielemdTif6v2uAWFkjZLqYeqWW8A
-xihN1AjHJHqNsCOueuLOkF5TIwPo1N9PiQJUBBMBCgA+FiEEjtOCA07nP1J43uqm
-UTzQzH1i+3cFAl4Aum0CGwMFCQWjmoAFCwkIBwIGFQoJCAsCBBYCAwECHgECF4AA
-CgkQUTzQzH1i+3foqQ/8D742eSJOyBdrHPTLCSaNWZs5wcTjvFgzdKkCOPW7ZT7q
-QINht/QHoiBSFL3w4X/0u5GRWabiBBcfx5Nqzpfaxm7XjTPEHnUkVCEEuwzg6zql
-gajnWoebhieAdl9cUJxnnzPytWKFWglieGg6e40V1b/sUQUghe8bQgbUAohhHbtC
-2Ro9exvZLprwzQj/48d055gmRIBo6xswGqKeiznaoa0LLSqBnoQkZFa4KUaYMUZh
-BXZ+Har9bl/yWTS3MoiaRoyGzBJieEpEtCb+g8aLMunSDXOc0+x7ijHf1nHJoLmq
-aNAYk4AcXNeyNHexmNmwBGwXnChwUyXV8XtaQ6x92dTevTGH18TrKb6F9c1pjjqM
-WSXQPhy8ioLkNhhMqNAHq1duuPKCiNbIJqJxkBAMZmaSox1Ba77hIrx/wGeEBv0I
-zW4X7HBkgkO7IkMBR/SsY+J59/PQMtQrpCSnQX9XluzFsdNeaFiKjLgO8hiEilVM
-mUHPaqsHjAp3zSW+a+JBbKMG3bwGb1j8PITfnhA4Z+QQt5TR1ctdJQVsYIibpWyA
-FxykyULv4lI3fKaXC2pX9jzK1HgL1ASZNg59jJTzqhdMTZuLhMhXNd89/cfS50hK
-YTRy6VHGRPHeJEnInrkpF300I1H24r88vLP9oWqZqK84TPF0n3BLVyL9QzJGMQO0
-QERBTVNMYWIgKFN5c3RlbURTIFJlbGVhc2UgU2lnbmluZyBLZXkpIDxkYW1zbGFi
-QG1saXN0LnR1Z3Jhei5hdD6JAlQEEwEKAD4WIQSO04IDTuc/Unje6qZRPNDMfWL7
-dwUCXnk6tgIbAwUJBaOagAULCQgHAgYVCgkICwIEFgIDAQIeAQIXgAAKCRBRPNDM
-fWL7d5GyD/4q5jbX1UwtYr/troW0MntJnT6y66CdxJ7rXB1cQHx+kP/RzUWXx7G1
-nnse/7flR2Ke9kIEaWq5ENvSSavH2ugWtcpKsvcSVtLn6FurkJ5VT3jLanp08Yvs
-I0sE/hK5hQVvR/eS7LKKWYnrtbXGO1VxMOXDVz6ga3XvYatcBUgHjz7xPb6AEO5V
-AphjDIelJRfgfznL+ufmtpspX1izJhS5ihk3qrcUjzxVj6sD2pvJElSbhsgfQKhz
-29a5CYJLlo/52X+HGPw1YiSlra9enVzqLKbTNdowHhoEawpsLWTWg93sPbeRoAff
-anrXOvUKU4s9JQawPcocQn5LGr+k2Jd4nOsYgnG3ezQmwkERdLXAaGZHS7ti4PRM
-QE6MyzlKIfEEcydh+Ckfux82ObyRrIVajjspBVV36DxZNtnK+8bTIL8s6vWYuNzz
-gxz7ovILNnCEuGftwb4Vd1mZuMzf98rOmmsweyMqmT1Tj7JhZ1dMfTNO3AIUqXZq
-Jt2b9ap04fo67xsh144E/wrM9FRZcqUw4vJkTlGeuM3bkCsmIQQb/q6PkhINJRmq
-U0/ZfSitBcvLJhUIfKe0mnwano2fCdHc8OfMctLMQps0ZVvBGSuaBhYZp+CNckVM
-C24MCSFsrwuQo+0A0fxQU62DxRC3N7WKjbQSVYrMBsj6bIgZMqtOHrkCDQRdaUFi
-ARAAm5IQ/S23tO02F2KR4iyCNZSg3XdDI/qW8EBlZpLI09S73RokvxiCUoAC+AA3
-/A7jvJCa/OYtFohQ3g3XyoOlAy9r5iMs2JgxJvK4/92bBTCb05tE1mWe74fMJWXK
-mbUys5XnKvdNnPkdpVhdaZCLfhGhyj4oYOlO/4h0lcwhpwo4kRxcX3AFkv2mLssh
-UFoWqKmp5zrC4m5pD2SYDVuaxYDLAYQqGQmMXwsStBROVs64fol5ud4w9bJthQn2
-lVir06KxinCYsFkBpB4zoT47ACT19vwzgQogA+DDqh6AydYo6+JCiybyTdPFNz/o
-gBobH6sBXqGEK9mUSSXaBKPwJhb7KPaHStShRUhDXzrmKttOEVxfuvkGT82zx8Mm
-wri4fVe1Vp8azrmZ23jstyxRRXeXxgjMAeBC0r/mi5hKuEhlqyjPoLF/kt47Jl8+
-2DAc9nwHl7K7bsmbOygdEW9yKBrVeO3EvhB38RRpWEYfVXGqMNe3d5FhmWcPVM2o
-QxtzIGr2UyNfSpfWn5OwIax0mK4lO3lsMRlqTzanVg3LC639VS3WiZ4Vsk+nTx3g
-FULKin9nAESUJsosPH60wlC120nVr2NsLxbhn96Ze1alwDLjOHGagAiKmtYrOtZG
-E5+90/Ubz265PtZ7f/X387YreDYfumDB3nIXob73HtAbdfMAEQEAAYkCPAQYAQoA
-JhYhBI7TggNO5z9SeN7qplE80Mx9Yvt3BQJdaUFiAhsMBQkFo5qAAAoJEFE80Mx9
-Yvt3D3QP/1EWHk8GWr6QLp6Gaizxlf3Jc0nUhxaGo60QyLmIbFiCVCsU/DzsMtOf
-Q+iak7Prlz4C1IeBcH6jqS2MBtairGCmGpzj+wI3QG3DZSySuDqXaby/xWLExtHI
-ZOiWcl47WqGz8kBc6+cecLbt23v/UpS81dUog3kM/TDjEMbgLmIJLZn3/C4p6mGH
-YpOh2IZdNEuOpmN1FpMKgzSuDThltqocSV5uWAfP24OTBVnOMd1+fNyajvpAEgP4
-9ZzNUY0d2VGTXfe3cVspjpHH0ED102nIO6ypnlvKpbjbQTrg7cbpXbBWpWIJsyzo
-PLmfw5bUPxIc0RzaUIOLTzBkhLGBtzeWZCNVPel/+iPyUIFOZ8yEAN9Dwh3oTz6p
-FYJjGD3k1wGhOG69mHpbB+BTqNYJWNaVZDXgEnq7BQp/OO+jFdlwXzKedMIYBpa1
-IYcr1IaopJv8h3b7V2peNhIbl9OqlBFPz58MDfrf1tT9lNv4z+nno6DXp+ndv4Gb
-ahSCEZZowEBaOR9FGQ1l2OLwpdVsdtVz8ahL8Sx4spcBaDwZMoQEgAGcmmurz578
-O/iEpynhzh7dmCqUPV3LNq7bpm73MCHIgujvTU2fF2M/d6zE00PnT3pEvignBe5X
-NgPjbMY8o+HCyerS3MXjNEfQBw4jRqV6kJdLWlgWLUo3l3qFz+GR
-=qr4P
------END PGP PUBLIC KEY BLOCK-----
diff --git a/dev/release/do-release.sh b/dev/release/do-release.sh
new file mode 100755
index 0000000..36ab60a
--- /dev/null
+++ b/dev/release/do-release.sh
@@ -0,0 +1,72 @@
+#!/usr/bin/env bash
+#-------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------
+
+################################################################################
+## File: do-release.sh
+## Desc: Triggers the release workflow
+################################################################################
+
+SELF=$(cd $(dirname $0) && pwd)
+. "$SELF/release-utils.sh"
+
+# discussion on optional arguments
+# https://stackoverflow.com/q/18414054
+while getopts ":n" opt; do
+ case $opt in
+ n) DRY_RUN=1 ;;
+ \?) error "Invalid option: $OPTARG" ;;
+ esac
+done
+
+DRY_RUN=${DRY_RUN:-0}
+
+cleanup_repo
+
+# Ask for release information
+get_release_info
+
+# tag
+run_silent "Creating release tag $RELEASE_TAG..." "tag.log" \
+ "$SELF/create-tag.sh"
+
+# run_silent "Publish Release Candidates to the Nexus Repo..."
"publish-snapshot.log" \
+# "$SELF/release-build.sh" publish-snapshot
+
+if ! is_dry_run; then
+ git checkout $RELEASE_TAG
+ printf "\n checking out $RELEASE_TAG for building artifacts \n"
+fi
+
+# NOTE:
+# The following goals publishes the artifacts to
+# 1) Nexus repo at repository.apache.org
+# 2) SVN repo at dist.apache.org
+#
+# are to be used together.
+
+run_silent "Publish Release Candidates to the Nexus Repo..." "publish.log" \
+ "$SELF/release-build.sh" publish-release
+
+if is_dry_run; then
+ # restore the pom.xml file updated during release step
+ git restore pom.xml
+fi
diff --git a/dev/release/deploy.sh b/dev/release/old/deploy.sh
similarity index 100%
rename from dev/release/deploy.sh
rename to dev/release/old/deploy.sh
diff --git a/dev/release/old-release-build.sh
b/dev/release/old/old-release-build.sh
similarity index 100%
rename from dev/release/old-release-build.sh
rename to dev/release/old/old-release-build.sh
diff --git a/dev/release/publish.sh b/dev/release/old/publish.sh
similarity index 100%
rename from dev/release/publish.sh
rename to dev/release/old/publish.sh
diff --git a/dev/release/simple-release-build.sh
b/dev/release/old/simple-release-build.sh
similarity index 100%
rename from dev/release/simple-release-build.sh
rename to dev/release/old/simple-release-build.sh
diff --git a/dev/release/svn_dev_upload.sh b/dev/release/old/svn_dev_upload.sh
similarity index 100%
rename from dev/release/svn_dev_upload.sh
rename to dev/release/old/svn_dev_upload.sh
diff --git a/dev/release/release-architecture.svg
b/dev/release/release-architecture.svg
new file mode 100644
index 0000000..7488021
--- /dev/null
+++ b/dev/release/release-architecture.svg
@@ -0,0 +1,3 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN"
"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd";>
+<svg xmlns="http://www.w3.org/2000/svg";
xmlns:xlink="http://www.w3.org/1999/xlink"; version="1.1" width="723px"
height="474px" viewBox="-0.5 -0.5 723 474" content="<mxfile
host="app.diagrams.net" modified="2021-05-29T13:40:32.898Z"
agent="5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like
Gecko) Chrome/91.0.4472.77 Safari/537.36"
etag="g1yntjUMJD-_K7eiSg71" version="14.6.13"
type="device"><diagram id=" [...]
\ No newline at end of file
diff --git a/dev/release/release-build.sh b/dev/release/release-build.sh
new file mode 100755
index 0000000..890c9bf
--- /dev/null
+++ b/dev/release/release-build.sh
@@ -0,0 +1,259 @@
+#!/usr/bin/env bash
+#-------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------
+
+################################################################################
+## File: release-build.sh
+## Desc: Create Release artifacts
+################################################################################
+
+SELF=$(cd $(dirname $0) && pwd)
+. "$SELF/release-utils.sh"
+
+exit_with_usage() {
+
+ cat << EOF
+usage: release-build.sh <package|docs>
+
+Create build deliverables from a commit
+Top level targets are
+ - package: create binary packages and commit them
+ to staging repo.
+ - docs: Build docs and commit them to staging repo
+ - publish-release: Build maven artifacts and publish to Maven Release Repo
+ - publish-staging: Publish to staging repository
+
+GIT_REF - Release tag or commit to build from
+PACKAGE_VERSION - Release identifier in top level package directory (eg.
2.1.2-rc1)
+BUILD_VERSION - (optional) Version being built (eg. 2.1.2)
+ASF_USERNAME - Username of ASF committer
+ASF_PASSWORD - Password of ASF committer account
+GPG_KEY - GPG key used to sign release artifacts
+GPG_PASSPHRASE - Passphrase for GPG key
+EOF
+ exit 1
+}
+
+if [ $# -eq 0 ]; then
+ echo "usage: release-build.sh <docs|publish-release>"
+fi
+
+error() {
+ echo "$*"
+ exit 1
+}
+
+if [ $# -eq 0 ]; then
+ exit_with_usage
+fi
+
+
+if [[ $@ == *"help"* ]]; then
+ exit_with_usage
+fi
+
+
+# Build docs (production)
+if [[ "$1" == "docs" ]]; then
+ cd systemds
+ echo "Building SystemDS docs"
+
+ cd docs
+
+ bundle install
+ PRODUCTION=1 RELEASE_VERSION="$RELEASE_VERSION" bundle exec jekyll build
+fi
+
+GPG_OPTS="-Dgpg.keyname=${GPG_KEY} -Dgpg.passphrase=${GPG_PASSPHRASE}"
+
+cat <<EOF >../tmp-settings.xml
+<settings><servers><server>
+<id>apache.snapshots.https</id><username>${ASF_USERNAME}</username>
+<password>${ASF_PASSWORD}</password>
+</server>
+<server>
+<id>apache.releases.https</id><username>${ASF_USERNAME}</username>
+<password>${ASF_PASSWORD}</password>
+</server>
+</servers>
+</settings>
+EOF
+
+if [[ "$1" == "publish-snapshot" ]]; then
+
+ CMD="mvn --settings ../tmp-settings.xml deploy -DskipTests
-Dmaven.deploy.skip=${dry_run} \
+ -Daether.checksums.algorithms=SHA-512 \
+ ${GPG_OPTS}"
+ #
-DaltSnapshotDeploymentRepository=github::default::https://maven.pkg.github.com/j143/systemds
\
+ printf "\n #### Executing command: #### \n"
+ printf "\n $(bold $(greencolor $CMD)) \n\n"
+
+ $CMD
+
+fi
+
+
+
+if [[ "$1" == "publish-staging" ]]; then
+
+ mvn versions:set -DnewVersion=${PACKAGE_VERSION}
+
+ CMD="mvn --settings ../tmp-settings.xml clean -Pdistribution deploy \
+ -DskiptTests -Dmaven.deploy.skip=${dry_run} \
+ -Daether.checksums.algorithms=SHA-512 \
+ ${GPG_OPTS}"
+
+ printf "\n #### Executing command: #### \n"
+ printf "\n $(bold $(greencolor $CMD)) \n\n"
+
+ $CMD
+fi
+
+# if [[ -z "$GPG_KEY" ]]; then
+# echo "The environment variable $GPG_KEY is not set."
+# fi
+
+# GPG="gpg -u $GPG_KEY --no-tty --batch --pinentry-mode loopback"
+
+# Publishing to Sonatype repo, details:
+NEXUS_ROOT=https://repository.apache.org/service/local/staging
+NEXUS_PROFILE=1486a6e8f50cdf
+
+# Apache SVN Repo, details:
+RELEASE_STAGING_LOCATION="https://dist.apache.org/repos/dist/dev/systemds";
+DEST_DIR_NAME="$PACKAGE_VERSION"
+
+# NOTE:
+# 1. Build files will be saved to this folder.
+# This folder will be used by `publish-release`
+#
+# 2. this directory is passed via `file` protocol with
+# file:///${path} (3 slashes, specifies empty name)
+# refer: https://en.wikipedia.org/wiki/File_URI_scheme#How_many_slashes.3F
+mkdir temp
+tmp_repo=$(mktemp -d temp/systemds-repo-tmp-XXXXX)
+
+if [[ "$1" == "publish-release" ]]; then
+
+ # cd systemds
+
+ # Publishing spark to Maven Central Repo
+ printf "\nRelease version is ${PACKAGE_VERSION} \n"
+
+ mvn versions:set -DnewVersion=${RELEASE_VERSION}
+
+ # if ! is_dry_run; then
+ printf "Creating a Nexus staging repository \n"
+ promote_request="<promoteRequest><data><description>Apache
SystemDS</description></data></promoteRequest>"
+ out=$(curl -X POST -d "$promote_request" -u $ASF_USERNAME:$ASF_PASSWORD \
+ -H "Content-Type:application/xml" -v \
+ $NEXUS_ROOT/profiles/$NEXUS_PROFILE/start)
+ staged_repository_id=$(echo $out | sed -e
"s/.*\(orgapachesystemds-[0-9]\{4\}\).*/\1/")
+ # fi
+
+ cat <<EOF >../tmp-settings-nexus.xml
+<settings>
+<activeProfiles>
+ <activeProfile>local-temp</activeProfile>
+ </activeProfiles>
+
+ <profiles>
+ <profile>
+ <id>local-temp</id>
+ <repositories>
+ <repository>
+ <id>local-temp</id>
+ <url>file:///$PWD/${tmp_repo}</url>
+ </repository>
+ </repositories>
+ </profile>
+ </profiles>
+</settings>
+EOF
+
+ mvn --settings ../tmp-settings-nexus.xml -Pdistribution deploy \
+ -DaltDeploymentRepository=local-temp::default::file:///$PWD/${tmp_repo} \
+ -Daether.checksums.algorithms='SHA-512,SHA-1,MD5'
+
+ pushd "${tmp_repo}/org/apache/systemds"
+
+
+ if ! is_dry_run; then
+ # upload files to nexus repo
+ nexus_upload_id=$NEXUS_ROOT/deployByRepositoryId/$staged_repository_id
+ printf "\nUpload files to $nexus_upload_id \n"
+
+ for file in $(find . -type f)
+ do
+ # strip leading ./
+ file_short=$(echo $file | sed -e "s/\.\///")
+ dest_url="$nexus_upload_id/org/apache/systemds/$file_short"
+ printf "\nUploading $file_short \n"
+ curl -u $ASF_USERNAME:$ASF_PASSWORD --upload-file $file_short $dest_url
+ done
+
+ # Promote the staging repository
+
promote_request="<promoteRequest><data><stagedRepositoryId>$staged_repository_id</stagedRepositoryId></data></promoteRequest>"
+ out=$(curl -X POST -d "$promote_request" -u $ASF_USERNAME:$ASF_PASSWORD \
+ -H "Content-Type:application/xml" -v \
+ $NEXUS_ROOT/profiles/$NEXUS_PROFILE/finish)
+ printf "Closed Nexus staging repository: $staged_repository_id"
+
+ printf "\nAfter release vote passes make sure to hit release button.\n"
+
+ else
+ printf "Files will uploaded to Nexus Repo at this step."
+ fi
+
+ printf "\n ============== "
+ printf "\n Upload artifacts to dist.apache.org \n"
+
+ svn co --depth=empty $RELEASE_STAGING_LOCATION svn-systemds
+
+ if [[ ! is_dry_run ]]; then
+ stage_dir=svn-systemds/${PACKAGE_VERSION}
+ mkdir -p $stage_dir
+ else
+ stage_dir=$(mktemp -d svn-systemds/${DEST_DIR_NAME}-temp-XXXX)
+ fi
+
+ printf "\nCopy the release tarballs to svn repo \n"
+ ls *
+
+ # Remove extra files generated
+ # Keep only .zip, .tgz, and javadoc
+ find systemds -type f | grep -v -e \.zip -e \.tgz -e javadoc | xargs rm
+ eval cp systemds/${RELEASE_VERSION}/systemds-* "${stage_dir}"
+ svn add "${stage_dir}"
+
+ eval cd svn-systemds
+ svn ci --username "$ASF_USERNAME" --password "$ASF_PASSWORD" -m"Apache
SystemDS $SYSTEMDS_PACKAGE_VERSION" --no-auth-cache
+ eval cd ..
+ rm -rf svn-systemds
+
+ popd
+
+ # NOTE: Do not delete any generated release artifacts
+ # rm -rf "${tmp_repo}"
+ eval cd ..
+ exit 0
+fi
+
diff --git a/dev/release/release-utils.sh b/dev/release/release-utils.sh
new file mode 100755
index 0000000..1b688ee
--- /dev/null
+++ b/dev/release/release-utils.sh
@@ -0,0 +1,278 @@
+#!/usr/bin/env bash
+#-------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------
+
+################################################################################
+## File: release-utils.sh
+## Desc: Helper utilities for the release workflow
+################################################################################
+
+ASF_REPO="https://github.com/apache/systemds";
+ASF_REPO_CONTENT="https://raw.githubusercontent.com/apache/systemds";
+
+
+# https://github.com/j143/systemds/issues/75
+GPG_TTY=$(tty)
+export GPG_TTY
+
+# Output font formatting
+# TODO: Does not work in git bash and mac.
+export TERM=ansi
+
+bold() {
+ tput bold
+ echo -n "$@"
+
+ # turn off bold
+ tput sgr0
+}
+
+revcolor() {
+ # standout mode
+ tput smso
+ echo -n "$@"
+ tput rmso
+}
+
+
+# color values
+# red 1; green 2; blue 4; magenta 5;
+greencolor() {
+ tput setaf 2
+ echo -n "$@"
+ tput sgr0
+}
+
+# exit with error message
+error() {
+ echo "$*"
+ exit 1
+}
+
+
+# Read the configuration
+read_config() {
+ local PROMPT="$1"
+ local DEFAULT="$2"
+
+ local REPLY=
+
+ read -p "$PROMPT [$DEFAULT]: " REPLY
+ local RETVAL="${REPLY:-$DEFAULT}"
+ if [ -z "$RETVAL" ]; then
+ error "$PROMPT must be provided"
+ fi
+ echo "$RETVAL"
+}
+
+
+# parse version number from pom.xml
+# <version> tag.
+parse_version() {
+ grep -e '<version>.*</version>' | \
+ head -n 2 | tail -n 1 | cut -d '>' -f2 | cut -d '<' -f1
+}
+
+# function to log output to a .log file
+run_silent() {
+ local DESCRIPTION="$1"
+ local LOG_FILE="$2"
+
+ # Remove the first two arguments
+ # https://ss64.com/bash/shift.html
+ shift 2
+
+ printf "\n =============== "
+ printf "\n = $DESCRIPTION "
+ printf "\n Executing command: "
+ printf "\n $(bold $(greencolor $@ )) \n"
+ printf "\n Log file: $LOG_FILE "
+ printf "\n =============== \n"
+
+ # 2>&1 https://stackoverflow.com/a/818284
+ # 1 stdout, 2 stderr, >& redirect merger operator
+ . "$@" 1>"$LOG_FILE" 2>&1
+
+ # a successful command returns 0 exit code
+ local SUCCESS=$?
+ if [ $SUCCESS != 0 ]; then
+ printf "\n Command FAILED to Execute. Log files are available.\n"
+ tail "$LOG_FILE"
+ exit $SUCCESS
+ fi
+}
+
+# Clean Working directory of untracked files
+cleanup_repo() {
+ # https://git-scm.com/docs/git-clean
+ git clean -d -f -x
+}
+
+# check for the tag name in git repo
+check_for_tag() {
+ curl -s --head --fail "$ASF_REPO/releases/tag/$1" > /dev/null
+}
+
+
+# get the release info including
+# branch details, snapshot version
+# error validation
+get_release_info() {
+ if [[ -z "$GIT_BRANCH" ]]; then
+ # If not branch is specified, find the latest branch from repo
+ GIT_BRANCH=$(git ls-remote --heads "$ASF_REPO" |
+ awk '{print $2}' |
+ sort -r |
+ head -n 1 |
+ cut -d/ -f3)
+ fi
+
+ printf "\n================\n"
+ export GIT_BRANCH=$(read_config "Branch" "$GIT_BRANCH")
+
+ # Find the current version for the branch
+ local VERSION=$(curl -s "$ASF_REPO_CONTENT/$GIT_BRANCH/pom.xml" |
+ parse_version)
+
+ echo "Current branch version is $VERSION."
+
+ if [[ ! $VERSION =~ .*-SNAPSHOT ]]; then
+ error "Not a SNAPSHOT version: $VERSION"
+ fi
+
+ NEXT_VERSION="$VERSION"
+ RELEASE_VERSION="${VERSION/-SNAPSHOT/}"
+ SHORT_VERSION=$(echo "$VERSION" | cut -d . -f 1-2)
+ local REV=$(echo "$RELEASE_VERSION" | cut -d . -f 3)
+
+ # Find out what rc is being prepared.
+ # - If the current version is "x.y.0", then this is rc1 of the "x.y.0"
release.
+ # - If not, need to check whether the previous version has been already
released or not.
+ # - If it has, then we're building rc1 of the current version.
+ # - If it has not, we're building the next RC of the previous version.
+ local RC_COUNT
+ if [[ $REV != 0 ]]; then
+ local PREV_REL_REV=$((REV - 1))
+ local PREV_REL_TAG="v${SHORT_VERSION}.${PREV_REL_REV}"
+
+ if check_for_tag "$PREV_REL_TAG"; then
+ RC_COUNT=1
+ REV=$((REV + 1))
+ NEXT_VERSION="${SHORT_VERSION}.${REV}-SNAPSHOT"
+ else
+ RELEASE_VERSION="${SHORT_VERSION}.${PREV_REL_REV}"
+ RC_COUNT=$(git ls-remote --tags "$ASF_REPO" "v${RELEASE_VERSION}-rc*" |
wc -l)
+ RC_COUNT=$((RC_COUNT + 1))
+ fi
+ else
+ REV=$((REV + 1))
+ NEXT_VERSION="${SHORT_VERSION}.${REV}-SNAPSHOT"
+ RC_COUNT=1
+ fi
+
+ export NEXT_VERSION=$(read_config "Next development version" "$NEXT_VERSION")
+ export RELEASE_VERSION=$(read_config "Release" "$RELEASE_VERSION")
+
+ RC_COUNT=$(read_config "RC #" "$RC_COUNT")
+
+ # Check if the RC already exists, and if re-creating the RC, skip tag
+ # creation
+ RELEASE_TAG="${RELEASE_VERSION}-rc${RC_COUNT}"
+ SKIP_TAG=0
+
+ if check_for_tag "$RELEASE_TAG"; then
+ read -p "$RELEASE_TAG already exists. Continue anyway [Y/n]? " ANSWER
+ if [[ "$ANSWER" != "Y" ]]; then
+ error "Exiting."
+ fi
+ SKIP_TAG
+ fi
+
+ export RELEASE_TAG
+
+ GIT_REF="$RELEASE_TAG"
+
+ export GIT_REF
+ export PACKAGE_VERSION="$RELEASE_TAG"
+
+ # Git configuration info
+ # The ASF ID is obtained from
+ # https://people.apache.org/phonebook.html?unix=systemds
+ if [[ -z "$ASF_USERNAME" ]]; then
+ export ASF_USERNAME=$(read_config "ASF ID" "$LOGNAME")
+ fi
+
+ if [[ -z "$GIT_NAME" ]]; then
+ GIT_NAME=$(git config user.name || echo "")
+ export GIT_NAME=$(read_config "Full name" "$GIT_NAME")
+ fi
+
+ # git configuration info
+ if [[ -z "$GIT_EMAIL" ]]; then
+ export GIT_EMAIL="[email protected]"
+ fi
+
+ # GPG key configuration info
+ if [[ -z "$GPG_KEY" ]]; then
+ export GPG_KEY=$(read_config "GPG key" "$GIT_EMAIL")
+ fi
+
+ cat <<EOF
+================
+Release details:
+BRANCH: $GIT_BRANCH
+VERSION: $RELEASE_VERSION
+TAG: $RELEASE_TAG
+NEXT: $NEXT_VERSION
+ASF ID: $ASF_USERNAME
+GPG KEY ID: $GPG_KEY
+FULL NAME: $GIT_NAME
+E-MAIL: $GIT_EMAIL
+================
+EOF
+
+ if [[ -z "$CORRECT_RELEASE_INFO" ]]; then
+ CORRECT_RELEASE_INFO=$(read_config "Is the release info correct (1 for
Yes, 0 for No) ?" "$CORRECT_RELEASE_INFO")
+ fi
+
+ if [[ ! "$CORRECT_RELEASE_INFO" = '1' ]]; then
+ echo "Exiting."
+ exit 1
+ fi
+
+ if [[ -z "$ASF_PASSWORD" ]]; then
+ stty -echo && printf "ASF password: " && read ASF_PASSWORD && printf '\n'
&& stty echo
+ fi
+
+ if [[ -z "$GPG_PASSPHRASE" ]]; then
+ stty -echo && printf "GPG passphrase: " && read GPG_PASSPHRASE && printf
'\n' && stty echo
+ fi
+
+ export ASF_PASSWORD
+ export GPG_PASSPHRASE
+
+}
+
+is_dry_run() {
+ # By default, evaluates to false
+ [[ "$DRY_RUN" = 1 ]]
+}
+
diff --git a/pom.xml b/pom.xml
index 5d35963..f54ed8a 100644
--- a/pom.xml
+++ b/pom.xml
@@ -592,7 +592,6 @@
<exclude>**/target/**</exclude>
<exclude>**/README.md</exclude>
<exclude>**/*.svg</exclude>
-
<exclude>dev/release/damslab-pubkey.asc</exclude>
<!-- Jupyter
Notebooks -->
<exclude>**/*.ipynb</exclude>
<!-- Generated
antlr files -->
