This is an automated email from the ASF dual-hosted git repository.
dpgaspar pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-superset.git
The following commit(s) were added to refs/heads/master by this push:
new 3b13fb4 [docs] Improve and automate apache source releases (#8584)
3b13fb4 is described below
commit 3b13fb4143fedc172eded10eb20f1cedb52c7217
Author: Daniel Vaz Gaspar <[email protected]>
AuthorDate: Tue Nov 19 15:44:38 2019 +0000
[docs] Improve and automate apache source releases (#8584)
---
....from_tarball => Dockerfile.from_local_tarball} | 14 ++-
...le.from_tarball => Dockerfile.from_svn_tarball} | 5 +-
RELEASING/Dockerfile.make_tarball | 6 +-
RELEASING/README.md | 104 ++++++++++-----------
RELEASING/make_tarball.sh | 65 ++++++++++---
RELEASING/make_tarball_entrypoint.sh | 53 +++++++++++
RELEASING/{make_tarball.sh => set_release_env.sh} | 38 ++++----
RELEASING/test_run_tarball.sh | 27 +++++-
8 files changed, 208 insertions(+), 104 deletions(-)
diff --git a/RELEASING/Dockerfile.from_tarball
b/RELEASING/Dockerfile.from_local_tarball
similarity index 88%
copy from RELEASING/Dockerfile.from_tarball
copy to RELEASING/Dockerfile.from_local_tarball
index c7a9c1c..6ca3a3b 100644
--- a/RELEASING/Dockerfile.from_tarball
+++ b/RELEASING/Dockerfile.from_local_tarball
@@ -43,26 +43,24 @@ RUN chown superset /home/superset
WORKDIR /home/superset
ARG VERSION
+ARG SUPERSET_RELEASE_RC_TARBALL
-RUN svn co https://dist.apache.org/repos/dist/dev/incubator/superset/$VERSION
./
+# Can fetch source from svn or copy tarball from local mounted directory
+COPY $SUPERSET_RELEASE_RC_TARBALL ./
RUN tar -xvf *.tar.gz
-WORKDIR apache-superset-incubating-$VERSION
+WORKDIR /home/superset/apache-superset-incubating-$VERSION/superset/assets
-RUN cd superset/assets \
- && npm ci \
+RUN npm ci \
&& npm run build \
&& rm -rf node_modules
-
WORKDIR /home/superset/apache-superset-incubating-$VERSION
RUN pip install --upgrade setuptools pip \
+ && pip install -r requirements.txt \
&& pip install --no-cache-dir .
RUN flask fab babel-compile --target superset/translations
-RUN pip install -e . \
- && rm -rf /root/.cache/pip
-
ENV PATH=/home/superset/superset/bin:$PATH \
PYTHONPATH=/home/superset/superset/:$PYTHONPATH
COPY from_tarball_entrypoint.sh /entrypoint.sh
diff --git a/RELEASING/Dockerfile.from_tarball
b/RELEASING/Dockerfile.from_svn_tarball
similarity index 95%
rename from RELEASING/Dockerfile.from_tarball
rename to RELEASING/Dockerfile.from_svn_tarball
index c7a9c1c..f3acd77 100644
--- a/RELEASING/Dockerfile.from_tarball
+++ b/RELEASING/Dockerfile.from_svn_tarball
@@ -44,6 +44,7 @@ RUN chown superset /home/superset
WORKDIR /home/superset
ARG VERSION
+# Can fetch source from svn or copy tarball from local mounted directory
RUN svn co https://dist.apache.org/repos/dist/dev/incubator/superset/$VERSION
./
RUN tar -xvf *.tar.gz
WORKDIR apache-superset-incubating-$VERSION
@@ -56,13 +57,11 @@ RUN cd superset/assets \
WORKDIR /home/superset/apache-superset-incubating-$VERSION
RUN pip install --upgrade setuptools pip \
+ && pip install -r requirements.txt \
&& pip install --no-cache-dir .
RUN flask fab babel-compile --target superset/translations
-RUN pip install -e . \
- && rm -rf /root/.cache/pip
-
ENV PATH=/home/superset/superset/bin:$PATH \
PYTHONPATH=/home/superset/superset/:$PYTHONPATH
COPY from_tarball_entrypoint.sh /entrypoint.sh
diff --git a/RELEASING/Dockerfile.make_tarball
b/RELEASING/Dockerfile.make_tarball
index c7d516c..d4c4347 100644
--- a/RELEASING/Dockerfile.make_tarball
+++ b/RELEASING/Dockerfile.make_tarball
@@ -15,5 +15,9 @@
# limitations under the License.
#
FROM python:3.6-jessie
-COPY make_tarball.sh /entrypoint.sh
+
+RUN apt-get update -y
+RUN apt-get install -y jq
+
+COPY make_tarball_entrypoint.sh /entrypoint.sh
ENTRYPOINT ["/entrypoint.sh"]
diff --git a/RELEASING/README.md b/RELEASING/README.md
index ed87431..21ef756 100644
--- a/RELEASING/README.md
+++ b/RELEASING/README.md
@@ -64,13 +64,21 @@ that belong to the MAJOR.MINOR version.
The MAJOR.MINOR branch is normally a "cut" from a specific point in time from
the master branch.
Then (if needed) apply all cherries that will make the PATCH
+Next update the `CHANGELOG.md` with all the changes that are included in the
release. Make sure you have
+set your GITHUB_TOKEN environment variable.
+
+```bash
+# will overwrites the local CHANGELOG.md, somehow you need to merge it in
+github-changes -o apache -r incubator-superset --token $GITHUB_TOKEN
--between-tags <PREVIOUS_RELEASE_TAG>...<CURRENT_RELEASE_TAG>
+```
+
Finally bump the version number on `superset/static/assets/package.json`:
```json
"version": "0.34.1"
```
-Commit the change with the version number, then git tag the version with the
release candidate and push
+Commit the change with the version number, then git tag the version with the
release candidate and push to the branch
## Setting up the release environment (do every time)
@@ -79,74 +87,58 @@ often stretching over several weeks calendar time if votes
don't pass, chances a
the same terminal session won't be used for crafting the release candidate and
the
final release. Therefore, it's a good idea to do the following every time you
work on a new phase of the release process to make sure you aren't releasing
-the wrong files/using wrong names:
+the wrong files/using wrong names. There's a script to help you set correctly
all the
+necessary environment variables. Change you current directory to
`superset/RELEASING`
```bash
- # Set SUPERSET_VERSION to the release being prepared, e.g. 0.34.1.
- export SUPERSET_VERSION=XX.YY.ZZ
- # Set RC to the release candindate number. Replacing QQ below with 1
- # indicates rc1 i.e. first vote on version above (0.34.1rc1)
- export SUPERSET_RC=QQ
+ # usage: set_release_env.sh <SUPERSET_VERSION> <SUPERSET_VERSION_RC>
"<PGP_KEY_FULLNAME>"
+ . ./set_release_env.sh XX.YY.ZZ QQ "YOUR PGP KEY NAME"
```
-Then you can generate other derived environment variables that are used
-throughout the release process:
+The script will output the exported variables, for example for 0.34.1 RC1:
-```bash
- # Replace SUPERSET_PGP_FULLNAME with your PGP key name for Apache
- export SUPERSET_PGP_FULLNAME="YOURFULLNAMEHERE"
- export SUPERSET_VERSION_RC=${SUPERSET_VERSION}rc${SUPERSET_RC}
- export SUPERSET_RELEASE=apache-superset-incubating-${SUPERSET_VERSION}
- export
SUPERSET_RELEASE_RC=apache-superset-incubating-${SUPERSET_VERSION_RC}
- export SUPERSET_RELEASE_TARBALL=${SUPERSET_RELEASE}-source.tar.gz
- export SUPERSET_RELEASE_RC_TARBALL=${SUPERSET_RELEASE_RC}-source.tar.gz
+```
+ -------------------------------
+ Set Release env variables
+ SUPERSET_VERSION=0.34.1
+ SUPERSET_RC=1
+ SUPERSET_PGP_FULLNAME=You PGP Key Name
+ SUPERSET_VERSION_RC=0.34.1rc1
+ SUPERSET_RELEASE=apache-superset-incubating-0.34.1
+ SUPERSET_RELEASE_RC=apache-superset-incubating-0.34.1rc1
+ SUPERSET_RELEASE_TARBALL=apache-superset-incubating-0.34.1-source.tar.gz
+
SUPERSET_RELEASE_RC_TARBALL=apache-superset-incubating-0.34.1rc1-source.tar.gz
+ -------------------------------
```
## Preparing the release candidate
The first step of preparing an Apache Release is packaging a release candidate
-to be voted on. Start by going to the root of the repo and making sure the
-prerequisites are in order:
+to be voted on. Make sure you have correctly prepared and tagged the ready to
ship
+release on Superset's repo (MAJOR.MINOR branch), the following script will
clone
+the tag and create a signed source tarball from it:
```bash
- # Go to the root directory of the repo, e.g. `~/src/incubator-superset`
- cd ~/src/incubator-superset/
- export SUPERSET_REPO_DIR=$(pwd)
- # make sure you're on the correct branch (e.g. 0.34)
- git branch
+ # make_tarball you use the previouly set environment variables
+ # you can override by passing arguments: make_tarball.sh
<SUPERSET_VERSION> <SUPERSET_VERSION_RC> "<PGP_KEY_FULLNAME>"
+ ./make_tarball.sh
```
-Make sure the version number under `superset/assets/package.json` corresponds
-to `SUPERSET_VERSION` above (`0.34.1` in example above), and has been
committed to the
-branch.
-
-```bash
- grep ${SUPERSET_VERSION} superset/assets/package.json
-```
+Note that `make_tarball.sh`:
-If nothing shows up, either the version isn't correctly set in `package.json`,
-or the environment variable is misconfigured.
+- By default assumes you have already executed an SVN checkout to
`$HOME/svn/superset_dev`.
+This can be overriden by setting `SUPERSET_SVN_DEV_PATH` environment var to a
different svn dev directory
+- Will refuse to craft a new release candidate if a release already exists on
your local svn dev directory
+- Will check `package.json` version number and fails if it's not correctly set
-### Crafting tarball and signatures
+### Build and test the created source tarball
-Now let's craft a source release
+To build and run the just created tarball
```bash
- # Let's create a git tag
- git tag -f ${SUPERSET_VERSION_RC}
-
- # Create the target folder
- mkdir -p ~/svn/superset_dev/${SUPERSET_VERSION_RC}/
- git archive \
- --format=tar.gz ${SUPERSET_VERSION_RC} \
- --prefix="${SUPERSET_RELEASE_RC}/" \
- -o
~/svn/superset_dev/${SUPERSET_VERSION_RC}/${SUPERSET_RELEASE_RC_TARBALL}
-
- cd ~/svn/superset_dev/${SUPERSET_VERSION_RC}/
- ${SUPERSET_REPO_DIR}/scripts/sign.sh "${SUPERSET_RELEASE_RC_TARBALL}"
"${SUPERSET_PGP_FULLNAME}"
-
- # To verify to signature
- gpg --verify "${SUPERSET_RELEASE_RC_TARBALL}".asc
"${SUPERSET_RELEASE_RC_TARBALL}"
-
+ # Build and run a release candidate tarball
+ ./test_run_tarball.sh local
+ # you should be able to access localhost:5001 on your browser
+ # login using admin/admin
```
### Shipping to SVN
@@ -159,14 +151,14 @@ Now let's ship this RC into svn's dev folder
svn commit -m "Release ${SUPERSET_VERSION_RC}"
```
-### Build and test from source tarball
+### Build and test from SVN source tarball
To make a working build given a tarball
```bash
-# Build and run a release candidate tarball
-./test_run_tarball.sh
-# you should be able to access localhost:5001 on your browser
-# login using admin/admin
+ # Build and run a release candidate tarball
+ ./test_run_tarball.sh
+ # you should be able to access localhost:5001 on your browser
+ # login using admin/admin
```
### Voting
diff --git a/RELEASING/make_tarball.sh b/RELEASING/make_tarball.sh
index 5959d0f..3d0c180 100755
--- a/RELEASING/make_tarball.sh
+++ b/RELEASING/make_tarball.sh
@@ -15,24 +15,59 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
-set -ex
+set -e
-if [ -z "$VERSION" ]; then
- echo "VERSION is required to run this container"
+usage() {
+ echo "usage: make_tarball.sh <SUPERSET_VERSION> <SUPERSET_RC>
<PGP_KEY_FULLBANE>"
+}
+
+if [ -z "$1" ] || [ -z "$2" ] || [ -z "$3" ]; then
+ if [ -z "${SUPERSET_VERSION}" ] || [ -z "${SUPERSET_RC}" ] || [ -z
"${SUPERSET_PGP_FULLNAME}" ]; then
+ echo "No parameters found an no required environment variables set"
+ echo "usage: make_tarball.sh <SUPERSET_VERSION> <SUPERSET_RC>
<PGP_KEY_FULLBANE>"
+ usage;
+ exit 1
+ fi
+else
+ SUPERSET_VERSION="${1}"
+ SUPERSET_RC="${2}"
+ SUPERSET_PGP_FULLNAME="${3}"
+fi
+
+SUPERSET_VERSION_RC="${SUPERSET_VERSION}rc${SUPERSET_RC}"
+
+if [ -z "${SUPERSET_SVN_DEV_PATH}" ]; then
+ SUPERSET_SVN_DEV_PATH="$HOME/svn/superset_dev"
+fi
+
+if [[ ! -d "${SUPERSET_SVN_DEV_PATH}" ]]; then
+ echo "${SUPERSET_SVN_DEV_PATH} does not exist, you need to: svn checkout"
exit 1
fi
+if [ -d "${SUPERSET_SVN_DEV_PATH}/${SUPERSET_VERSION_RC}" ]; then
+ echo "${SUPERSET_VERSION_RC} Already exists on svn, refusing to overwrite"
+ exit 1
+fi
+
+SUPERSET_RELEASE_RC_TARBALL_PATH="${SUPERSET_SVN_DEV_PATH}"/"${SUPERSET_VERSION_RC}"/"${SUPERSET_RELEASE_RC_TARBALL}"
+DOCKER_SVN_PATH="/docker_svn"
+
+# Building docker that will produce a tarball
+docker build -t apache-builder -f Dockerfile.make_tarball .
+
+# Running docker to produce a tarball
+docker run \
+ -e SUPERSET_SVN_DEV_PATH="${DOCKER_SVN_PATH}" \
+ -e SUPERSET_VERSION="${SUPERSET_VERSION}" \
+ -e SUPERSET_VERSION_RC="${SUPERSET_VERSION_RC}" \
+ -e HOST_UID=${UID} \
+ -v "${SUPERSET_SVN_DEV_PATH}":"${DOCKER_SVN_PATH}":rw \
+ -ti apache-builder
-echo "version: $VERSION"
-cd /tmp
-git clone --depth 1 --branch $VERSION
https://github.com/apache/incubator-superset.git
-mkdir ~/$VERSION
-cd incubator-superset && \
-git archive \
- --format=tar.gz \
- --prefix=apache-superset-incubating-$VERSION/ \
- HEAD \
- -o ~/$VERSION/apache-superset-incubating.tar.gz
+gpg --armor --local-user "${SUPERSET_PGP_FULLNAME}" --output
"${SUPERSET_RELEASE_RC_TARBALL_PATH}".asc --detach-sig
"${SUPERSET_RELEASE_RC_TARBALL_PATH}"
+gpg --print-md --local-user "${SUPERSET_PGP_FULLNAME}" SHA512
"${SUPERSET_RELEASE_RC_TARBALL_PATH}" >
"${SUPERSET_RELEASE_RC_TARBALL_PATH}".sha512
-gpg --armor --output apache-superset-incubating.tar.gz.asc --detach-sig
apache-superset-incubating.tar.gz
-gpg --print-md SHA512 apache-superset-incubating.tar.gz >
apache-superset-incubating.tar.gz.sha512
+echo ---------------------------------------
+echo Release candidate tarball is ready
+echo ---------------------------------------
diff --git a/RELEASING/make_tarball_entrypoint.sh
b/RELEASING/make_tarball_entrypoint.sh
new file mode 100755
index 0000000..dde0118
--- /dev/null
+++ b/RELEASING/make_tarball_entrypoint.sh
@@ -0,0 +1,53 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+set -e
+
+if [ -z "${SUPERSET_VERSION_RC}" ] || [ -z "${SUPERSET_SVN_DEV_PATH}" ] || [
-z "${SUPERSET_VERSION}" ]; then
+ echo "SUPERSET_VERSION_RC, SUPERSET_SVN_DEV_PATH and SUPERSET_VERSION are
required to run this container"
+ exit 1
+fi
+
+SUPERSET_RELEASE_RC=apache-superset-incubating-"${SUPERSET_VERSION_RC}"
+SUPERSET_RELEASE_RC_TARBALL="${SUPERSET_RELEASE_RC}"-source.tar.gz
+SUPERSET_RELEASE_RC_BASE_PATH="${SUPERSET_SVN_DEV_PATH}"/"${SUPERSET_VERSION_RC}"
+SUPERSET_RELEASE_RC_TARBALL_PATH="${SUPERSET_RELEASE_RC_BASE_PATH}"/"${SUPERSET_RELEASE_RC_TARBALL}"
+
+# Create directory release version
+mkdir -p "${SUPERSET_SVN_DEV_PATH}"/"${SUPERSET_VERSION_RC}"
+
+# Clone superset from tag to /tmp
+cd /tmp
+git clone --depth 1 --branch ${SUPERSET_VERSION_RC}
https://github.com/apache/incubator-superset.git
+mkdir -p "${HOME}/${SUPERSET_VERSION_RC}"
+cd incubator-superset && \
+
+# Check RC version
+if ! jq -e --arg SUPERSET_VERSION $SUPERSET_VERSION '.version ==
$SUPERSET_VERSION' superset/assets/package.json
+then
+ SOURCE_VERSION=$(jq '.version' superset/assets/package.json)
+ echo "Source package.json version is wrong, found: ${SOURCE_VERSION} should
be: ${SUPERSET_VERSION}"
+ exit 1
+fi
+
+# Create source tarball
+git archive \
+ --format=tar.gz "${SUPERSET_VERSION_RC}" \
+ --prefix="${SUPERSET_RELEASE_RC}/" \
+ -o "${SUPERSET_RELEASE_RC_TARBALL_PATH}"
+
+chown -R ${HOST_UID}:${HOST_UID} "${SUPERSET_RELEASE_RC_BASE_PATH}"
diff --git a/RELEASING/make_tarball.sh b/RELEASING/set_release_env.sh
similarity index 51%
copy from RELEASING/make_tarball.sh
copy to RELEASING/set_release_env.sh
index 5959d0f..4e4423b 100755
--- a/RELEASING/make_tarball.sh
+++ b/RELEASING/set_release_env.sh
@@ -15,24 +15,24 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
-set -ex
-
-if [ -z "$VERSION" ]; then
- echo "VERSION is required to run this container"
- exit 1
-fi
+usage() {
+ echo "usage: . set_release_env.sh <SUPERSET_VERSION> <SUPERSET_RC>
<PGP_KEY_FULLBANE>"
+}
+if [ -z "$1" ] || [ -z "$2" ] || [ -z "$3" ]; then
+ usage;
+else
+ export SUPERSET_VERSION="${1}"
+ export SUPERSET_RC="${2}"
+ export SUPERSET_PGP_FULLNAME="${3}"
+ export SUPERSET_VERSION_RC="${SUPERSET_VERSION}rc${SUPERSET_RC}"
+ export SUPERSET_RELEASE=apache-superset-incubating-"${SUPERSET_VERSION}"
+ export
SUPERSET_RELEASE_RC=apache-superset-incubating-"${SUPERSET_VERSION_RC}"
+ export SUPERSET_RELEASE_TARBALL="${SUPERSET_RELEASE}"-source.tar.gz
+ export SUPERSET_RELEASE_RC_TARBALL="${SUPERSET_RELEASE_RC}"-source.tar.gz
-echo "version: $VERSION"
-cd /tmp
-git clone --depth 1 --branch $VERSION
https://github.com/apache/incubator-superset.git
-mkdir ~/$VERSION
-cd incubator-superset && \
-git archive \
- --format=tar.gz \
- --prefix=apache-superset-incubating-$VERSION/ \
- HEAD \
- -o ~/$VERSION/apache-superset-incubating.tar.gz
-
-gpg --armor --output apache-superset-incubating.tar.gz.asc --detach-sig
apache-superset-incubating.tar.gz
-gpg --print-md SHA512 apache-superset-incubating.tar.gz >
apache-superset-incubating.tar.gz.sha512
+ echo -------------------------------
+ echo Set Release env variables
+ env | grep SUPERSET
+ echo -------------------------------
+fi
diff --git a/RELEASING/test_run_tarball.sh b/RELEASING/test_run_tarball.sh
index 6521ac0..74ef747 100755
--- a/RELEASING/test_run_tarball.sh
+++ b/RELEASING/test_run_tarball.sh
@@ -22,8 +22,31 @@ if [ -z "${SUPERSET_VERSION_RC}" ]; then
exit 1
fi
-# Building a docker from a tarball
-docker build --no-cache -t apache-superset:${SUPERSET_VERSION_RC} -f
Dockerfile.from_tarball . --build-arg VERSION=${SUPERSET_VERSION_RC}
+if [ -z "${SUPERSET_SVN_DEV_PATH}" ]; then
+ SUPERSET_SVN_DEV_PATH="$HOME/svn/superset_dev"
+fi
+
+if [ ${1} == "local" ]; then
+ SUPERSET_RELEASE_RC=apache-superset-incubating-"${SUPERSET_VERSION_RC}"
+ SUPERSET_RELEASE_RC_TARBALL="${SUPERSET_RELEASE_RC}"-source.tar.gz
+
SUPERSET_TARBALL_PATH="${SUPERSET_SVN_DEV_PATH}"/${SUPERSET_VERSION_RC}/${SUPERSET_RELEASE_RC_TARBALL}
+ SUPERSET_TMP_TARBALL_FILENAME=_tmp_"${SUPERSET_VERSION_RC}".tar.gz
+ cp "${SUPERSET_TARBALL_PATH}" "${SUPERSET_TMP_TARBALL_FILENAME}"
+ docker build --no-cache \
+ -t apache-superset:${SUPERSET_VERSION_RC} \
+ -f Dockerfile.from_local_tarball . \
+ --build-arg VERSION=${SUPERSET_VERSION_RC} \
+ --build-arg SUPERSET_BUILD_FROM=local \
+ --build-arg
SUPERSET_RELEASE_RC_TARBALL="${SUPERSET_TMP_TARBALL_FILENAME}"
+ rm "${SUPERSET_TMP_TARBALL_FILENAME}"
+else
+ # Building a docker from a tarball
+ docker build --no-cache \
+ -t apache-superset:${SUPERSET_VERSION_RC} \
+ -f Dockerfile.from_svn_tarball . \
+ --build-arg VERSION=${SUPERSET_VERSION_RC} \
+ --build-arg SUPERSET_BUILD_FROM=svn
+fi
echo "---------------------------------------------------"
echo "After docker build and run, you should be able to access localhost:5001
on your browser"
