[docs] Improve and automate apache source releases (#8584)

RELEASING/Dockerfile.from_local_tarball

+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+FROM python:3.6-jessie
+
+RUN useradd --user-group --create-home --no-log-init --shell /bin/bash superset
+
+# Configure environment
+ENV LANG=C.UTF-8 \
+    LC_ALL=C.UTF-8
+
+RUN apt-get update -y
+
+# Install dependencies to fix `curl https support error` and `elaying package configuration warning`
+RUN apt-get install -y apt-transport-https apt-utils
+
+# Install superset dependencies
+# https://superset.incubator.apache.org/installation.html#os-dependencies
+RUN apt-get install -y build-essential libssl-dev \
+    libffi-dev python3-dev libsasl2-dev libldap2-dev libxi-dev
+
+# Install nodejs for custom build
+# https://superset.incubator.apache.org/installation.html#making-your-own-build
+# https://nodejs.org/en/download/package-manager/
+RUN curl -sL https://deb.nodesource.com/setup_10.x | bash - \
+    && apt-get install -y nodejs
+
+RUN mkdir -p /home/superset
+RUN chown superset /home/superset
+
+WORKDIR /home/superset
+ARG VERSION
+ARG SUPERSET_RELEASE_RC_TARBALL
+
+# Can fetch source from svn or copy tarball from local mounted directory
+COPY $SUPERSET_RELEASE_RC_TARBALL ./
+RUN tar -xvf *.tar.gz
+WORKDIR /home/superset/apache-superset-incubating-$VERSION/superset/assets
+
+RUN npm ci \
+    && npm run build \
+    && rm -rf node_modules
+
+WORKDIR /home/superset/apache-superset-incubating-$VERSION
+RUN pip install --upgrade setuptools pip \
+    && pip install -r requirements.txt \
+    && pip install --no-cache-dir .
+
+RUN flask fab babel-compile --target superset/translations
+
+ENV PATH=/home/superset/superset/bin:$PATH \
+    PYTHONPATH=/home/superset/superset/:$PYTHONPATH
+COPY from_tarball_entrypoint.sh /entrypoint.sh
+ENTRYPOINT ["/entrypoint.sh"]

RELEASING/Dockerfile.from_tarball → RELEASING/Dockerfile.from_svn_tarball

@@ -44,6 +44,7 @@ RUN chown superset /home/superset
 WORKDIR /home/superset
 ARG VERSION
 
+# Can fetch source from svn or copy tarball from local mounted directory
 RUN svn co https://dist.apache.org/repos/dist/dev/incubator/superset/$VERSION ./
 RUN tar -xvf *.tar.gz
 WORKDIR apache-superset-incubating-$VERSION
@@ -56,13 +57,11 @@ RUN cd superset/assets \
 
 WORKDIR /home/superset/apache-superset-incubating-$VERSION
 RUN pip install --upgrade setuptools pip \
+    && pip install -r requirements.txt \
     && pip install --no-cache-dir .
 
 RUN flask fab babel-compile --target superset/translations
 
-RUN pip install -e . \
-    && rm -rf /root/.cache/pip
-
 ENV PATH=/home/superset/superset/bin:$PATH \
     PYTHONPATH=/home/superset/superset/:$PYTHONPATH
 COPY from_tarball_entrypoint.sh /entrypoint.sh

RELEASING/Dockerfile.make_tarball

@@ -15,5 +15,9 @@
 # limitations under the License.
 #
 FROM python:3.6-jessie
-COPY make_tarball.sh /entrypoint.sh
+
+RUN apt-get update -y
+RUN apt-get install -y jq
+
+COPY make_tarball_entrypoint.sh /entrypoint.sh
 ENTRYPOINT ["/entrypoint.sh"]

RELEASING/README.md

@@ -64,13 +64,21 @@ that belong to the MAJOR.MINOR version.
 The MAJOR.MINOR branch is normally a "cut" from a specific point in time from the master branch.
 Then (if needed) apply all cherries that will make the PATCH
 
+Next update the `CHANGELOG.md` with all the changes that are included in the release. Make sure you have
+set your GITHUB_TOKEN environment variable.
+
+```bash
+# will overwrites the local CHANGELOG.md, somehow you need to merge it in
+github-changes -o apache -r incubator-superset --token $GITHUB_TOKEN --between-tags <PREVIOUS_RELEASE_TAG>...<CURRENT_RELEASE_TAG>
+```
+
 Finally bump the version number on `superset/static/assets/package.json`:
 
 ```json
     "version": "0.34.1"
 ```
 
-Commit the change with the version number, then git tag the version with the release candidate and push
+Commit the change with the version number, then git tag the version with the release candidate and push to the branch
 
 ## Setting up the release environment (do every time)
 
@@ -79,74 +87,58 @@ often stretching over several weeks calendar time if votes don't pass, chances a
 the same terminal session won't be used for crafting the release candidate and the
 final release. Therefore, it's a good idea to do the following every time you
 work on a new phase of the release process to make sure you aren't releasing
-the wrong files/using wrong names:
+the wrong files/using wrong names. There's a script to help you set correctly all the
+necessary environment variables. Change you current directory to `superset/RELEASING`
 
 ```bash
-    # Set SUPERSET_VERSION to the release being prepared, e.g. 0.34.1.
-    export SUPERSET_VERSION=XX.YY.ZZ
-    # Set RC to the release candindate number. Replacing QQ below with 1
-    # indicates rc1 i.e. first vote on version above (0.34.1rc1)
-    export SUPERSET_RC=QQ
+    # usage: set_release_env.sh <SUPERSET_VERSION> <SUPERSET_VERSION_RC> "<PGP_KEY_FULLNAME>"
+    . ./set_release_env.sh XX.YY.ZZ QQ "YOUR PGP KEY NAME"
 ```
 
-Then you can generate other derived environment variables that are used
-throughout the release process:
+The script will output the exported variables, for example for 0.34.1 RC1:
 
-```bash
-    # Replace SUPERSET_PGP_FULLNAME with your PGP key name for Apache
-    export SUPERSET_PGP_FULLNAME="YOURFULLNAMEHERE"
-    export SUPERSET_VERSION_RC=${SUPERSET_VERSION}rc${SUPERSET_RC}
-    export SUPERSET_RELEASE=apache-superset-incubating-${SUPERSET_VERSION}
-    export SUPERSET_RELEASE_RC=apache-superset-incubating-${SUPERSET_VERSION_RC}
-    export SUPERSET_RELEASE_TARBALL=${SUPERSET_RELEASE}-source.tar.gz
-    export SUPERSET_RELEASE_RC_TARBALL=${SUPERSET_RELEASE_RC}-source.tar.gz
+```
+    -------------------------------
+    Set Release env variables
+    SUPERSET_VERSION=0.34.1
+    SUPERSET_RC=1
+    SUPERSET_PGP_FULLNAME=You PGP Key Name
+    SUPERSET_VERSION_RC=0.34.1rc1
+    SUPERSET_RELEASE=apache-superset-incubating-0.34.1
+    SUPERSET_RELEASE_RC=apache-superset-incubating-0.34.1rc1
+    SUPERSET_RELEASE_TARBALL=apache-superset-incubating-0.34.1-source.tar.gz
+    SUPERSET_RELEASE_RC_TARBALL=apache-superset-incubating-0.34.1rc1-source.tar.gz
+    -------------------------------
 ```
 
 ## Preparing the release candidate
 
 The first step of preparing an Apache Release is packaging a release candidate
-to be voted on. Start by going to the root of the repo and making sure the
-prerequisites are in order:
+to be voted on. Make sure you have correctly prepared and tagged the ready to ship
+release on Superset's repo (MAJOR.MINOR branch), the following script will clone 
+the tag and create a signed source tarball from it:
 
 ```bash
-    # Go to the root directory of the repo, e.g. `~/src/incubator-superset`
-    cd ~/src/incubator-superset/
-    export SUPERSET_REPO_DIR=$(pwd)
-    # make sure you're on the correct branch (e.g. 0.34)
-    git branch
+    # make_tarball you use the previouly set environment variables
+    # you can override by passing arguments: make_tarball.sh <SUPERSET_VERSION> <SUPERSET_VERSION_RC> "<PGP_KEY_FULLNAME>"
+    ./make_tarball.sh
 ```
 
-Make sure the version number under `superset/assets/package.json` corresponds
-to `SUPERSET_VERSION` above (`0.34.1` in example above), and has been committed to the
-branch.
-
-```bash
-    grep ${SUPERSET_VERSION} superset/assets/package.json
-```
+Note that `make_tarball.sh`:
 
-If nothing shows up, either the version isn't correctly set in `package.json`,
-or the environment variable is misconfigured.
+- By default assumes you have already executed an SVN checkout to `$HOME/svn/superset_dev`. 
+This can be overriden by setting `SUPERSET_SVN_DEV_PATH` environment var to a different svn dev directory 
+- Will refuse to craft a new release candidate if a release already exists on your local svn dev directory
+- Will check `package.json` version number and fails if it's not correctly set
 
-### Crafting tarball and signatures
+### Build and test the created source tarball
 
-Now let's craft a source release
+To build and run the just created tarball 
 ```bash
-    # Let's create a git tag
-    git tag -f ${SUPERSET_VERSION_RC}
-
-    # Create the target folder
-    mkdir -p ~/svn/superset_dev/${SUPERSET_VERSION_RC}/
-    git archive \
-        --format=tar.gz ${SUPERSET_VERSION_RC} \
-        --prefix="${SUPERSET_RELEASE_RC}/" \
-        -o ~/svn/superset_dev/${SUPERSET_VERSION_RC}/${SUPERSET_RELEASE_RC_TARBALL}
-
-    cd ~/svn/superset_dev/${SUPERSET_VERSION_RC}/
-    ${SUPERSET_REPO_DIR}/scripts/sign.sh "${SUPERSET_RELEASE_RC_TARBALL}" "${SUPERSET_PGP_FULLNAME}"
-
-    # To verify to signature
-    gpg --verify "${SUPERSET_RELEASE_RC_TARBALL}".asc "${SUPERSET_RELEASE_RC_TARBALL}"
-
+    # Build and run a release candidate tarball
+    ./test_run_tarball.sh local
+    # you should be able to access localhost:5001 on your browser
+    # login using admin/admin
 ```
 
 ### Shipping to SVN
@@ -159,14 +151,14 @@ Now let's ship this RC into svn's dev folder
     svn commit -m "Release ${SUPERSET_VERSION_RC}"
 ```
 
-### Build and test from source tarball
+### Build and test from SVN source tarball
 
 To make a working build given a tarball
 ```bash
-# Build and run a release candidate tarball
-./test_run_tarball.sh
-# you should be able to access localhost:5001 on your browser
-# login using admin/admin
+    # Build and run a release candidate tarball
+    ./test_run_tarball.sh
+    # you should be able to access localhost:5001 on your browser
+    # login using admin/admin
 ```
 
 ### Voting

RELEASING/make_tarball.sh

@@ -15,24 +15,59 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-set -ex
+set -e
 
-if [ -z "$VERSION" ]; then
-  echo "VERSION is required to run this container"
+usage() {
+   echo "usage: make_tarball.sh <SUPERSET_VERSION> <SUPERSET_RC> <PGP_KEY_FULLBANE>"
+}
+
+if [ -z "$1" ] || [ -z "$2" ] || [ -z "$3" ]; then
+  if [ -z "${SUPERSET_VERSION}" ] || [ -z "${SUPERSET_RC}" ] || [ -z "${SUPERSET_PGP_FULLNAME}" ]; then
+    echo "No parameters found an no required environment variables set"
+    echo "usage: make_tarball.sh <SUPERSET_VERSION> <SUPERSET_RC> <PGP_KEY_FULLBANE>"
+    usage;
+    exit 1
+  fi
+else
+  SUPERSET_VERSION="${1}"
+  SUPERSET_RC="${2}"
+  SUPERSET_PGP_FULLNAME="${3}"
+fi
+
+SUPERSET_VERSION_RC="${SUPERSET_VERSION}rc${SUPERSET_RC}"
+
+if [ -z "${SUPERSET_SVN_DEV_PATH}" ]; then
+  SUPERSET_SVN_DEV_PATH="$HOME/svn/superset_dev"
+fi
+
+if [[ ! -d "${SUPERSET_SVN_DEV_PATH}" ]]; then
+  echo "${SUPERSET_SVN_DEV_PATH} does not exist, you need to: svn checkout"
   exit 1
 fi
 
+if [ -d "${SUPERSET_SVN_DEV_PATH}/${SUPERSET_VERSION_RC}" ]; then
+  echo "${SUPERSET_VERSION_RC} Already exists on svn, refusing to overwrite"
+  exit 1
+fi
+
+SUPERSET_RELEASE_RC_TARBALL_PATH="${SUPERSET_SVN_DEV_PATH}"/"${SUPERSET_VERSION_RC}"/"${SUPERSET_RELEASE_RC_TARBALL}"
+DOCKER_SVN_PATH="/docker_svn"
+
+# Building docker that will produce a tarball
+docker build -t apache-builder -f Dockerfile.make_tarball .
+
+# Running docker to produce a tarball
+docker run \
+      -e SUPERSET_SVN_DEV_PATH="${DOCKER_SVN_PATH}" \
+      -e SUPERSET_VERSION="${SUPERSET_VERSION}" \
+      -e SUPERSET_VERSION_RC="${SUPERSET_VERSION_RC}" \
+      -e HOST_UID=${UID} \
+      -v "${SUPERSET_SVN_DEV_PATH}":"${DOCKER_SVN_PATH}":rw \
+      -ti apache-builder
 
-echo "version: $VERSION"
-cd /tmp
-git clone --depth 1 --branch $VERSION https://github.com/apache/incubator-superset.git
-mkdir ~/$VERSION
-cd incubator-superset && \
-git archive \
-    --format=tar.gz \
-    --prefix=apache-superset-incubating-$VERSION/ \
-    HEAD \
-    -o ~/$VERSION/apache-superset-incubating.tar.gz
+gpg --armor --local-user "${SUPERSET_PGP_FULLNAME}" --output "${SUPERSET_RELEASE_RC_TARBALL_PATH}".asc --detach-sig "${SUPERSET_RELEASE_RC_TARBALL_PATH}"
+gpg --print-md --local-user "${SUPERSET_PGP_FULLNAME}" SHA512 "${SUPERSET_RELEASE_RC_TARBALL_PATH}" > "${SUPERSET_RELEASE_RC_TARBALL_PATH}".sha512
 
-gpg --armor --output apache-superset-incubating.tar.gz.asc --detach-sig apache-superset-incubating.tar.gz
-gpg --print-md SHA512 apache-superset-incubating.tar.gz > apache-superset-incubating.tar.gz.sha512
+echo ---------------------------------------
+echo Release candidate tarball is ready
+echo ---------------------------------------

RELEASING/make_tarball_entrypoint.sh

+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+set -e
+
+if [ -z "${SUPERSET_VERSION_RC}" ] || [ -z "${SUPERSET_SVN_DEV_PATH}" ] || [ -z "${SUPERSET_VERSION}" ]; then
+  echo "SUPERSET_VERSION_RC, SUPERSET_SVN_DEV_PATH and SUPERSET_VERSION are required to run this container"
+  exit 1
+fi
+
+SUPERSET_RELEASE_RC=apache-superset-incubating-"${SUPERSET_VERSION_RC}"
+SUPERSET_RELEASE_RC_TARBALL="${SUPERSET_RELEASE_RC}"-source.tar.gz
+SUPERSET_RELEASE_RC_BASE_PATH="${SUPERSET_SVN_DEV_PATH}"/"${SUPERSET_VERSION_RC}"
+SUPERSET_RELEASE_RC_TARBALL_PATH="${SUPERSET_RELEASE_RC_BASE_PATH}"/"${SUPERSET_RELEASE_RC_TARBALL}"
+
+# Create directory release version
+mkdir -p "${SUPERSET_SVN_DEV_PATH}"/"${SUPERSET_VERSION_RC}"
+
+# Clone superset from tag to /tmp
+cd /tmp
+git clone --depth 1 --branch ${SUPERSET_VERSION_RC} https://github.com/apache/incubator-superset.git
+mkdir -p "${HOME}/${SUPERSET_VERSION_RC}"
+cd incubator-superset && \
+
+# Check RC version
+if ! jq -e --arg SUPERSET_VERSION $SUPERSET_VERSION '.version == $SUPERSET_VERSION' superset/assets/package.json
+then
+  SOURCE_VERSION=$(jq '.version' superset/assets/package.json)
+  echo "Source package.json version is wrong, found: ${SOURCE_VERSION} should be: ${SUPERSET_VERSION}"
+  exit 1
+fi
+
+# Create source tarball
+git archive \
+    --format=tar.gz "${SUPERSET_VERSION_RC}" \
+    --prefix="${SUPERSET_RELEASE_RC}/" \
+    -o "${SUPERSET_RELEASE_RC_TARBALL_PATH}"
+
+chown -R ${HOST_UID}:${HOST_UID} "${SUPERSET_RELEASE_RC_BASE_PATH}"

RELEASING/set_release_env.sh

+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+usage() {
+   echo "usage: . set_release_env.sh <SUPERSET_VERSION> <SUPERSET_RC> <PGP_KEY_FULLBANE>"
+}
+
+if [ -z "$1" ] || [ -z "$2" ] || [ -z "$3" ]; then
+  usage;
+else
+  export SUPERSET_VERSION="${1}"
+  export SUPERSET_RC="${2}"
+  export SUPERSET_PGP_FULLNAME="${3}"
+  export SUPERSET_VERSION_RC="${SUPERSET_VERSION}rc${SUPERSET_RC}"
+  export SUPERSET_RELEASE=apache-superset-incubating-"${SUPERSET_VERSION}"
+  export SUPERSET_RELEASE_RC=apache-superset-incubating-"${SUPERSET_VERSION_RC}"
+  export SUPERSET_RELEASE_TARBALL="${SUPERSET_RELEASE}"-source.tar.gz
+  export SUPERSET_RELEASE_RC_TARBALL="${SUPERSET_RELEASE_RC}"-source.tar.gz
+
+  echo -------------------------------
+  echo Set Release env variables
+  env | grep SUPERSET
+  echo -------------------------------
+fi

RELEASING/test_run_tarball.sh

@@ -22,8 +22,31 @@ if [ -z "${SUPERSET_VERSION_RC}" ]; then
   exit 1
 fi
 
-# Building a docker from a tarball
-docker build --no-cache -t apache-superset:${SUPERSET_VERSION_RC} -f Dockerfile.from_tarball . --build-arg VERSION=${SUPERSET_VERSION_RC}
+if [ -z "${SUPERSET_SVN_DEV_PATH}" ]; then
+  SUPERSET_SVN_DEV_PATH="$HOME/svn/superset_dev"
+fi
+
+if [ ${1} == "local" ]; then
+  SUPERSET_RELEASE_RC=apache-superset-incubating-"${SUPERSET_VERSION_RC}"
+  SUPERSET_RELEASE_RC_TARBALL="${SUPERSET_RELEASE_RC}"-source.tar.gz
+  SUPERSET_TARBALL_PATH="${SUPERSET_SVN_DEV_PATH}"/${SUPERSET_VERSION_RC}/${SUPERSET_RELEASE_RC_TARBALL}
+  SUPERSET_TMP_TARBALL_FILENAME=_tmp_"${SUPERSET_VERSION_RC}".tar.gz
+  cp "${SUPERSET_TARBALL_PATH}" "${SUPERSET_TMP_TARBALL_FILENAME}"
+  docker build --no-cache \
+        -t apache-superset:${SUPERSET_VERSION_RC} \
+        -f Dockerfile.from_local_tarball . \
+        --build-arg VERSION=${SUPERSET_VERSION_RC} \
+        --build-arg SUPERSET_BUILD_FROM=local \
+        --build-arg SUPERSET_RELEASE_RC_TARBALL="${SUPERSET_TMP_TARBALL_FILENAME}"
+  rm "${SUPERSET_TMP_TARBALL_FILENAME}"
+else
+  # Building a docker from a tarball
+  docker build --no-cache \
+        -t apache-superset:${SUPERSET_VERSION_RC} \
+        -f Dockerfile.from_svn_tarball . \
+        --build-arg VERSION=${SUPERSET_VERSION_RC} \
+        --build-arg SUPERSET_BUILD_FROM=svn
+fi
 
 echo "---------------------------------------------------"
 echo "After docker build and run, you should be able to access localhost:5001 on your browser"