Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Automate release process #7133

Merged
merged 2 commits into from
Jan 20, 2020
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions Jenkinsfile
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
#!groovy

def dockerVersions = ['19.03.5', '18.09.9']
def dockerVersions = ['19.03.5']
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why removing the second docker version?

Copy link
Contributor Author

@ndeloof ndeloof Jan 9, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

18.09 is EOL (according to @thaJeztah)

def baseImages = ['alpine', 'debian']
def pythonVersions = ['py27', 'py37']
def pythonVersions = ['py37']

pipeline {
agent none
Expand Down
76 changes: 31 additions & 45 deletions Release.Jenkinsfile
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

def dockerVersions = ['19.03.5', '18.09.9']
def baseImages = ['alpine', 'debian']
def pythonVersions = ['py27', 'py37']
def pythonVersions = ['py37']

pipeline {
agent none
Expand Down Expand Up @@ -75,7 +75,7 @@ pipeline {
steps {
checkout scm
sh './script/setup/osx'
sh 'tox -e py27,py37 -- tests/unit'
sh 'tox -e py37 -- tests/unit'
sh './script/build/osx'
dir ('dist') {
checksum('docker-compose-Darwin-x86_64')
Expand Down Expand Up @@ -112,7 +112,7 @@ pipeline {
}
steps {
checkout scm
bat 'tox.exe -e py27,py37 -- tests/unit'
bat 'tox.exe -e py37 -- tests/unit'
powershell '.\\script\\build\\windows.ps1'
dir ('dist') {
checksum('docker-compose-Windows-x86_64.exe')
Expand Down Expand Up @@ -159,6 +159,9 @@ pipeline {
agent {
label 'linux'
}
environment {
GITHUB_TOKEN = credentials('github-release-token')
}
steps {
checkout scm
sh 'mkdir -p dist'
Expand All @@ -167,28 +170,39 @@ pipeline {
unstash "bin-linux"
unstash "bin-win"
unstash "changelog"
githubRelease()
sh("""
curl -SfL https://github.com/github/hub/releases/download/v2.13.0/hub-linux-amd64-2.13.0.tgz | tar xzv --wildcards 'hub-*/bin/hub' --strip=2
./hub release create --draft --prerelease=${env.TAG_NAME !=~ /v[0-9\.]+/} \\
-a docker-compose-Darwin-x86_64 \\
-a docker-compose-Darwin-x86_64.sha256 \\
-a docker-compose-Darwin-x86_64.tgz \\
-a docker-compose-Darwin-x86_64.tgz.sha256 \\
-a docker-compose-Linux-x86_64 \\
-a docker-compose-Linux-x86_64.sha256 \\
-a docker-compose-Windows-x86_64.exe \\
-a docker-compose-Windows-x86_64.exe.sha256 \\
-a ../script/run/run.sh \\
-F CHANGELOG.md \${TAG_NAME}
""")
}
}
}
stage('Publishing Python packages') {
agent {
label 'linux'
}
environment {
PYPIRC = credentials('pypirc-docker-dsg-cibot')
}
steps {
checkout scm
withCredentials([[$class: "FileBinding", credentialsId: 'pypirc-docker-dsg-cibot', variable: 'PYPIRC']]) {
sh """
virtualenv venv-publish
source venv-publish/bin/activate
python setup.py sdist bdist_wheel
pip install twine
twine upload --config-file ${PYPIRC} ./dist/docker-compose-${env.TAG_NAME}.tar.gz ./dist/docker_compose-${env.TAG_NAME}-py2.py3-none-any.whl
"""
}
}
post {
sh 'deactivate; rm -rf venv-publish'
sh """
rm -rf build/ dist/
pip install wheel
python setup.py sdist bdist_wheel
pip install twine
~/.local/bin/twine upload --config-file ${PYPIRC} ./dist/docker-compose-*.tar.gz ./dist/docker_compose-*-py2.py3-none-any.whl
"""
}
}
}
Expand Down Expand Up @@ -268,9 +282,8 @@ def buildRuntimeImage(baseImage) {

def pushRuntimeImage(baseImage) {
unstash "compose-${baseImage}"
sh 'echo -n "${DOCKERHUB_CREDS_PSW}" | docker login --username "${DOCKERHUB_CREDS_USR}" --password-stdin'
sh "docker load -i dist/docker-compose-${baseImage}.tar"
withDockerRegistry(credentialsId: 'dockerbuildbot-hub.docker.com') {
withDockerRegistry(credentialsId: 'dockerhub-dockerdsgcibot') {
sh "docker push docker/compose:${baseImage}-${env.TAG_NAME}"
if (baseImage == "alpine" && env.TAG_NAME != null) {
sh "docker tag docker/compose:alpine-${env.TAG_NAME} docker/compose:${env.TAG_NAME}"
Expand All @@ -279,33 +292,6 @@ def pushRuntimeImage(baseImage) {
}
}

def githubRelease() {
withCredentials([string(credentialsId: 'github-compose-release-test-token', variable: 'GITHUB_TOKEN')]) {
def prerelease = !( env.TAG_NAME ==~ /v[0-9\.]+/ )
changelog = readFile "CHANGELOG.md"
def data = """{
\"tag_name\": \"${env.TAG_NAME}\",
\"name\": \"${env.TAG_NAME}\",
\"draft\": true,
\"prerelease\": ${prerelease},
\"body\" : \"${changelog}\"
"""
echo $data

def url = "https://api.github.com/repos/docker/compose/releases"
def upload_url = sh(returnStdout: true, script: """
curl -sSf -H 'Authorization: token ${GITHUB_TOKEN}' -H 'Accept: application/json' -H 'Content-type: application/json' -X POST -d '$data' $url") \\
| jq '.upload_url | .[:rindex("{")]'
""")
sh("""
for f in * ; do
curl -sf -H 'Authorization: token ${GITHUB_TOKEN}' -H 'Accept: application/json' -H 'Content-type: application/octet-stream' \\
-X POST --data-binary @\$f ${upload_url}?name=\$f;
done
""")
}
}

def checksum(filepath) {
if (isUnix()) {
sh "openssl sha256 -r -out ${filepath}.sha256 ${filepath}"
Expand Down
202 changes: 5 additions & 197 deletions script/release/README.md
Original file line number Diff line number Diff line change
@@ -1,201 +1,9 @@
# Release HOWTO

This file describes the process of making a public release of `docker-compose`.
Please read it carefully before proceeding!
The release process is fully automated by `Release.Jenkinsfile`.

## Prerequisites
## Usage

The following things are required to bring a release to a successful conclusion

### Local Docker engine (Linux Containers)

The release script builds images that will be part of the release.

### Docker Hub account

You should be logged into a Docker Hub account that allows pushing to the
following repositories:

- docker/compose
- docker/compose-tests

### Python

The release script is written in Python and requires Python 3.3 at minimum.

### A Github account and Github API token

Your Github account needs to have write access on the `docker/compose` repo.
To generate a Github token, head over to the
[Personal access tokens](https://github.com/settings/tokens) page in your
Github settings and select "Generate new token". Your token should include
(at minimum) the following scopes:

- `repo:status`
- `public_repo`

This API token should be exposed to the release script through the
`GITHUB_TOKEN` environment variable.

### A Bintray account and Bintray API key

Your Bintray account will need to be an admin member of the
[docker-compose organization](https://bintray.com/docker-compose).
Additionally, you should generate a personal API key. To do so, click your
username in the top-right hand corner and select "Edit profile" ; on the new
page, select "API key" in the left-side menu.

This API key should be exposed to the release script through the
`BINTRAY_TOKEN` environment variable.

### A PyPi account

Said account needs to be a member of the maintainers group for the
[`docker-compose` project](https://pypi.org/project/docker-compose/).

Moreover, the `~/.pypirc` file should exist on your host and contain the
relevant pypi credentials.

The following is a sample `.pypirc` provided as a guideline:

```
[distutils]
index-servers =
pypi

[pypi]
username = user
password = pass
```

## Start a feature release

A feature release is a release that includes all changes present in the
`master` branch when initiated. It's typically versioned `X.Y.0-rc1`, where
Y is the minor version of the previous release incremented by one. A series
of one or more Release Candidates (RCs) should be made available to the public
to find and squash potential bugs.

From the root of the Compose repository, run the following command:
```
./script/release/release.sh -b <BINTRAY_USERNAME> start X.Y.0-rc1
```

After a short initialization period, the script will invite you to edit the
`CHANGELOG.md` file. Do so by being careful to respect the same format as
previous releases. Once done, the script will display a `diff` of the staged
changes for the bump commit. Once you validate these, a bump commit will be
created on the newly created release branch and pushed remotely.

The release tool then waits for the CI to conclude before proceeding.
If failures are reported, the release will be aborted until these are fixed.
Please refer to the "Resume a draft release" section below for more details.

Once all resources have been prepared, the release script will exit with a
message resembling this one:

```
You're almost done! Please verify that everything is in order and you are ready
to make the release public, then run the following command:
./script/release/release.sh -b user finalize X.Y.0-rc1
```

Once you are ready to finalize the release (making binaries and other versioned
assets public), proceed to the "Finalize a release" section of this guide.

## Start a patch release

A patch release is a release that builds off a previous release with discrete
additions. This can be an RC release after RC1 (`X.Y.0-rcZ`, `Z > 1`), a GA release
based off the final RC (`X.Y.0`), or a bugfix release based off a previous
GA release (`X.Y.Z`, `Z > 0`).

From the root of the Compose repository, run the following command:
```
./script/release/release.sh -b <BINTRAY_USERNAME> start --patch=BASE_VERSION RELEASE_VERSION
```

The process of starting a patch release is identical to starting a feature
release except for one difference ; at the beginning, the script will ask for
PR numbers you wish to cherry-pick into the release. These numbers should
correspond to existing PRs on the docker/compose repository. Multiple numbers
should be separated by whitespace.

Once you are ready to finalize the release (making binaries and other versioned
assets public), proceed to the "Finalize a release" section of this guide.

## Finalize a release

Once you're ready to make your release public, you may execute the following
command from the root of the Compose repository:
```
./script/release/release.sh -b <BINTRAY_USERNAME> finalize RELEASE_VERSION
```

Note that this command will create and publish versioned assets to the public.
As a result, it can not be reverted. The command will perform some basic
sanity checks before doing so, but it is your responsibility to ensure
everything is in order before pushing the button.

After the command exits, you should make sure:

- The `docker/compose:VERSION` image is available on Docker Hub and functional
- The `pip install -U docker-compose==VERSION` command correctly installs the
specified version
- The install command on the Github release page installs the new release

## Resume a draft release

"Resuming" a release lets you address the following situations occurring before
a release is made final:

- Cherry-pick additional PRs to include in the release
- Resume a release that was aborted because of CI failures after they've been
addressed
- Rebuild / redownload assets after manual changes have been made to the
release branch
- etc.

From the root of the Compose repository, run the following command:
```
./script/release/release.sh -b <BINTRAY_USERNAME> resume RELEASE_VERSION
```

The release tool will attempt to determine what steps it's already been through
for the specified release and pick up where it left off. Some steps are
executed again no matter what as it's assumed they'll produce different
results, like building images or downloading binaries.

## Cancel a draft release

If issues snuck into your release branch, it is sometimes easier to start from
scratch. Before a release has been finalized, it is possible to cancel it using
the following command:
```
./script/release/release.sh -b <BINTRAY_USERNAME> cancel RELEASE_VERSION
```

This will remove the release branch with this release (locally and remotely),
close the associated PR, remove the release page draft on Github and delete
the Bintray repository for it, allowing you to start fresh.

## Manual operations

Some common, release-related operations are not covered by this tool and should
be handled manually by the operator:

- After any release:
- Announce new release on Slack
- After a GA release:
- Close the release milestone
- Merge back `CHANGELOG.md` changes from the `release` branch into `master`
- Bump the version in `compose/__init__.py` to the *next* minor version
number with `dev` appended. For example, if you just released `1.4.0`,
update it to `1.5.0dev`
- Update compose_version in [github.com/docker/docker.github.io/blob/master/_config.yml](https://github.com/docker/docker.github.io/blob/master/_config.yml) and [github.com/docker/docker.github.io/blob/master/_config_authoring.yml](https://github.com/docker/docker.github.io/blob/master/_config_authoring.yml)
- Update the release note in [github.com/docker/docker.github.io](https://github.com/docker/docker.github.io/blob/master/release-notes/docker-compose.md)

## Advanced options

You can consult the full list of options for the release tool by executing
`./script/release/release.sh --help`.
1. edit `compose/__init__.py` to set release version number
1. commit and tag as `v{major}.{minor}.{patch}`
1. edit `compose/__init__.py` again to set next development version number
19 changes: 11 additions & 8 deletions script/release/generate_changelog.sh
Original file line number Diff line number Diff line change
Expand Up @@ -26,14 +26,17 @@ changes=$(pullrequests | uniq)
echo "pull requests merged within range:"
echo $changes

echo '#Features' > CHANGELOG.md
echo '#Features' > FEATURES.md
echo '#Bugs' > BUGS.md
for pr in $changes; do
curl -fs -H "Authorization: token ${GITHUB_TOKEN}" https://api.github.com/repos/docker/compose/pulls/${pr} \
| jq -r ' select( .labels[].name | contains("kind/feature") ) | "* "+.title' >> CHANGELOG.md
done
curl -fs -H "Authorization: token ${GITHUB_TOKEN}" https://api.github.com/repos/docker/compose/pulls/${pr} -o PR.json

echo '#Bugs' >> CHANGELOG.md
for pr in $changes; do
curl -fs -H "Authorization: token ${GITHUB_TOKEN}" https://api.github.com/repos/docker/compose/pulls/${pr} \
| jq -r ' select( .labels[].name | contains("kind/bug") ) | "* "+.title' >> CHANGELOG.md
cat PR.json | jq -r ' select( .labels[].name | contains("kind/feature") ) | "- "+.title' >> FEATURES.md
cat PR.json | jq -r ' select( .labels[].name | contains("kind/bug") ) | "- "+.title' >> BUGS.md
done

echo ${TAG_NAME} > CHANGELOG.md
echo >> CHANGELOG.md
cat FEATURES.md >> CHANGELOG.md
echo >> CHANGELOG.md
cat BUGS.md >> CHANGELOG.md
Loading