Fix the CI release system

setup.py rework has caused breakage in the wheels department.
Fix that and a few other shortcomings identified in the process.
Drop setuptools_scm, as it doesn't play nice with shallow clones
made by Travis.  Use git directly instead.

Author: elprans

.ci/build-manylinux-wheels.sh

@@ -5,25 +5,23 @@ set -e -x
 # Compile wheels
 PYTHON="/opt/python/${PYTHON_VERSION}/bin/python"
 PIP="/opt/python/${PYTHON_VERSION}/bin/pip"
-${PIP} install --upgrade pip wheel
-${PIP} install --upgrade setuptools
-${PIP} install -r /io/.ci/requirements.txt
-make -C /io/ PYTHON="${PYTHON}"
-${PIP} wheel /io/ -w /io/dist/
+${PIP} install --upgrade setuptools pip wheel
+cd /io
+make clean
+${PYTHON} setup.py bdist_wheel
 
 # Bundle external shared libraries into the wheels.
 for whl in /io/dist/*.whl; do
     auditwheel repair $whl -w /io/dist/
     rm /io/dist/*-linux_*.whl
 done
 
+${PIP} install ${PYMODULE}[test] -f "file:///io/dist"
+
 # Grab docker host, where Postgres should be running.
 export PGHOST=$(ip route | awk '/default/ { print $3 }' | uniq)
 export PGUSER="postgres"
 
-PYTHON="/opt/python/${PYTHON_VERSION}/bin/python"
-PIP="/opt/python/${PYTHON_VERSION}/bin/pip"
-${PIP} install ${PYMODULE} --no-index -f file:///io/dist
 rm -rf /io/tests/__pycache__
-make -C /io/ PYTHON="${PYTHON}" testinstalled
+make -C /io PYTHON="${PYTHON}" testinstalled
 rm -rf /io/tests/__pycache__

.ci/travis-build-docs.sh

@@ -7,5 +7,5 @@ if [[ "${BUILD}" != *docs* ]]; then
     exit 0
 fi
 
-pip install -U .[dev]
+pip install -U -e .[docs]
 make htmldocs SPHINXOPTS="-q -W -j4"

.ci/travis-build-wheels.sh

@@ -24,9 +24,7 @@ if [ "${PACKAGE_VERSION}" == "${PYPI_VERSION}" ]; then
 fi
 
 
-pushd $(dirname $0) > /dev/null
-_root=$(dirname $(pwd -P))
-popd > /dev/null
+_root="${TRAVIS_BUILD_DIR}"
 
 
 _upload_wheels() {
@@ -35,6 +33,9 @@ _upload_wheels() {
 }
 
 
+pip install -U -r ".ci/requirements-publish.txt"
+
+
 if [ "${TRAVIS_OS_NAME}" == "linux" ]; then
     for pyver in ${RELEASE_PYTHON_VERSIONS}; do
         ML_PYTHON_VERSION=$(python3 -c \
@@ -49,6 +50,7 @@ if [ "${TRAVIS_OS_NAME}" == "linux" ]; then
                 -v "${_root}":/io \
                 -e "PYMODULE=${PYMODULE}" \
                 -e "PYTHON_VERSION=${ML_PYTHON_VERSION}" \
+                -e "ASYNCPG_VERSION=${PACKAGE_VERSION}" \
                 "${ML_IMAGE}" /io/.ci/build-manylinux-wheels.sh
 
             _upload_wheels
@@ -58,13 +60,11 @@ if [ "${TRAVIS_OS_NAME}" == "linux" ]; then
 elif [ "${TRAVIS_OS_NAME}" == "osx" ]; then
     export PGINSTALLATION="/usr/local/opt/postgresql@${PGVERSION}/bin"
 
-    make clean && make -C "${_root}"
-    pip wheel "${_root}" -w "${_root}/dist/"
+    make clean
+    python setup.py bdist_wheel
 
-    pip install ${PYMODULE} --no-index -f "file:///${_root}/dist"
-    pushd / >/dev/null
-    make -C "${_root}" testinstalled
-    popd >/dev/null
+    pip install ${PYMODULE}[test] -f "file:///${_root}/dist"
+    make -C "${_root}" ASYNCPG_VERSION="${PACKAGE_VERSION}" testinstalled
 
     _upload_wheels
 

.ci/travis-install.sh

@@ -9,4 +9,5 @@ if [ "${TRAVIS_OS_NAME}" == "osx" ]; then
 fi
 
 pip install --upgrade setuptools pip wheel
-pip install --upgrade -e .[dev]
+pip download --dest=/tmp/deps .[test]
+pip install -U --no-index --find-links=/tmp/deps /tmp/deps/*

.ci/travis-release.sh

@@ -3,11 +3,11 @@
 set -e -x
 
 if [ -z "${TRAVIS_TAG}" ]; then
-    # Not a release
+    # Not a tagged commit.
     exit 0
 fi
 
-pip install -U ".ci/requirements-publish.txt"
+pip install -U -r ".ci/requirements-publish.txt"
 
 PACKAGE_VERSION=$(python ".ci/package-version.py")
 PYPI_VERSION=$(python ".ci/pypi-check.py" "${PYMODULE}")

.github/release_log.py

@@ -1,3 +1,5 @@
+#!/usr/bin/env python3
+#
 # Copyright (C) 2016-present the asyncpg authors and contributors
 # <see AUTHORS file>
 #

Makefile

@@ -2,6 +2,7 @@
 
 
 PYTHON ?= python
+ROOT = $(dir $(realpath $(firstword $(MAKEFILE_LIST))))
 
 
 all: compile
@@ -30,13 +31,14 @@ test:
 
 
 testinstalled:
-	$(PYTHON) tests/__init__.py
-	USE_UVLOOP=1 $(PYTHON) tests/__init__.py
+	cd /tmp && $(PYTHON) $(ROOT)/tests/__init__.py
+	cd /tmp && USE_UVLOOP=1 $(PYTHON) $(ROOT)/tests/__init__.py
 
 
 quicktest:
 	$(PYTHON) setup.py test
 
 
-htmldocs: compile
+htmldocs:
+	$(PYTHON) setup.py build_ext --inplace
 	$(MAKE) -C docs html

asyncpg/__init__.py

@@ -29,6 +29,6 @@
 #
 #    Source and wheel distributions built from development
 #    snapshots will automatically include the git revision
-#    in __version__, for example: '0.16.0.dev22+ge06ad03'
+#    in __version__, for example: '0.16.0.dev0+ge06ad03'
 
 __version__ = '0.16.0.dev0'

setup.py

@@ -15,6 +15,7 @@
 import pathlib
 import platform
 import re
+import subprocess
 
 # We use vanilla build_ext, to avoid importing Cython via
 # the setuptools version.
@@ -28,18 +29,27 @@
 
 CYTHON_DEPENDENCY = 'Cython==0.28.3'
 
+# Minimal dependencies required to test asyncpg.
+TEST_DEPENDENCIES = [
+    'flake8~=3.5.0',
+    'uvloop>=0.8.0;platform_system!="Windows"',
+]
+
+# Dependencies required to build documentation.
+DOC_DEPENDENCIES = [
+    'Sphinx~=1.7.3',
+    'sphinxcontrib-asyncio~=0.2.0',
+    'sphinx_rtd_theme~=0.2.4',
+]
+
 EXTRA_DEPENDENCIES = {
+    'docs': DOC_DEPENDENCIES,
+    'test': TEST_DEPENDENCIES,
     # Dependencies required to develop asyncpg.
     'dev': [
         CYTHON_DEPENDENCY,
-        'flake8~=3.5.0',
-        'pytest~=3.0.7',
-        'uvloop>=0.8.0;platform_system!="Windows"',
-        # Docs
-        'Sphinx~=1.7.3',
-        'sphinxcontrib-asyncio~=0.2.0',
-        'sphinx_rtd_theme~=0.2.4',
-    ]
+        'pytest>=3.6.0',
+    ] + DOC_DEPENDENCIES + TEST_DEPENDENCIES
 }
 
 
@@ -50,6 +60,45 @@
     CFLAGS.extend(['-fsigned-char', '-Wall', '-Wsign-compare', '-Wconversion'])
 
 
+_ROOT = pathlib.Path(__file__).parent
+
+
+with open(str(_ROOT / 'README.rst')) as f:
+    readme = f.read()
+
+
+with open(str(_ROOT / 'asyncpg' / '__init__.py')) as f:
+    for line in f:
+        if line.startswith('__version__ ='):
+            _, _, version = line.partition('=')
+            VERSION = version.strip(" \n'\"")
+            break
+    else:
+        raise RuntimeError(
+            'unable to read the version from asyncpg/__init__.py')
+
+
+if (_ROOT / '.git').is_dir() and 'dev' in VERSION:
+    # This is a git checkout, use git to
+    # generate a precise version.
+    def git_commitish():
+        env = {}
+        v = os.environ.get('PATH')
+        if v is not None:
+            env['PATH'] = v
+
+        git = subprocess.run(['git', 'rev-parse', 'HEAD'], env=env, cwd=_ROOT,
+                             stdout=subprocess.PIPE)
+        if git.returncode == 0:
+            commitish = git.stdout.strip().decode('ascii')
+        else:
+            commitish = 'unknown'
+
+        return commitish
+
+    VERSION += '+' + git_commitish()[:7]
+
+
 class VersionMixin:
 
     def _fix_version(self, filename):
@@ -183,55 +232,10 @@ def finalize_options(self):
         super(build_ext, self).finalize_options()
 
 
-_ROOT = pathlib.Path(__file__).parent
-
-
-with open(str(_ROOT / 'README.rst')) as f:
-    readme = f.read()
-
-
-with open(str(_ROOT / 'asyncpg' / '__init__.py')) as f:
-    for line in f:
-        if line.startswith('__version__ ='):
-            _, _, version = line.partition('=')
-            VERSION = version.strip(" \n'\"")
-            break
-    else:
-        raise RuntimeError(
-            'unable to read the version from asyncpg/__init__.py')
-
-
-extra_setup_kwargs = {}
 setup_requires = []
 
-if (_ROOT / '.git').is_dir():
-    # This is a git checkout, use setuptools_scm to
-    # generage a precise version.
-    def version_scheme(v):
-        from setuptools_scm import version
-
-        if v.exact:
-            fv = v.format_with("{tag}")
-            if fv != VERSION:
-                raise RuntimeError(
-                    'asyncpg.__version__ does not match the git tag')
-        else:
-            fv = v.format_next_version(
-                version.guess_next_simple_semver,
-                retain=version.SEMVER_MINOR)
-
-            if not fv.startswith(VERSION[:-1]):
-                raise RuntimeError(
-                    'asyncpg.__version__ does not match the git tag')
-
-        return fv
-
-    setup_requires.append('setuptools_scm')
-    extra_setup_kwargs['use_scm_version'] = {
-        'version_scheme': version_scheme
-    }
-
-if not (_ROOT / 'asyncpg' / 'protocol' / 'protocol.c').exists():
+if (not (_ROOT / 'asyncpg' / 'protocol' / 'protocol.c').exists() or
+        '--cython-always' in sys.argv):
     # No Cython output, require Cython to build.
     setup_requires.append(CYTHON_DEPENDENCY)
 
@@ -273,5 +277,4 @@ def version_scheme(v):
     test_suite='tests.suite',
     extras_require=EXTRA_DEPENDENCIES,
     setup_requires=setup_requires,
-    **extra_setup_kwargs
 )

tests/__init__.py

@@ -5,13 +5,15 @@
 # the Apache 2.0 License: http://www.apache.org/licenses/LICENSE-2.0
 
 
+import pathlib
 import sys
 import unittest
 
 
 def suite():
     test_loader = unittest.TestLoader()
-    test_suite = test_loader.discover('.', pattern='test_*.py')
+    test_suite = test_loader.discover(str(pathlib.Path(__file__).parent),
+                                      pattern='test_*.py')
     return test_suite