Initial implementation

Author: asottile

.coveragerc

@@ -1,7 +1,8 @@
 [run]
+parallel = True
 branch = True
-source =
-    .
+source = $TOP
+data_file = $TOP/.coverage
 omit =
     .tox/*
     /usr/*

.gitignore

@@ -5,7 +5,7 @@
 .*.sw[a-z]
 .coverage
 .tox
-.venv-touch
+.venv.touch
 /.cache
 /build
 /venv*

.travis.yml

@@ -1,15 +1,25 @@
 language: python
-env: # These should match the tox env list
-    - TOXENV=py27
-    - TOXENV=py34
-    - TOXENV=py35
-    - TOXENV=pypy
-install: pip install coveralls tox
-script: tox
+env:
+    - TOXENV=py27 GO=1.5
+    - TOXENV=py27 GO=1.6
+    - TOXENV=py34 GO=1.6
+    - TOXENV=py35 GO=1.6
+    - TOXENV=pypy GO=1.6
+install:
+    - eval "$(gimme $GO)"
+    - pip install coveralls tox
+script:
+    - tox
 after_success:
     - coveralls
 sudo: false
 cache:
     directories:
         - $HOME/.cache/pip
         - $HOME/.pre-commit
+addons:
+    apt:
+        sources:
+            - deadsnakes
+        packages:
+            - python3.5-dev

README.md

@@ -6,6 +6,10 @@ setuptools-golang
 
 A setuptools extension for building cpython extensions written in golang.
 
+## Requirements
+
+This requires golang >= 1.5 to be installed on your system.
+
 ## Usage
 
 Add `setuptools-golang` to the `setup_requires` in your setup.py and

requirements-dev.txt

@@ -1,4 +1,4 @@
 -e .
-coverage
+coverage-enable-subprocess
 pre-commit
 pytest

setuptools_golang.py

@@ -1,11 +1,110 @@
-from setuptools.command.build_ext import build_ext
+from __future__ import print_function
+from __future__ import unicode_literals
 
+import distutils.sysconfig
+import os
+import pipes
+import subprocess
+import sys
 
-class BuildExtGolang(build_ext):
-    pass
+from setuptools.command.build_ext import build_ext as _build_ext
+
+
+PYPY = '__pypy__' in sys.builtin_module_names
+
+
+def _get_ldflags_pypy():
+    if PYPY:  # pragma: no cover (pypy only)
+        return '-L{} -lpypy-c'.format(
+            os.path.dirname(os.path.realpath(sys.executable)),
+        )
+    else:
+        return None
+
+
+def _get_ldflags_pkg_config():
+    try:
+        return subprocess.check_output((
+            'pkg-config', '--libs',
+            'python-{}.{}'.format(*sys.version_info[:2]),
+        )).decode('UTF-8').strip()
+    except (subprocess.CalledProcessError, OSError):
+        return None
+
+
+def _get_ldflags_bldlibrary():
+    return distutils.sysconfig.get_config_var('BLDLIBRARY')
+
+
+def _get_ldflags():
+    for func in (
+            _get_ldflags_pypy,
+            _get_ldflags_pkg_config,
+            _get_ldflags_bldlibrary,
+    ):
+        ret = func()
+        if ret is not None:
+            return ret
+    else:
+        raise AssertionError('Could not determine ldflags!')
+
+
+def _print_cmd(env, cmd):
+    envparts = [
+        '{}={}'.format(k, pipes.quote(v))
+        for k, v in sorted(tuple(env.items()))
+    ]
+    print(
+        '$ {}'.format(' '.join(envparts + [pipes.quote(p) for p in cmd])),
+        file=sys.stderr,
+    )
+
+
+class build_ext(_build_ext):
+    def build_extension(self, ext):
+        # If there are no .go files then the parent should handle this
+        if not any(source.endswith('.go') for source in ext.sources):
+            return _build_ext.build_extension(self, ext)
+
+        for source in ext.sources:
+            if not os.path.exists(source):
+                raise IOError(
+                    'Error building extension `{}`: {} does not exist'.format(
+                        ext.name, source,
+                    ),
+                )
+
+        # Passing non-.go files to `go build` results in a failure
+        # Passing only .go files to `go build` causes it to ignore C files
+        # So we'll set our cwd to the root of the files and go from there!
+        source_dirs = {os.path.dirname(src) for src in ext.sources}
+        if len(source_dirs) != 1:
+            raise IOError(
+                'Error building extension `{}`: '
+                'Cannot compile across directories: {}'.format(
+                    ext.name, ' '.join(sorted(source_dirs)),
+                )
+            )
+        source_dir, = source_dirs
+        source_dir = os.path.abspath(source_dir)
+
+        env = {
+            'CGO_CFLAGS': ' '.join(
+                '-I{}'.format(p) for p in self.compiler.include_dirs
+            ),
+            'CGO_LDFLAGS': _get_ldflags(),
+        }
+        cmd = (
+            'go', 'build', '-buildmode=c-shared',
+            '-o', os.path.abspath(self.get_ext_fullpath(ext.name)),
+        )
+        _print_cmd(env, cmd)
+        subprocess.check_call(
+            cmd, cwd=source_dir, env=dict(os.environ, **env),
+        )
 
 
 def set_build_ext(dist, attr, value):
     if not value:
         return
-    dist.cmdclass['build_ext'] = BuildExtGolang
+    dist.cmdclass['build_ext'] = build_ext

setuptools_golang_test.py

@@ -1,15 +1,152 @@
+from __future__ import unicode_literals
+
+import collections
+import os
+import subprocess
+import sys
+
+import pytest
 from setuptools.dist import Distribution
 
 import setuptools_golang
 
 
+xfailif_pypy = pytest.mark.xfail(
+    setuptools_golang.PYPY, reason='pypy is a special snowflake',
+)
+
+
+@pytest.fixture(autouse=True, scope='session')
+def enable_coverage_subprocesses():
+    here = os.path.dirname(os.path.abspath(__file__))
+    os.environ['TOP'] = here
+    os.environ['COVERAGE_PROCESS_START'] = os.path.join(here, '.coveragerc')
+
+
+def auto_namedtuple(**kwargs):
+    return collections.namedtuple('auto_namedtuple', kwargs.keys())(**kwargs)
+
+
+def run(*cmd, **kwargs):
+    returncode = kwargs.pop('returncode', 0)
+    proc = subprocess.Popen(cmd, **kwargs)
+    out, err = proc.communicate()
+    out = out.decode('UTF-8') if out is not None else None
+    err = err.decode('UTF-8') if err is not None else None
+    if returncode is not None:
+        if proc.returncode != returncode:
+            raise AssertionError(
+                '{!r} returned {} (expected {})\nout:\n{}err:\n{}'.format(
+                    cmd, proc.returncode, returncode, out, err,
+                )
+            )
+    return auto_namedtuple(returncode=proc.returncode, out=out, err=err)
+
+
+def run_output(*cmd, **kwargs):
+    return run(*cmd, stdout=subprocess.PIPE, **kwargs).out
+
+
 def test_sets_cmdclass():
     dist = Distribution()
     setuptools_golang.set_build_ext(dist, 'build_golang', True)
-    assert dist.cmdclass['build_ext'] == setuptools_golang.BuildExtGolang
+    assert dist.cmdclass['build_ext'] == setuptools_golang.build_ext
 
 
 def test_sets_cmdclass_value_falsey():
     dist = Distribution()
     setuptools_golang.set_build_ext(dist, 'build_golang', False)
-    assert dist.cmdclass.get('build_ext') != setuptools_golang.BuildExtGolang
+    assert dist.cmdclass.get('build_ext') != setuptools_golang.build_ext
+
+
+GET_LDFLAGS = (
+    'import distutils.spawn;'
+    "print(bool(distutils.spawn.find_executable('pkg-config')));"
+    'import setuptools_golang;'
+    'print(setuptools_golang._get_ldflags());'
+)
+
+
+@xfailif_pypy
+def test_from_pkg_config():
+    output = run_output(sys.executable, '-c', GET_LDFLAGS)
+    assert output.startswith('True\n')
+    assert '-lpython' in output
+
+
+@xfailif_pypy
+def test_no_pkg_config():
+    # Blank PATH so we don't have pkg-config
+    env = dict(os.environ, PATH='')
+    output = run_output(sys.executable, '-c', GET_LDFLAGS, env=env)
+    assert output.startswith('False\n')
+    assert '-lpython' in output
+
+
+@pytest.yield_fixture(scope='session')
+def venv(tmpdir_factory):
+    """A shared virtualenv fixture, be careful not to install two of the same
+    package into this -- or sadness...
+    """
+    venv = tmpdir_factory.mktemp('venv').join('venv')
+    pip = venv.join('bin/pip').strpath
+    python = venv.join('bin/python').strpath
+    # Make sure this virtualenv has the same executable
+    run('virtualenv', venv.strpath, '-p', sys.executable)
+    # Install this so we can get coverage
+    run(pip, 'install', 'coverage-enable-subprocess')
+    # Install us!
+    run(pip, 'install', '-e', '.')
+    yield auto_namedtuple(venv=venv, pip=pip, python=python)
+
+
+SUM = 'import {0}; print({0}.sum(1, 2))'
+
+
+@pytest.mark.parametrize(
+    ('pkg', 'mod'),
+    (
+        ('testing/sum', 'sum'),
+        ('testing/sum_pure_go', 'sum_pure_go'),
+        ('testing/sum_sub_package', 'sum_sub_package.sum'),
+    ),
+)
+def test_sum_integration(venv, pkg, mod):
+    run(venv.pip, 'install', '-v', pkg)
+    out = run_output(venv.python, '-c', SUM.format(mod))
+    assert out == '3\n'
+
+
+HELLO_WORLD = 'import project_with_c; print(project_with_c.hello_world())'
+
+
+def test_integration_project_with_c(venv):
+    test_sum_integration(
+        venv, 'testing/project_with_c', 'project_with_c_sum.sum',
+    )
+    out = run_output(venv.python, '-c', HELLO_WORLD)
+    assert out == 'hello world\n'
+
+
+def test_integration_notfound(venv):
+    ret = run(
+        venv.pip, 'install', 'testing/notfound',
+        returncode=None, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
+    )
+    assert ret.returncode != 0
+    assert (
+        'Error building extension `notfound`: notfound.go does not exist' in
+        ret.out
+    )
+
+
+def test_integration_multidir(venv):
+    ret = run(
+        venv.pip, 'install', 'testing/multidir',
+        returncode=None, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
+    )
+    assert ret.returncode != 0
+    assert (
+        'Error building extension `multidir`: '
+        'Cannot compile across directories: dir1 dir2' in ret.out
+    )

testing/multidir/dir1/sum.go

@@ -0,0 +1,17 @@
+package main
+
+// #include <Python.h>
+// int PyArg_ParseTuple_ll(PyObject*, long*, long*);
+import "C"
+
+//export sum
+func sum(self *C.PyObject, args *C.PyObject) *C.PyObject {
+	var a C.long
+	var b C.long
+	if C.PyArg_ParseTuple_ll(args, &a, &b) == 0 {
+		return nil
+	}
+	return C.PyLong_FromLong(a + b)
+}
+
+func main() {}

testing/multidir/dir2/sum_support.go

@@ -0,0 +1,33 @@
+package main
+
+// #include <Python.h>
+//
+// PyObject* sum(PyObject* , PyObject*);
+//
+// int PyArg_ParseTuple_ll(PyObject* args, long* a, long* b) {
+//     return PyArg_ParseTuple(args, "ll", a, b);
+// }
+//
+// static struct PyMethodDef methods[] = {
+//     {"sum", (PyCFunction)sum, METH_VARARGS},
+//     {NULL, NULL}
+// };
+//
+// #if PY_MAJOR_VERSION >= 3
+// static struct PyModuleDef module = {
+//     PyModuleDef_HEAD_INIT,
+//     "sum_pure_go",
+//     NULL,
+//     -1,
+//     methods
+// };
+//
+// PyMODINIT_FUNC PyInit_sum_pure_go(void) {
+//     return PyModule_Create(&module);
+// }
+// #else
+// PyMODINIT_FUNC initsum_pure_go(void) {
+//     Py_InitModule3("sum_pure_go", methods, NULL);
+// }
+// #endif
+import "C"