C Extensions

Table of Contents

Simple setup.py
Customize CFLAGS
Doc String
Simple C Extension
Release the GIL
Acquire the GIL
Get Reference Count
Parse Arguments
Calling Python Functions
Raise Exception
Customize Exception
Iterate a List
Iterate a Dictionary
Simple Class
Simple Class with Members and Methods
Simplie Class with Getter and Setter
Inherit from Other Class
Run a Python Command
Run a Python File
Import a Python Module
Import everything of a Module
Access Attributes
Performance of C Extension
Performance of ctypes
ctypes Error handling

Simple setup.py

from distutils.core import setup, Extension

ext = Extension('foo', sources=['foo.c'])
setup(name="Foo", version="1.0", ext_modules=[ext])

Customize CFLAGS

import sysconfig
from distutils.core import setup, Extension

cflags = sysconfig.get_config_var("CFLAGS")

extra_compile_args = cflags.split()
extra_compile_args += ["-Wextra"]

ext = Extension(
    "foo", ["foo.c"],
    extra_compile_args=extra_compile_args
)

setup(name="foo", version="1.0", ext_modules=[ext])

Doc String

PyDoc_STRVAR(doc_mod, "Module document\n");
PyDoc_STRVAR(doc_foo, "foo() -> None\n\nFoo doc");

static PyMethodDef methods[] = {
    {"foo", (PyCFunction)foo, METH_NOARGS, doc_foo},
    {NULL, NULL, 0, NULL}
};

static struct PyModuleDef module = {
    .m_base    = PyModuleDef_HEAD_INIT,
    .m_name    = "Foo",
    .m_doc     = doc_mod,
    .m_size    = -1,
    .m_methods = methods
};

Simple C Extension

foo.c

#include <Python.h>

PyDoc_STRVAR(doc_mod, "Module document\n");
PyDoc_STRVAR(doc_foo, "foo() -> None\n\nFoo doc");

static PyObject* foo(PyObject* self)
{
    PyObject* s = PyUnicode_FromString("foo");
    PyObject_Print(s, stdout, 0);
    Py_RETURN_NONE;
}

static PyMethodDef methods[] = {
    {"foo", (PyCFunction)foo, METH_NOARGS, doc_foo},
    {NULL, NULL, 0, NULL}
};

static struct PyModuleDef module = {
    PyModuleDef_HEAD_INIT, "Foo", doc_mod, -1, methods
};

PyMODINIT_FUNC PyInit_foo(void)
{
    return PyModule_Create(&module);
}

output:

$ python setup.py -q build
$ python setup.py -q install
$ python -c "import foo; foo.foo()"
'foo'

Release the GIL

#include <Python.h>

static PyObject* foo(PyObject* self)
{
    Py_BEGIN_ALLOW_THREADS
    sleep(3);
    Py_END_ALLOW_THREADS
    Py_RETURN_NONE;
}

static PyMethodDef methods[] = {
    {"foo", (PyCFunction)foo, METH_NOARGS, NULL},
    {NULL, NULL, 0, NULL}
};

static struct PyModuleDef module = {
    PyModuleDef_HEAD_INIT, "Foo", NULL, -1, methods
};

PyMODINIT_FUNC PyInit_foo(void)
{
    return PyModule_Create(&module);
}

output:

$ python setup.py -q build
$ python setup.py -q install
$ python -c "
> import threading
> import foo
> from datetime import datetime
> def f(n):
>     now = datetime.now()
>     print(f'{now}: thread {n}')
>     foo.foo()
> ts = [threading.Thread(target=f, args=(n,)) for n in range(3)]
> [t.start() for t in ts]
> [t.join() for t in ts]"
2018-11-04 20:15:34.860454: thread 0
2018-11-04 20:15:34.860592: thread 1
2018-11-04 20:15:34.860705: thread 2

In C extension, blocking I/O should be inserted into a block which is wrapped by Py_BEGIN_ALLOW_THREADS and Py_END_ALLOW_THREADS for releasing the GIL temporarily; Otherwise, a blocking I/O operation has to wait until previous operation finish. For example

#include <Python.h>

static PyObject* foo(PyObject* self)
{
    sleep(3);
    Py_RETURN_NONE;
}

static PyMethodDef methods[] = {
    {"foo", (PyCFunction)foo, METH_NOARGS, NULL},
    {NULL, NULL, 0, NULL}
};

static struct PyModuleDef module = {
    PyModuleDef_HEAD_INIT, "Foo", NULL, -1, methods
};

PyMODINIT_FUNC PyInit_foo(void)
{
    return PyModule_Create(&module);
}

output:

$ python -c "
> import threading
> import foo
> from datetime import datetime
> def f(n):
>     now = datetime.now()
>     print(f'{now}: thread {n}')
>     foo.foo()
> ts = [threading.Thread(target=f, args=(n,)) for n in range(3)]
> [t.start() for t in ts]
> [t.join() for t in ts]"
2018-11-04 20:16:44.055932: thread 0
2018-11-04 20:16:47.059718: thread 1
2018-11-04 20:16:50.063579: thread 2

Warning

The GIL can only be safely released when there is NO Python C API functions between Py_BEGIN_ALLOW_THREADS and Py_END_ALLOW_THREADS.

Acquire the GIL

#include <pthread.h>
#include <Python.h>

typedef struct {
    PyObject *sec;
    PyObject *py_callback;
} foo_args;

void *
foo_thread(void *args)
{
    long n = -1;
    PyObject *rv = NULL, *sec = NULL,* py_callback = NULL;
    foo_args *a = NULL;

    if (!args)
        return NULL;

    a = (foo_args *)args;
    sec = a->sec;
    py_callback = a->py_callback;

    n = PyLong_AsLong(sec);
    if ((n == -1) && PyErr_Occurred()) {
        return NULL;
    }

    sleep(n);  // slow task

    // acquire the GIL
    PyGILState_STATE state = PyGILState_Ensure();
    rv = PyObject_CallFunction(py_callback, "s", "Awesome Python!");
    // release the GIL
    PyGILState_Release(state);
    Py_XDECREF(rv);
    return NULL;
}

static PyObject *
foo(PyObject *self, PyObject *args)
{
    long i = 0, n = 0;
    pthread_t *arr = NULL;
    PyObject *py_callback = NULL;
    PyObject *sec = NULL, *num = NULL;
    PyObject *rv = NULL;
    foo_args a = {};

    if (!PyArg_ParseTuple(args, "OOO:callback", &num, &sec, &py_callback))
        return NULL;

    // allow releasing GIL
    Py_BEGIN_ALLOW_THREADS

    if (!PyLong_Check(sec) || !PyLong_Check(num)) {
        PyErr_SetString(PyExc_TypeError, "should be int");
        goto error;
    }

    if (!PyCallable_Check(py_callback)) {
        PyErr_SetString(PyExc_TypeError, "should be callable");
        goto error;
    }

    n = PyLong_AsLong(num);
    if (n == -1 && PyErr_Occurred())
        goto error;

    arr = (pthread_t *)PyMem_RawCalloc(n, sizeof(pthread_t));
    if (!arr)
        goto error;

    a.sec = sec;
    a.py_callback = py_callback;
    for (i = 0; i < n; i++) {
        if (pthread_create(&arr[i], NULL, foo_thread, &a)) {
            PyErr_SetString(PyExc_TypeError, "create a thread failed");
            goto error;
        }
    }

    for (i = 0; i < n; i++) {
        if (pthread_join(arr[i], NULL)) {
            PyErr_SetString(PyExc_TypeError, "thread join failed");
            goto error;
        }
    }
    Py_XINCREF(Py_None);
    rv = Py_None;
error:
    PyMem_RawFree(arr);
    Py_XDECREF(sec);
    Py_XDECREF(num);
    Py_XDECREF(py_callback);
    // restore GIL
    Py_END_ALLOW_THREADS
    return rv;
}

static PyMethodDef methods[] = {
    {"foo", (PyCFunction)foo, METH_VARARGS, NULL},
    {NULL, NULL, 0, NULL}
};

static struct PyModuleDef module = {
    PyModuleDef_HEAD_INIT, "foo", NULL, -1, methods
};

PyMODINIT_FUNC PyInit_foo(void)
{
    return PyModule_Create(&module);
}

output:

$ python setup.py -q build
$ python setup.py -q install
$ pyton -q
>>> import foo
>>> from datetime import datetime
>>> def cb(s):
...     now = datetime.now()
...     print(f'{now}: {s}')
...
>>> foo.foo(3, 1, cb)
2018-11-05 09:33:50.642543: Awesome Python!
2018-11-05 09:33:50.642634: Awesome Python!
2018-11-05 09:33:50.642672: Awesome Python!

If threads are created from C/C++, those threads do not hold the GIL. Without acquiring the GIL, the interpreter cannot access Python functions safely. For example

void *
foo_thread(void *args)
{
    ...
    // without acquiring the GIL
    rv = PyObject_CallFunction(py_callback, "s", "Awesome Python!");
    Py_XDECREF(rv);
    return NULL;
}

output:

>>> import foo
>>> from datetime import datetime
>>> def cb(s):
...     now = datetime.now()
...     print(f"{now}: {s}")
...
>>> foo.foo(1, 1, cb)
[2]    8590 segmentation fault  python -q

Warning

In order to call python function safely, we can simply warp Python Functions between PyGILState_Ensure and PyGILState_Release in C extension code.

PyGILState_STATE state = PyGILState_Ensure();
// Perform Python actions
result = PyObject_CallFunction(callback)
// Error handling
PyGILState_Release(state);

Get Reference Count

#include <Python.h>

static PyObject *
getrefcount(PyObject *self, PyObject *a)
{
    return PyLong_FromSsize_t(Py_REFCNT(a));
}

static PyMethodDef methods[] = {
    {"getrefcount", (PyCFunction)getrefcount, METH_O, NULL},
    {NULL, NULL, 0, NULL}
};

static struct PyModuleDef module = {
    PyModuleDef_HEAD_INIT, "foo", NULL, -1, methods
};

PyMODINIT_FUNC PyInit_foo(void)
{
    return PyModule_Create(&module);
}

output:

$ python setup.py -q build
$ python setup.py -q install
$ python -q
>>> import sys
>>> import foo
>>> l = [1, 2, 3]
>>> sys.getrefcount(l[0])
104
>>> foo.getrefcount(l[0])
104
>>> i = l[0]
>>> sys.getrefcount(l[0])
105
>>> foo.getrefcount(l[0])
105

Parse Arguments

#include <Python.h>

static PyObject *
foo(PyObject *self)
{
    Py_RETURN_NONE;
}

static PyObject *
bar(PyObject *self, PyObject *arg)
{
    return Py_BuildValue("O", arg);
}

static PyObject *
baz(PyObject *self, PyObject *args)
{
    PyObject *x = NULL, *y = NULL;
    if (!PyArg_ParseTuple(args, "OO", &x, &y)) {
        return NULL;
    }
    return Py_BuildValue("OO", x, y);
}

static PyObject *
qux(PyObject *self, PyObject *args, PyObject *kwargs)
{
    static char *keywords[] = {"x", "y", NULL};
    PyObject *x = NULL, *y = NULL;
    if (!PyArg_ParseTupleAndKeywords(args, kwargs,
                                     "O|O", keywords,
                                     &x, &y))
    {
        return NULL;
    }
    if (!y) {
        y = Py_None;
    }
    return Py_BuildValue("OO", x, y);
}

static PyMethodDef methods[] = {
    {"foo", (PyCFunction)foo, METH_NOARGS, NULL},
    {"bar", (PyCFunction)bar, METH_O, NULL},
    {"baz", (PyCFunction)baz, METH_VARARGS, NULL},
    {"qux", (PyCFunction)qux, METH_VARARGS | METH_KEYWORDS, NULL},
    {NULL, NULL, 0, NULL}
};

static struct PyModuleDef module = {
    PyModuleDef_HEAD_INIT, "foo", NULL, -1, methods
};

PyMODINIT_FUNC PyInit_foo(void)
{
    return PyModule_Create(&module);
}

output:

$ python setup.py -q build
$ python setup.py -q install
$ python -q
>>> import foo
>>> foo.foo()
>>> foo.bar(3.7)
3.7
>>> foo.baz(3, 7)
(3, 7)
>>> foo.qux(3, y=7)
(3, 7)
>>> foo.qux(x=3, y=7)
(3, 7)
>>> foo.qux(x=3)
(3, None)

Calling Python Functions

#include <Python.h>

static PyObject *
foo(PyObject *self, PyObject *args)
{
    PyObject *py_callback = NULL;
    PyObject *rv = NULL;

    if (!PyArg_ParseTuple(args, "O:callback", &py_callback))
        return NULL;

    if (!PyCallable_Check(py_callback)) {
        PyErr_SetString(PyExc_TypeError, "should be callable");
        return NULL;
    }

    // Make sure we own the GIL
    PyGILState_STATE state = PyGILState_Ensure();
    // similar to py_callback("Awesome Python!")
    rv = PyObject_CallFunction(py_callback, "s", "Awesome Python!");
    // Restore previous GIL state
    PyGILState_Release(state);
    return rv;
}

static PyMethodDef methods[] = {
    {"foo", (PyCFunction)foo, METH_VARARGS, NULL},
    {NULL, NULL, 0, NULL}
};

static struct PyModuleDef module = {
    PyModuleDef_HEAD_INIT, "foo", NULL, -1, methods
};

PyMODINIT_FUNC PyInit_foo(void)
{
    return PyModule_Create(&module);
}

output:

$ python setup.py -q build
$ python setup.py -q install
$ python -c "import foo; foo.foo(print)"
Awesome Python!

Raise Exception

#include <Python.h>

PyDoc_STRVAR(doc_mod, "Module document\n");
PyDoc_STRVAR(doc_foo, "foo() -> None\n\nFoo doc");

static PyObject*
foo(PyObject* self)
{
    // raise NotImplementedError
    PyErr_SetString(PyExc_NotImplementedError, "Not implemented");
    return NULL;
}

static PyMethodDef methods[] = {
    {"foo", (PyCFunction)foo, METH_NOARGS, doc_foo},
    {NULL, NULL, 0, NULL}
};

static struct PyModuleDef module = {
    PyModuleDef_HEAD_INIT, "Foo", doc_mod, -1, methods
};

PyMODINIT_FUNC PyInit_foo(void)
{
    return PyModule_Create(&module);
}

output:

$ python setup.py -q build
$ python setup.py -q install
$ python -c "import foo; foo.foo(print)"
$ python -c "import foo; foo.foo()"
Traceback (most recent call last):
  File "<string>", line 1, in <module>
NotImplementedError: Not implemented

Customize Exception

#include <stdio.h>
#include <Python.h>

static PyObject *FooError;

PyDoc_STRVAR(doc_foo, "foo() -> void\n\n"
    "Equal to the following example:\n\n"
    "def foo():\n"
    "    raise FooError(\"Raise exception in C\")"
);

static PyObject *
foo(PyObject *self __attribute__((unused)))
{
    PyErr_SetString(FooError, "Raise exception in C");
    return NULL;
}

static PyMethodDef methods[] = {
    {"foo", (PyCFunction)foo, METH_NOARGS, doc_foo},
    {NULL, NULL, 0, NULL}
};

static struct PyModuleDef module = {
    PyModuleDef_HEAD_INIT, "foo", "doc", -1, methods
};

PyMODINIT_FUNC PyInit_foo(void)
{
    PyObject *m = NULL;
    m = PyModule_Create(&module);
    if (!m) return NULL;

    FooError = PyErr_NewException("foo.FooError", NULL, NULL);
    Py_INCREF(FooError);
    PyModule_AddObject(m, "FooError", FooError);
    return m;
}

output:

$ python setup.py -q build
$ python setup.py -q install
$ python -c "import foo; foo.foo()"
Traceback (most recent call last):
  File "<string>", line 1, in <module>
foo.FooError: Raise exception in C

Iterate a List

#include <Python.h>

#define PY_PRINTF(o) \
    PyObject_Print(o, stdout, 0); printf("\n");

static PyObject *
iter_list(PyObject *self, PyObject *args)
{
    PyObject *list = NULL, *item = NULL, *iter = NULL;
    PyObject *result = NULL;

    if (!PyArg_ParseTuple(args, "O", &list))
        goto error;

    if (!PyList_Check(list))
        goto error;

    // Get iterator
    iter = PyObject_GetIter(list);
    if (!iter)
        goto error;

    // for i in arr: print(i)
    while ((item = PyIter_Next(iter)) != NULL) {
        PY_PRINTF(item);
        Py_XDECREF(item);
    }

    Py_XINCREF(Py_None);
    result = Py_None;
error:
    Py_XDECREF(iter);
    return result;
}

static PyMethodDef methods[] = {
    {"iter_list", (PyCFunction)iter_list, METH_VARARGS, NULL},
    {NULL, NULL, 0, NULL}
};

static struct PyModuleDef module = {
    PyModuleDef_HEAD_INIT, "foo", NULL, -1, methods
};

PyMODINIT_FUNC PyInit_foo(void)
{
    return PyModule_Create(&module);
}

output:

$ python setup.py -q build
$ python setup.py -q install
$ python -c "import foo; foo.iter_list([1,2,3])"
1
2
3

Iterate a Dictionary

#include <Python.h>

#define PY_PRINTF(o) \
    PyObject_Print(o, stdout, 0); printf("\n");

static PyObject *
iter_dict(PyObject *self, PyObject *args)
{
    PyObject *dict = NULL;
    PyObject *key = NULL, *val = NULL;
    PyObject *o = NULL, *result = NULL;
    Py_ssize_t pos = 0;

    if (!PyArg_ParseTuple(args, "O", &dict))
        goto error;

    // for k, v in d.items(): print(f"({k}, {v})")
    while (PyDict_Next(dict, &pos, &key, &val)) {
        o = PyUnicode_FromFormat("(%S, %S)", key, val);
        if (!o) continue;
        PY_PRINTF(o);
        Py_XDECREF(o);
    }

    Py_INCREF(Py_None);
    result = Py_None;
error:
    return result;
}

static PyMethodDef methods[] = {
    {"iter_dict", (PyCFunction)iter_dict, METH_VARARGS, NULL},
    {NULL, NULL, 0, NULL}
};

static struct PyModuleDef module = {
    PyModuleDef_HEAD_INIT, "foo", NULL, -1, methods
};

PyMODINIT_FUNC PyInit_foo(void)
{
    return PyModule_Create(&module);
}

output:

$ python setup.py -q build
$ python setup.py -q install
$ python -c "import foo; foo.iter_dict({'k': 'v'})"
'(k, v)'

Simple Class

#include <Python.h>

typedef struct {
    PyObject_HEAD
} FooObject;

/* calss Foo(object): pass */

static PyTypeObject FooType = {
    PyVarObject_HEAD_INIT(NULL, 0)
    .tp_name = "foo.Foo",
    .tp_doc = "Foo objects",
    .tp_basicsize = sizeof(FooObject),
    .tp_itemsize = 0,
    .tp_flags = Py_TPFLAGS_DEFAULT,
    .tp_new = PyType_GenericNew
};

static PyModuleDef module = {
    PyModuleDef_HEAD_INIT,
    .m_name = "foo",
    .m_doc = "module foo",
    .m_size = -1
};

PyMODINIT_FUNC
PyInit_foo(void)
{
    PyObject *m = NULL;
    if (PyType_Ready(&FooType) < 0)
        return NULL;
    if ((m = PyModule_Create(&module)) == NULL)
        return NULL;
    Py_XINCREF(&FooType);
    PyModule_AddObject(m, "Foo", (PyObject *) &FooType);
    return m;
}

output:

$ python setup.py -q build
$ python setup.py -q install
$ python -q
>>> import foo
>>> print(type(foo.Foo))
<class 'type'>
>>> o = foo.Foo()
>>> print(type(o))
<class 'foo.Foo'>
>>> class Foo(object): ...
...
>>> print(type(Foo))
<class 'type'>
>>> o = Foo()
>>> print(type(o))
<class '__main__.Foo'>

Simple Class with Members and Methods

#include <Python.h>
#include <structmember.h>

/*
 * class Foo:
 *     def __new__(cls, *a, **kw):
 *         foo_obj = object.__new__(cls)
 *         foo_obj.foo = ""
 *         foo_obj.bar = ""
 *         return foo_obj
 *
 *     def __init__(self, foo, bar):
 *         self.foo = foo
 *         self.bar = bar
 *
 *     def fib(self, n):
 *         if n < 2:
 *             return n
 *         return self.fib(n - 1) + self.fib(n - 2)
 */

typedef struct {
    PyObject_HEAD
    PyObject *foo;
    PyObject *bar;
} FooObject;

static void
Foo_dealloc(FooObject *self)
{
    Py_XDECREF(self->foo);
    Py_XDECREF(self->bar);
    Py_TYPE(self)->tp_free((PyObject *) self);
}

static PyObject *
Foo_new(PyTypeObject *type, PyObject *args, PyObject *kw)
{
    int rc = -1;
    FooObject *self = NULL;
    self = (FooObject *) type->tp_alloc(type, 0);

    if (!self) goto error;

    /* allocate attributes */
    self->foo = PyUnicode_FromString("");
    if (self->foo == NULL) goto error;

    self->bar = PyUnicode_FromString("");
    if (self->bar == NULL) goto error;

    rc = 0;
error:
    if (rc < 0) {
        Py_XDECREF(self->foo);
        Py_XINCREF(self->bar);
        Py_XDECREF(self);
    }
    return (PyObject *) self;
}

static int
Foo_init(FooObject *self, PyObject *args, PyObject *kw)
{
    int rc = -1;
    static char *keywords[] = {"foo", "bar", NULL};
    PyObject *foo = NULL, *bar = NULL, *ptr = NULL;

    if (!PyArg_ParseTupleAndKeywords(args, kw,
                                    "|OO", keywords,
                                    &foo, &bar))
    {
        goto error;
    }

    if (foo) {
        ptr = self->foo;
        Py_INCREF(foo);
        self->foo = foo;
        Py_XDECREF(ptr);
    }

    if (bar) {
        ptr = self->bar;
        Py_INCREF(bar);
        self->bar = bar;
        Py_XDECREF(ptr);
    }
    rc = 0;
error:
    return rc;
}

static unsigned long
fib(unsigned long n)
{
    if (n < 2) return n;
    return fib(n - 1) + fib(n - 2);
}

static PyObject *
Foo_fib(FooObject *self, PyObject *args)
{
    unsigned long n = 0;
    if (!PyArg_ParseTuple(args, "k", &n)) return NULL;
    return PyLong_FromUnsignedLong(fib(n));
}

static PyMemberDef Foo_members[] = {
    {"foo", T_OBJECT_EX, offsetof(FooObject, foo), 0, NULL},
    {"bar", T_OBJECT_EX, offsetof(FooObject, bar), 0, NULL}
};

static PyMethodDef Foo_methods[] = {
    {"fib", (PyCFunction)Foo_fib, METH_VARARGS | METH_KEYWORDS, NULL},
    {NULL, NULL, 0, NULL}
};

static PyTypeObject FooType = {
    PyVarObject_HEAD_INIT(NULL, 0)
    .tp_name = "foo.Foo",
    .tp_doc = "Foo objects",
    .tp_basicsize = sizeof(FooObject),
    .tp_itemsize = 0,
    .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE,
    .tp_new = Foo_new,
    .tp_init = (initproc) Foo_init,
    .tp_dealloc = (destructor) Foo_dealloc,
    .tp_members = Foo_members,
    .tp_methods = Foo_methods
};

static PyModuleDef module = {
    PyModuleDef_HEAD_INIT, "foo", NULL, -1, NULL
};

PyMODINIT_FUNC
PyInit_foo(void)
{
    PyObject *m = NULL;
    if (PyType_Ready(&FooType) < 0)
        return NULL;
    if ((m = PyModule_Create(&module)) == NULL)
        return NULL;
    Py_XINCREF(&FooType);
    PyModule_AddObject(m, "Foo", (PyObject *) &FooType);
    return m;
}

output:

$ python setup.py -q build
$ python setup.py -q install
$ python -q
>>> import foo
>>> o = foo.Foo('foo', 'bar')
>>> o.foo
'foo'
>>> o.bar
'bar'
>>> o.fib(10)
55

Simplie Class with Getter and Setter

#include <Python.h>

/*
 * class Foo:
 *     def __new__(cls, *a, **kw):
 *         foo_obj = object.__new__(cls)
 *         foo_obj._foo = ""
 *         return foo_obj
 *
 *     def __init__(self, foo=None):
 *         if foo and isinstance(foo, 'str'):
 *             self._foo = foo
 *
 *     @property
 *     def foo(self):
 *         return self._foo
 *
 *     @foo.setter
 *     def foo(self, value):
 *         if not value or not isinstance(value, str):
 *             raise TypeError("value should be unicode")
 *         self._foo = value
 */

typedef struct {
    PyObject_HEAD
    PyObject *foo;
} FooObject;

static void
Foo_dealloc(FooObject *self)
{
    Py_XDECREF(self->foo);
    Py_TYPE(self)->tp_free((PyObject *) self);
}

static PyObject *
Foo_new(PyTypeObject *type, PyObject *args, PyObject *kw)
{
    int rc = -1;
    FooObject *self = NULL;
    self = (FooObject *) type->tp_alloc(type, 0);

    if (!self) goto error;

    /* allocate attributes */
    self->foo = PyUnicode_FromString("");
    if (self->foo == NULL) goto error;

    rc = 0;
error:
    if (rc < 0) {
        Py_XDECREF(self->foo);
        Py_XDECREF(self);
    }
    return (PyObject *) self;
}

static int
Foo_init(FooObject *self, PyObject *args, PyObject *kw)
{
    int rc = -1;
    static char *keywords[] = {"foo", NULL};
    PyObject *foo = NULL, *ptr = NULL;

    if (!PyArg_ParseTupleAndKeywords(args, kw,
                                    "|O", keywords,
                                    &foo))
    {
        goto error;
    }

    if (foo && PyUnicode_Check(foo)) {
        ptr = self->foo;
        Py_INCREF(foo);
        self->foo = foo;
        Py_XDECREF(ptr);
    }

    rc = 0;
error:
    return rc;
}

static PyObject *
Foo_getfoo(FooObject *self, void *closure)
{
    Py_INCREF(self->foo);
    return self->foo;
}

static int
Foo_setfoo(FooObject *self, PyObject *value, void *closure)
{
    int rc = -1;

    if (!value || !PyUnicode_Check(value)) {
        PyErr_SetString(PyExc_TypeError, "value should be unicode");
        goto error;
    }
    Py_INCREF(value);
    Py_XDECREF(self->foo);
    self->foo = value;
    rc = 0;
error:
    return rc;
}

static PyGetSetDef Foo_getsetters[] = {
    {"foo", (getter)Foo_getfoo, (setter)Foo_setfoo}
};

static PyTypeObject FooType = {
    PyVarObject_HEAD_INIT(NULL, 0)
    .tp_name = "foo.Foo",
    .tp_doc = "Foo objects",
    .tp_basicsize = sizeof(FooObject),
    .tp_itemsize = 0,
    .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE,
    .tp_new = Foo_new,
    .tp_init = (initproc) Foo_init,
    .tp_dealloc = (destructor) Foo_dealloc,
    .tp_getset = Foo_getsetters,
};

static PyModuleDef module = {
    PyModuleDef_HEAD_INIT, "foo", NULL, -1, NULL
};

PyMODINIT_FUNC
PyInit_foo(void)
{
    PyObject *m = NULL;
    if (PyType_Ready(&FooType) < 0)
        return NULL;
    if ((m = PyModule_Create(&module)) == NULL)
        return NULL;
    Py_XINCREF(&FooType);
    PyModule_AddObject(m, "Foo", (PyObject *) &FooType);
    return m;
}

output:

$ python setup.py -q build
$ python setup.py -q install
$ python -q
>>> import foo
>>> o = foo.Foo()
>>> o.foo
''
>>> o.foo = "foo"
>>> o.foo
'foo'
>>> o.foo = None
Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
TypeError: value should be unicode

Inherit from Other Class

#include <Python.h>
#include <structmember.h>

/*
 * class Foo:
 *     def __new__(cls, *a, **kw):
 *         foo_obj = object.__new__(cls)
 *         foo_obj.foo = ""
 *         return foo_obj
 *
 *     def __init__(self, foo):
 *         self.foo = foo
 *
 *     def fib(self, n):
 *         if n < 2:
 *             return n
 *         return self.fib(n - 1) + self.fib(n - 2)
 */

/* FooObject */

typedef struct {
    PyObject_HEAD
    PyObject *foo;
} FooObject;

static void
Foo_dealloc(FooObject *self)
{
    Py_XDECREF(self->foo);
    Py_TYPE(self)->tp_free((PyObject *) self);
}

static PyObject *
Foo_new(PyTypeObject *type, PyObject *args, PyObject *kw)
{
    int rc = -1;
    FooObject *self = NULL;
    self = (FooObject *) type->tp_alloc(type, 0);

    if (!self) goto error;

    /* allocate attributes */
    self->foo = PyUnicode_FromString("");
    if (self->foo == NULL) goto error;

    rc = 0;
error:
    if (rc < 0) {
        Py_XDECREF(self->foo);
        Py_XDECREF(self);
    }
    return (PyObject *) self;
}

static int
Foo_init(FooObject *self, PyObject *args, PyObject *kw)
{
    int rc = -1;
    static char *keywords[] = {"foo", NULL};
    PyObject *foo = NULL, *ptr = NULL;

    if (!PyArg_ParseTupleAndKeywords(args, kw, "|O", keywords, &foo)) {
        goto error;
    }

    if (foo) {
        ptr = self->foo;
        Py_INCREF(foo);
        self->foo = foo;
        Py_XDECREF(ptr);
    }
    rc = 0;
error:
    return rc;
}

static unsigned long
fib(unsigned long n)
{
    if (n < 2) return n;
    return fib(n - 1) + fib(n - 2);
}

static PyObject *
Foo_fib(FooObject *self, PyObject *args)
{
    unsigned long n = 0;
    if (!PyArg_ParseTuple(args, "k", &n)) return NULL;
    return PyLong_FromUnsignedLong(fib(n));
}

static PyMemberDef Foo_members[] = {
    {"foo", T_OBJECT_EX, offsetof(FooObject, foo), 0, NULL}
};

static PyMethodDef Foo_methods[] = {
    {"fib", (PyCFunction)Foo_fib, METH_VARARGS | METH_KEYWORDS, NULL},
    {NULL, NULL, 0, NULL}
};

static PyTypeObject FooType = {
    PyVarObject_HEAD_INIT(NULL, 0)
    .tp_name = "foo.Foo",
    .tp_doc = "Foo objects",
    .tp_basicsize = sizeof(FooObject),
    .tp_itemsize = 0,
    .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE,
    .tp_new = Foo_new,
    .tp_init = (initproc) Foo_init,
    .tp_dealloc = (destructor) Foo_dealloc,
    .tp_members = Foo_members,
    .tp_methods = Foo_methods
};

/*
 * class Bar(Foo):
 *     def __init__(self, bar):
 *         super().__init__(bar)
 *
 *     def gcd(self, a, b):
 *         while b:
 *             a, b = b, a % b
 *         return a
 */

/* BarObject */

typedef struct {
    FooObject super;
} BarObject;

static unsigned long
gcd(unsigned long a, unsigned long b)
{
    unsigned long t = 0;
    while (b) {
        t = b;
        b = a % b;
        a = t;
    }
    return a;
}

static int
Bar_init(FooObject *self, PyObject *args, PyObject *kw)
{
    return FooType.tp_init((PyObject *) self, args, kw);
}

static PyObject *
Bar_gcd(BarObject *self, PyObject *args)
{
    unsigned long a = 0, b = 0;
    if (!PyArg_ParseTuple(args, "kk", &a, &b)) return NULL;
    return PyLong_FromUnsignedLong(gcd(a, b));
}

static PyMethodDef Bar_methods[] = {
    {"gcd", (PyCFunction)Bar_gcd, METH_VARARGS, NULL},
    {NULL, NULL, 0, NULL}
};

static PyTypeObject BarType = {
    PyVarObject_HEAD_INIT(NULL, 0)
    .tp_name = "foo.Bar",
    .tp_doc = "Bar objects",
    .tp_basicsize = sizeof(BarObject),
    .tp_itemsize = 0,
    .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE,
    .tp_base = &FooType,
    .tp_init = (initproc) Bar_init,
    .tp_methods = Bar_methods
};

/* Module */

static PyModuleDef module = {
    PyModuleDef_HEAD_INIT, "foo", NULL, -1, NULL
};

PyMODINIT_FUNC
PyInit_foo(void)
{
    PyObject *m = NULL;
    if (PyType_Ready(&FooType) < 0)
        return NULL;
    if (PyType_Ready(&BarType) < 0)
        return NULL;
    if ((m = PyModule_Create(&module)) == NULL)
        return NULL;

    Py_XINCREF(&FooType);
    Py_XINCREF(&BarType);
    PyModule_AddObject(m, "Foo", (PyObject *) &FooType);
    PyModule_AddObject(m, "Bar", (PyObject *) &BarType);
    return m;
}

output:

$ python setup.py -q build
$ python setup.py -q install
$ python -q
>>> import foo
>>> bar = foo.Bar('bar')
>>> bar.foo
'bar'
>>> bar.fib(10)
55
>>> bar.gcd(3, 7)
1

Run a Python Command

#include <stdio.h>
#include <Python.h>

int
main(int argc, char *argv[])
{
    int rc = -1;
    Py_Initialize();
    rc = PyRun_SimpleString(argv[1]);
    Py_Finalize();
    return rc;
}

output:

$ clang `python3-config --cflags` -c foo.c -o foo.o
$ clang `python3-config --ldflags` foo.o -o foo
$ ./foo "print('Hello Python')"
Hello Python

Run a Python File

#include <stdio.h>
#include <Python.h>

int
main(int argc, char *argv[])
{
    int rc = -1, i = 0;
    wchar_t **argv_copy = NULL;
    const char *filename = NULL;
    FILE *fp = NULL;
    PyCompilerFlags cf = {.cf_flags = 0};

    filename = argv[1];
    fp = fopen(filename, "r");
    if (!fp)
        goto error;

    // copy argv
    argv_copy = PyMem_RawMalloc(sizeof(wchar_t*) * argc);
    if (!argv_copy)
        goto error;

    for (i = 0; i < argc; i++) {
        argv_copy[i] = Py_DecodeLocale(argv[i], NULL);
        if (argv_copy[i]) continue;
        fprintf(stderr, "Unable to decode the argument");
        goto error;
    }

    Py_Initialize();
    Py_SetProgramName(argv_copy[0]);
    PySys_SetArgv(argc, argv_copy);
    rc = PyRun_AnyFileExFlags(fp, filename, 0, &cf);

error:
    if (argv_copy) {
        for (i = 0; i < argc; i++)
            PyMem_RawFree(argv_copy[i]);
        PyMem_RawFree(argv_copy);
    }
    if (fp) fclose(fp);
    Py_Finalize();
    return rc;
}

output:

$ clang `python3-config --cflags` -c foo.c -o foo.o
$ clang `python3-config --ldflags` foo.o -o foo
$ echo "import sys; print(sys.argv)" > foo.py
$ ./foo foo.py arg1 arg2 arg3
['./foo', 'foo.py', 'arg1', 'arg2', 'arg3']

Import a Python Module

#include <stdio.h>
#include <Python.h>

#define PYOBJECT_CHECK(obj, label) \
    if (!obj) { \
        PyErr_Print(); \
        goto label; \
    }

int
main(int argc, char *argv[])
{
    int rc = -1;
    wchar_t *program = NULL;
    PyObject *json_module = NULL, *json_dict = NULL;
    PyObject *json_dumps = NULL;
    PyObject *dict = NULL;
    PyObject *result = NULL;

    program = Py_DecodeLocale(argv[0], NULL);
    if (!program) {
        fprintf(stderr, "unable to decode the program name");
        goto error;
    }

    Py_SetProgramName(program);
    Py_Initialize();

    // import json
    json_module = PyImport_ImportModule("json");
    PYOBJECT_CHECK(json_module, error);

    // json_dict = json.__dict__
    json_dict = PyModule_GetDict(json_module);
    PYOBJECT_CHECK(json_dict, error);

    // json_dumps = json.__dict__['dumps']
    json_dumps = PyDict_GetItemString(json_dict, "dumps");
    PYOBJECT_CHECK(json_dumps, error);

    // dict = {'foo': 'Foo', 'bar': 123}
    dict = Py_BuildValue("({sssi})", "foo", "Foo", "bar", 123);
    PYOBJECT_CHECK(dict, error);

    // result = json.dumps(dict)
    result = PyObject_CallObject(json_dumps, dict);
    PYOBJECT_CHECK(result, error);
    PyObject_Print(result, stdout, 0);
    printf("\n");
    rc = 0;

error:
    Py_XDECREF(result);
    Py_XDECREF(dict);
    Py_XDECREF(json_dumps);
    Py_XDECREF(json_dict);
    Py_XDECREF(json_module);

    PyMem_RawFree(program);
    Py_Finalize();
    return rc;
}

output:

$ clang `python3-config --cflags` -c foo.c -o foo.o
$ clang `python3-config --ldflags` foo.o -o foo
$ ./foo
'{"foo": "Foo", "bar": 123}'

Import everything of a Module

#include <stdio.h>
#include <Python.h>

#define PYOBJECT_CHECK(obj, label) \
    if (!obj) { \
        PyErr_Print(); \
        goto label; \
    }


int
main(int argc, char *argv[])
{
    int rc = -1;
    wchar_t *program = NULL;
    PyObject *main_module = NULL, *main_dict = NULL;
    PyObject *uname = NULL;
    PyObject *sysname = NULL;
    PyObject *result = NULL;

    program = Py_DecodeLocale(argv[0], NULL);
    if (!program) {
        fprintf(stderr, "unable to decode the program name");
        goto error;
    }

    Py_SetProgramName(program);
    Py_Initialize();

    // import __main__
    main_module = PyImport_ImportModule("__main__");
    PYOBJECT_CHECK(main_module, error);

    // main_dict = __main__.__dict__
    main_dict = PyModule_GetDict(main_module);
    PYOBJECT_CHECK(main_dict, error);

    // from os import *
    result = PyRun_String("from os import *",
                          Py_file_input,
                          main_dict,
                          main_dict);
    PYOBJECT_CHECK(result, error);
    Py_XDECREF(result);
    Py_XDECREF(main_dict);

    // uname = __main__.__dict__['uname']
    main_dict = PyModule_GetDict(main_module);
    PYOBJECT_CHECK(main_dict, error);

    // result = uname()
    uname = PyDict_GetItemString(main_dict, "uname");
    PYOBJECT_CHECK(uname, error);
    result = PyObject_CallObject(uname, NULL);
    PYOBJECT_CHECK(result, error);

    // sysname = result.sysname
    sysname = PyObject_GetAttrString(result, "sysname");
    PYOBJECT_CHECK(sysname, error);
    PyObject_Print(sysname, stdout, 0);
    printf("\n");

    rc = 0;
error:
    Py_XDECREF(sysname);
    Py_XDECREF(result);
    Py_XDECREF(uname);
    Py_XDECREF(main_dict);
    Py_XDECREF(main_module);

    PyMem_RawFree(program);
    Py_Finalize();
    return rc;
}

output:

$ clang `python3-config --cflags` -c foo.c -o foo.o
$ clang `python3-config --ldflags` foo.o -o foo
$ ./foo
'Darwin'

Access Attributes

#include <stdio.h>
#include <Python.h>

#define PYOBJECT_CHECK(obj, label) \
    if (!obj) { \
        PyErr_Print(); \
        goto label; \
    }

int
main(int argc, char *argv[])
{
    int rc = -1;
    wchar_t *program = NULL;
    PyObject *json_module = NULL;
    PyObject *json_dumps = NULL;
    PyObject *dict = NULL;
    PyObject *result = NULL;

    program = Py_DecodeLocale(argv[0], NULL);
    if (!program) {
        fprintf(stderr, "unable to decode the program name");
        goto error;
    }

    Py_SetProgramName(program);
    Py_Initialize();

    // import json
    json_module = PyImport_ImportModule("json");
    PYOBJECT_CHECK(json_module, error);

    // json_dumps = json.dumps
    json_dumps = PyObject_GetAttrString(json_module, "dumps");
    PYOBJECT_CHECK(json_dumps, error);

    // dict = {'foo': 'Foo', 'bar': 123}
    dict = Py_BuildValue("({sssi})", "foo", "Foo", "bar", 123);
    PYOBJECT_CHECK(dict, error);

    // result = json.dumps(dict)
    result = PyObject_CallObject(json_dumps, dict);
    PYOBJECT_CHECK(result, error);
    PyObject_Print(result, stdout, 0);
    printf("\n");
    rc = 0;
error:
    Py_XDECREF(result);
    Py_XDECREF(dict);
    Py_XDECREF(json_dumps);
    Py_XDECREF(json_module);

    PyMem_RawFree(program);
    Py_Finalize();
    return rc;
}

output:

$ clang `python3-config --cflags` -c foo.c -o foo.o
$ clang `python3-config --ldflags` foo.o -o foo
$ ./foo
'{"foo": "Foo", "bar": 123}'

Performance of C Extension

#include <Python.h>

static unsigned long
fib(unsigned long n)
{
    if (n < 2) return n;
    return fib(n - 1) + fib(n - 2);
}

static PyObject *
fibonacci(PyObject *self, PyObject *args)
{
    unsigned long n = 0;
    if (!PyArg_ParseTuple(args, "k", &n)) return NULL;
    return PyLong_FromUnsignedLong(fib(n));
}

static PyMethodDef methods[] = {
    {"fib", (PyCFunction)fibonacci, METH_VARARGS, NULL},
    {NULL, NULL, 0, NULL}
};

static struct PyModuleDef module = {
    PyModuleDef_HEAD_INIT, "foo", NULL, -1, methods
};

PyMODINIT_FUNC PyInit_foo(void)
{
    return PyModule_Create(&module);
}

Compare the performance with pure Python

>>> from time import time
>>> import foo
>>> def fib(n):
...     if n < 2: return n
...     return fib(n - 1) + fib(n - 2)
...
>>> s = time(); _ = fib(35); e = time(); e - s
4.953313112258911
>>> s = time(); _ = foo.fib(35); e = time(); e - s
0.04628586769104004

Performance of ctypes

// Compile (Mac)
// -------------
//
//   $ clang -Wall -Werror -shared -fPIC -o libfib.dylib fib.c
//
unsigned int fib(unsigned int n)
{
    if ( n < 2) {
        return n;
    }
    return fib(n-1) + fib(n-2);
}

Compare the performance with pure Python

>>> from time import time
>>> from ctypes import CDLL
>>> def fib(n):
...     if n < 2: return n
...     return fib(n - 1) + fib(n - 2)
...
>>> cfib = CDLL("./libfib.dylib").fib
>>> s = time(); _ = fib(35); e = time(); e - s
4.918856859207153
>>> s = time(); _ = cfib(35); e = time(); e - s
0.07283687591552734

ctypes Error handling

from __future__ import print_function

import os

from ctypes import *
from sys import platform, maxsize

is_64bits = maxsize > 2 ** 32

if is_64bits and platform == "darwin":
    libc = CDLL("libc.dylib", use_errno=True)
else:
    raise RuntimeError("Not support platform: {}".format(platform))

stat = libc.stat

class Stat(Structure):
    """
    From /usr/include/sys/stat.h

    struct stat {
        dev_t         st_dev;
        ino_t         st_ino;
        mode_t        st_mode;
        nlink_t       st_nlink;
        uid_t         st_uid;
        gid_t         st_gid;
        dev_t         st_rdev;
    #ifndef _POSIX_SOURCE
        struct      timespec st_atimespec;
        struct      timespec st_mtimespec;
        struct      timespec st_ctimespec;
    #else
        time_t        st_atime;
        long          st_atimensec;
        time_t        st_mtime;
        long          st_mtimensec;
        time_t        st_ctime;
        long          st_ctimensec;
    #endif
        off_t         st_size;
        int64_t       st_blocks;
        u_int32_t     st_blksize;
        u_int32_t     st_flags;
        u_int32_t     st_gen;
        int32_t       st_lspare;
        int64_t       st_qspare[2];
    };
    """
    _fields_ = [
        ("st_dev", c_ulong),
        ("st_ino", c_ulong),
        ("st_mode", c_ushort),
        ("st_nlink", c_uint),
        ("st_uid", c_uint),
        ("st_gid", c_uint),
        ("st_rdev", c_ulong),
        ("st_atime", c_longlong),
        ("st_atimendesc", c_long),
        ("st_mtime", c_longlong),
        ("st_mtimendesc", c_long),
        ("st_ctime", c_longlong),
        ("st_ctimendesc", c_long),
        ("st_size", c_ulonglong),
        ("st_blocks", c_int64),
        ("st_blksize", c_uint32),
        ("st_flags", c_uint32),
        ("st_gen", c_uint32),
        ("st_lspare", c_int32),
        ("st_qspare", POINTER(c_int64) * 2),
    ]

# stat success
path = create_string_buffer(b"/etc/passwd")
st = Stat()
ret = stat(path, byref(st))
assert ret == 0

# if stat fail, check errno
path = create_string_buffer(b"&%$#@!")
st = Stat()
ret = stat(path, byref(st))
if ret != 0:
    errno = get_errno()  # get errno
    errmsg = "stat({}) failed. {}".format(path.raw, os.strerror(errno))
    raise OSError(errno, errmsg)

output:

$ python err_handling.py   # python2
Traceback (most recent call last):
  File "err_handling.py", line 85, in <module>
    raise OSError(errno_, errmsg)
OSError: [Errno 2] stat(&%$#@!) failed. No such file or directory

$ python3 err_handling.py  # python3
Traceback (most recent call last):
  File "err_handling.py", line 85, in <module>
    raise OSError(errno_, errmsg)
FileNotFoundError: [Errno 2] stat(b'&%$#@!\x00') failed. No such file or directory

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

python-c-extensions.rst

python-c-extensions.rst

C Extensions

Simple setup.py

Customize CFLAGS

Doc String

Simple C Extension

Release the GIL

Acquire the GIL

Get Reference Count

Parse Arguments

Calling Python Functions

Raise Exception

Customize Exception

Iterate a List

Iterate a Dictionary

Simple Class

Simple Class with Members and Methods

Simplie Class with Getter and Setter

Inherit from Other Class

Run a Python Command

Run a Python File

Import a Python Module

Import everything of a Module

Access Attributes

Performance of C Extension

Performance of ctypes

ctypes Error handling

Files

python-c-extensions.rst

Latest commit

History

python-c-extensions.rst

File metadata and controls

C Extensions

Simple setup.py

Customize CFLAGS

Doc String

Simple C Extension

Release the GIL

Acquire the GIL

Get Reference Count

Parse Arguments

Calling Python Functions

Raise Exception

Customize Exception

Iterate a List

Iterate a Dictionary

Simple Class

Simple Class with Members and Methods

Simplie Class with Getter and Setter

Inherit from Other Class

Run a Python Command

Run a Python File

Import a Python Module

Import everything of a Module

Access Attributes

Performance of C Extension

Performance of ctypes

ctypes Error handling