Add a section on how to add an extension module to the stdlib.

[ericsnowcurrently]

Every once in a while someone adds a new extension module to the stdlib. How to do so isn't very clear, particularly regarding all the cross-platform bits. I suppose nobody has added that info to the devguide because it doesn't happen very often; however, it would be particularly helpful in the devguide specifically because we don't do it very often. :) You can mostly look for prior art (e.g. search for what was done when the last module was added), but that's not ideal and it's easy to miss stuff.

I'd expect a new (short) section, probably in stdlibchanges.rst, with a brief summary (including expectations) and a step-by-step for supported platforms. Each platform would have it's own subsection with the steps as well as any caveats. For myself, the Windows section would be where I'd get the most value. However, I imagine that Windows folks could say the same for non-Windows (trivial as it might seem to me).

[ncoghlan]

Heh, "look for the commits from the last time an extension module was added" is exactly how I've handled our current lack of docs myself :)

[picnixz]

EDIT: this is incomplete since I forgot to include the modifications for the configure scripts so I'll edit the PR directly.

Incomplete tutorial

I can help here by dumping something I needed to do this week. It's probably incomplete and this might not be the only steps to do if external libraries are needed, but for simple accelerators I think it should work:

Note to myself for creating a C accelerator module (simple one, no external dependencies).

// Modules/foo/foomodule.h

#ifndef FOOMODULE_H
#define FOOMODULE_H
#include "Python.h"

typedef struct {
...
} foomodule_state;

static inline foomodule_state *
get_foomodule_state(PyObject *module)
{
    void *state = PyModule_GetState(module);
    assert(state != NULL);
    return (foomodule_state *)state;
}

extern int Py_helper1();
extern int Py_helper2();

#endif // FOOMODULE_H

// Modules/foo/foomodule.c

#include "foomodule.h" // for pre-declarations
#include "clinic/foomodule.c.h"    // if needed

static int
foomodule_exec(PyObject *module)
{
	// imports, static attributes, exported classes, etc
	return 0;
}

static int
foomodule_traverse(PyObject *m, visitproc visit, void *arg)
{
    foomodule_state *st = get_foomodule_state(m);
    // call Py_VISIT() on the state attributes
    return 0;
}

static int
foomodule_clear(PyObject *m)
{
    foomodule_state *st = get_foomodule_state(m);
    // call Py_CLEAR() on the state attributes
    return 0;
}

static void
foomodule_free(void *m) { (void)foomodule_clear((PyObject *)m); }

/*[clinic input]
module foo
[clinic start generated code]*/
/*[clinic end generated code: output=... input=...]*/

/*[clinic input]
foo.bar1 -> bool

[clinic start generated code]*/

static int 
foo_bar1_impl(PyObject *module)
/*[clinic end generated code: output=... input=...]*/
{
	return Py_helper1();
}

/*[clinic input]
foo.bar2 -> bool

[clinic start generated code]*/

static int 
foo_bar2_impl(PyObject *module)
/*[clinic end generated code: output=... input=...]*/
{
	return Py_helper2();
}

static PyMethodDef foomodule_methods[] = {
    FOO_BAR1_METHODDEF
    FOO_BAR2_METHODDEF
    {NULL, NULL}
};

static struct PyModuleDef_Slot foomodule_slots[] = {
    {Py_mod_exec, foomodule_exec}, // foomodule_exec may be NULL if the state is trivial
    {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED},
    {Py_mod_gil, Py_MOD_GIL_NOT_USED},
    {0, NULL},
};

static struct PyModuleDef foomodule = {
    PyModuleDef_HEAD_INIT,
    .m_name = "foo",
    .m_doc = NULL,
    .m_size = sizeof(foomodule_state),
    .m_methods = foomodule_methods,	  
    .m_slots = foomodule_slots,		  
    .m_traverse = foomodule_traverse, // or NULL if the state is trivial
    .m_clear = foomodule_clear,	      // or NULL if the state is trivial
    .m_free = foomodule_free,		  // or NULL if the state is trivial
};

PyMODINIT_FUNC
PyInit_foo(void)
{
    return PyModuleDef_Init(&_foomodule);
}

// Modules/foo/helper1.c
#include "foomodule.h"

int Py_helper1() { return 1; }

// Modules/foo/helper2.c
#include "foomodule.h"

int Py_helper2() { return 2; }

Files to modify

# Makefile.pre.in
FOO_H = Modules/foo/foomodule.h

FOO_OBJS =	\
	Modules/foo/foomodule.o \
	Modules/foo/helper1.o \
	Modules/foo/helper2.o`

# Add the following rule in the '# Special rules for object files' section
$(FOO_OBJS): $(FOO_H)
# Add the following rule in the dependencies section (maybe more .h if needed)
MODULE_FOO_DEPS=$(srcdir)/Modules/foo/foomodule.h

For Windows, don't forget to:

• Add extern PyObject* PyInit_foo(void); and the entry {"foo", PyInit_foo} in PC/config.c.

• In pythoncore.vcxproj, add:

  // in the ItemGroup containing `..\Modules\*.h` files 
  <ClInclude Include="..\Modules\foo\foomodule.h" />

  // in the ItemGroup containing `..\Modules\*.c` files
  <ClCompile Include="..\Modules\foo\foomodule.c" />
  <ClCompile Include="..\Modules\foo\helper1.c" />
  <ClCompile Include="..\Modules\foo\helper2.c" />

• In pythoncore.vcxproj.filters, add

  // in the ItemGroup containing `..\Modules\*.h` files 
  <ClInclude Include="..\Modules\foo\foomodule.h">
      <Filter>Modules\foo</Filter>
  </ClInclude>

  // in the ItemGroup containing `..\Modules\*.c` files
  <ClCompile Include="..\Modules\foo\foomodule.c">
    <Filter>Modules\foo</Filter>
  </ClCompile>
  <ClCompile Include="..\Modules\foo\helper1.c">
    <Filter>Modules\foo</Filter>
  </ClCompile>
  <ClCompile Include="..\Modules\foo\helper2.c">
    <Filter>Modules\foo</Filter>
  </ClCompile>

Additional steps

• Run make -j12 regen-configure (requires Docker)
• Run make -j12 regen-all
• Run make -j12 regen-stdlib-module-names