py/make_root_pointers: Add MP_REGISTER_ROOT_POINTER parser/generator.

This adds new compile-time infrastructure to parse source code files for
`MP_REGISTER_ROOT_POINTER()` and generates a new `root_pointers.h` header
file containing the collected declarations.  This works the same as the
existing `MP_REGISTER_MODULE()` feature.

Signed-off-by: David Lechner <[email protected]>

Author: dlech

Diff for: ports/cc3200/bootmgr/bootloader.mk

@@ -134,3 +134,7 @@ $(HEADER_BUILD)/mpversion.h: | $(HEADER_BUILD)
 # Create an empty "moduledefs.h" needed by py/mkrules.mk
 $(HEADER_BUILD)/moduledefs.h: | $(HEADER_BUILD)
 	touch $@
+
+# Create an empty "root_pointers.h" needed by py/mkrules.mk
+$(HEADER_BUILD)/root_pointers.h: | $(HEADER_BUILD)
+	touch $@

Diff for: ports/stm32/mboot/Makefile

@@ -213,21 +213,25 @@ PREFIX_FILE = ../boards/stm32f4xx_prefix.c
 BOARD_PINS = $(BOARD_DIR)/pins.csv
 HEADER_BUILD = $(BUILD)/genhdr
 GEN_QSTRDEFS_GENERATED = $(HEADER_BUILD)/qstrdefs.generated.h
+GEN_ROOT_POINTERS = $(HEADER_BUILD)/root_pointers.h
 GEN_PINS_SRC = $(BUILD)/pins_$(BOARD).c
 GEN_PINS_HDR = $(HEADER_BUILD)/pins.h
 GEN_PINS_QSTR = $(HEADER_BUILD)/pins_qstr.h
 GEN_PINS_AF_CONST = $(HEADER_BUILD)/pins_af_const.h
 GEN_PINS_AF_DEFS = $(HEADER_BUILD)/pins_af_defs.h
 GEN_PINS_AF_PY = $(BUILD)/pins_af.py
 
-$(OBJ): $(GEN_QSTRDEFS_GENERATED) $(GEN_PINS_AF_DEFS)
+$(OBJ): $(GEN_QSTRDEFS_GENERATED) $(GEN_ROOT_POINTERS) $(GEN_PINS_AF_DEFS)
 
 $(HEADER_BUILD):
 	$(MKDIR) -p $(BUILD)/genhdr
 
 $(GEN_QSTRDEFS_GENERATED): | $(HEADER_BUILD)
 	$(Q)echo "// empty" > $@
 
+$(GEN_ROOT_POINTERS): | $(HEADER_BUILD)
+	$(Q)echo "// empty" > $@
+
 $(GEN_PINS_AF_DEFS): $(BOARD_PINS) $(MAKE_PINS) ../$(AF_FILE) $(PREFIX_FILE) | $(HEADER_BUILD)
 	$(ECHO) "GEN $@"
 	$(Q)$(PYTHON) $(MAKE_PINS) --board $(BOARD_PINS) --af ../$(AF_FILE) \

Diff for: ports/windows/msvc/genhdr.targets

@@ -4,7 +4,7 @@
   <Import Project="paths.props" Condition="'$(PyPathsIncluded)' != 'True'"/>
 
   <!--Generate qstrdefs.generated.h and mpversion.h similar to what is done in py/mkrules.mk and py/py.mk-->
-  <Target Name="GenerateHeaders" DependsOnTargets="MakeVersionHdr;MakeModuleDefs;MakeQstrData">
+  <Target Name="GenerateHeaders" DependsOnTargets="MakeVersionHdr;MakeModuleDefs;MakeRootPointers;MakeQstrData">
   </Target>
 
   <PropertyGroup>
@@ -15,6 +15,7 @@
     <QstrDefsCollected>$(DestDir)qstrdefscollected.h</QstrDefsCollected>
     <QstrGen>$(DestDir)qstrdefs.generated.h</QstrGen>
     <ModuleDefsCollected>$(DestDir)/moduledefs.collected</ModuleDefsCollected>
+    <RootPointersCollected>$(DestDir)/root_pointers.collected</RootPointersCollected>
     <PyPython Condition="'$(PyPython)' == ''">$(MICROPY_CPYTHON3)</PyPython>
     <PyPython Condition="'$(PyPython)' == ''">python</PyPython>
     <CLToolExe Condition="'$(CLToolExe)' == ''">cl.exe</CLToolExe>
@@ -45,7 +46,7 @@
 using(var outFile = System.IO.File.CreateText(OutputFile)) {
   foreach(var inFile in InputFiles)
     foreach(var line in System.IO.File.ReadAllLines(inFile))
-      if((line.Contains(".c") && line.StartsWith("#line")) || line.Contains("MP_QSTR"))
+      if((line.Contains(".c") && line.StartsWith("#line")) || line.Contains("MP_QSTR") || line.Contains("MP_REGISTER"))
         outFile.WriteLine( line );
 }
         ]]>
@@ -114,6 +115,20 @@ using(var outFile = System.IO.File.CreateText(OutputFile)) {
     <MSBuild Projects="$(MSBuildThisFileFullPath)" Targets="CopyFileIfDifferent" Properties="SourceFile=$(TmpFile);DestFile=$(DestFile)"/>
   </Target>
 
+  <Target Name="CollectRootPointers" DependsOnTargets="MakeQstrDefs" Inputs="$(DestDir)qstr.i.last" Outputs="$(RootPointersCollected)">
+    <Exec Command="$(PyPython) $(PySrcDir)makeqstrdefs.py split root_pointer $(DestDir)qstr.i.last $(DestDir)root_pointer _"/>
+    <Exec Command="$(PyPython) $(PySrcDir)makeqstrdefs.py cat root_pointer _ $(DestDir)root_pointer $(RootPointersCollected)"/>
+  </Target>
+
+  <Target Name="MakeRootPointers" DependsOnTargets="CollectRootPointers" Inputs="$(PySrcDir)make_root_pointers.py;$(RootPointersCollected)" Outputs="$(DestDir)root_pointers.h">
+    <PropertyGroup>
+      <DestFile>$(DestDir)root_pointers.h</DestFile>
+      <TmpFile>$(DestFile).tmp</TmpFile>
+    </PropertyGroup>
+    <Exec Command="$(PyPython) $(PySrcDir)make_root_pointers.py $(RootPointersCollected) > $(TmpFile)"/>
+    <MSBuild Projects="$(MSBuildThisFileFullPath)" Targets="CopyFileIfDifferent" Properties="SourceFile=$(TmpFile);DestFile=$(DestFile)"/>
+  </Target>
+
   <Target Name="MakeQstrData" DependsOnTargets="MakeQstrDefs" Inputs="$(QstrDefsCollected);$(PyQstrDefs);$(QstrDefs)" Outputs="$(QstrGen)">
     <PropertyGroup>
       <TmpFile>$(QstrGen).tmp</TmpFile>
@@ -132,7 +147,7 @@ using(var outFile = System.IO.File.CreateText(OutputFile)) {
     <MSBuild Projects="$(MSBuildThisFileFullPath)" Targets="CopyFileIfDifferent" Properties="SourceFile=$(TmpFile);DestFile=$(DestFile)"/>
   </Target>
 
-  <Target Name="FreezeModules" Condition="'$(FrozenManifest)' != ''" DependsOnTargets="MakeQstrData" Inputs="$(FrozenManifest)" Outputs="$(PyBuildDir)frozen_content.c">
+  <Target Name="FreezeModules" Condition="'$(FrozenManifest)' != ''" DependsOnTargets="MakeQstrData;MakeRootPointers" Inputs="$(FrozenManifest)" Outputs="$(PyBuildDir)frozen_content.c">
     <ItemGroup>
       <ClCompile Include="$(PyBuildDir)frozen_content.c"/>
       <ClCompile>

Diff for: py/make_root_pointers.py

@@ -0,0 +1,55 @@
+"""
+This pre-processor parses a single file containing a list of
+MP_REGISTER_ROOT_POINTER(variable declaration) items.
+
+These are used to generate a header with the required entries for
+"struct _mp_state_vm_t" in py/mpstate.h
+"""
+
+from __future__ import print_function
+
+import argparse
+import io
+import re
+
+
+PATTERN = re.compile(r"MP_REGISTER_ROOT_POINTER\((.*?)\);")
+
+
+def find_root_pointer_registrations(filename):
+    """Find any MP_REGISTER_ROOT_POINTER definitions in the provided file.
+
+    :param str filename: path to file to check
+    :return: List[variable_declaration]
+    """
+    with io.open(filename, encoding="utf-8") as c_file_obj:
+        return set(re.findall(PATTERN, c_file_obj.read()))
+
+
+def generate_root_pointer_header(root_pointers):
+    """Generate header with root pointer entries.
+
+    :param List[variable_declaration] root_pointers: root pointer declarations
+    :return: None
+    """
+
+    # Print header file for all external modules.
+    print("// Automatically generated by make_root_pointers.py.")
+    print()
+
+    for item in root_pointers:
+        print(item, end=";")
+        print()
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("file", nargs=1, help="file with MP_REGISTER_ROOT_POINTER definitions")
+    args = parser.parse_args()
+
+    root_pointers = find_root_pointer_registrations(args.file[0])
+    generate_root_pointer_header(sorted(root_pointers))
+
+
+if __name__ == "__main__":
+    main()

Diff for: py/makeqstrdefs.py

@@ -24,6 +24,9 @@
 # Extract MP_REGISTER_MODULE(...) macros.
 _MODE_MODULE = "module"
 
+# Extract MP_REGISTER_ROOT_POINTER(...) macros.
+_MODE_ROOT_POINTER = "root_pointer"
+
 
 def is_c_source(fname):
     return os.path.splitext(fname)[1] in [".c"]
@@ -90,6 +93,8 @@ def process_file(f):
         re_match = re.compile(r'MP_COMPRESSED_ROM_TEXT\("([^"]*)"\)')
     elif args.mode == _MODE_MODULE:
         re_match = re.compile(r"MP_REGISTER_MODULE\(.*?,\s*.*?\);")
+    elif args.mode == _MODE_ROOT_POINTER:
+        re_match = re.compile(r"MP_REGISTER_ROOT_POINTER\(.*?\);")
     output = []
     last_fname = None
     for line in f:
@@ -111,7 +116,7 @@ def process_file(f):
             if args.mode == _MODE_QSTR:
                 name = match.replace("MP_QSTR_", "")
                 output.append("Q(" + name + ")")
-            elif args.mode in (_MODE_COMPRESS, _MODE_MODULE):
+            elif args.mode in (_MODE_COMPRESS, _MODE_MODULE, _MODE_ROOT_POINTER):
                 output.append(match)
 
     if last_fname:
@@ -148,6 +153,8 @@ def cat_together():
         mode_full = "Compressed data"
     elif args.mode == _MODE_MODULE:
         mode_full = "Module registrations"
+    elif args.mode == _MODE_ROOT_POINTER:
+        mode_full = "Root pointer registrations"
     if old_hash != new_hash:
         print(mode_full, "updated")
         try:
@@ -208,7 +215,7 @@ class Args:
     args.output_dir = sys.argv[4]
     args.output_file = None if len(sys.argv) == 5 else sys.argv[5]  # Unused for command=split
 
-    if args.mode not in (_MODE_QSTR, _MODE_COMPRESS, _MODE_MODULE):
+    if args.mode not in (_MODE_QSTR, _MODE_COMPRESS, _MODE_MODULE, _MODE_ROOT_POINTER):
         print("error: mode %s unrecognised" % sys.argv[2])
         sys.exit(2)
 

Diff for: py/mkrules.cmake

@@ -11,6 +11,9 @@ set(MICROPY_QSTRDEFS_GENERATED "${MICROPY_GENHDR_DIR}/qstrdefs.generated.h")
 set(MICROPY_MODULEDEFS_SPLIT "${MICROPY_GENHDR_DIR}/moduledefs.split")
 set(MICROPY_MODULEDEFS_COLLECTED "${MICROPY_GENHDR_DIR}/moduledefs.collected")
 set(MICROPY_MODULEDEFS "${MICROPY_GENHDR_DIR}/moduledefs.h")
+set(MICROPY_ROOT_POINTERS_SPLIT "${MICROPY_GENHDR_DIR}/root_pointers.split")
+set(MICROPY_ROOT_POINTERS_COLLECTED "${MICROPY_GENHDR_DIR}/root_pointers.collected")
+set(MICROPY_ROOT_POINTERS "${MICROPY_GENHDR_DIR}/root_pointers.h")
 
 # Need to do this before extracting MICROPY_CPP_DEF below. Rest of frozen
 # manifest handling is at the end of this file.
@@ -46,6 +49,7 @@ target_sources(${MICROPY_TARGET} PRIVATE
     ${MICROPY_MPVERSION}
     ${MICROPY_QSTRDEFS_GENERATED}
     ${MICROPY_MODULEDEFS}
+    ${MICROPY_ROOT_POINTERS}
 )
 
 # Command to force the build of another command
@@ -139,6 +143,31 @@ add_custom_command(
     DEPENDS ${MICROPY_MODULEDEFS_COLLECTED}
 )
 
+# Generate root_pointers.h
+
+add_custom_command(
+    OUTPUT ${MICROPY_ROOT_POINTERS_SPLIT}
+    COMMAND ${Python3_EXECUTABLE} ${MICROPY_PY_DIR}/makeqstrdefs.py split root_pointer ${MICROPY_GENHDR_DIR}/qstr.i.last ${MICROPY_GENHDR_DIR}/root_pointer _
+    COMMAND touch ${MICROPY_ROOT_POINTERS_SPLIT}
+    DEPENDS ${MICROPY_QSTRDEFS_LAST}
+    VERBATIM
+    COMMAND_EXPAND_LISTS
+)
+
+add_custom_command(
+    OUTPUT ${MICROPY_ROOT_POINTERS_COLLECTED}
+    COMMAND ${Python3_EXECUTABLE} ${MICROPY_PY_DIR}/makeqstrdefs.py cat root_pointer _ ${MICROPY_GENHDR_DIR}/root_pointer ${MICROPY_ROOT_POINTERS_COLLECTED}
+    DEPENDS ${MICROPY_ROOT_POINTERS_SPLIT}
+    VERBATIM
+    COMMAND_EXPAND_LISTS
+)
+
+add_custom_command(
+    OUTPUT ${MICROPY_ROOT_POINTERS}
+    COMMAND ${Python3_EXECUTABLE} ${MICROPY_PY_DIR}/make_root_pointers.py ${MICROPY_ROOT_POINTERS_COLLECTED} > ${MICROPY_ROOT_POINTERS}
+    DEPENDS ${MICROPY_ROOT_POINTERS_COLLECTED} ${MICROPY_PY_DIR}/make_root_pointers.py
+)
+
 # Build frozen code if enabled
 
 if(MICROPY_FROZEN_MANIFEST)
@@ -174,6 +203,7 @@ if(MICROPY_FROZEN_MANIFEST)
         COMMAND ${Python3_EXECUTABLE} ${MICROPY_DIR}/tools/makemanifest.py -o ${MICROPY_FROZEN_CONTENT} -v "MPY_DIR=${MICROPY_DIR}" -v "MPY_LIB_DIR=${MICROPY_LIB_DIR}" -v "PORT_DIR=${MICROPY_PORT_DIR}" -v "BOARD_DIR=${MICROPY_BOARD_DIR}" -b "${CMAKE_BINARY_DIR}" -f${MICROPY_CROSS_FLAGS} ${MICROPY_FROZEN_MANIFEST}
         DEPENDS MICROPY_FORCE_BUILD
             ${MICROPY_QSTRDEFS_GENERATED}
+            ${MICROPY_ROOT_POINTERS}
             ${MICROPY_MPYCROSS_DEPENDENCY}
         VERBATIM
     )

Diff for: py/mkrules.mk

@@ -7,8 +7,8 @@ endif
 # Extra deps that need to happen before object compilation.
 OBJ_EXTRA_ORDER_DEPS =
 
-# Generate moduledefs.h.
-OBJ_EXTRA_ORDER_DEPS += $(HEADER_BUILD)/moduledefs.h
+# Generate header files.
+OBJ_EXTRA_ORDER_DEPS += $(HEADER_BUILD)/moduledefs.h $(HEADER_BUILD)/root_pointers.h
 
 ifeq ($(MICROPY_ROM_TEXT_COMPRESSION),1)
 # If compression is enabled, trigger the build of compressed.data.h...
@@ -126,6 +126,16 @@ $(HEADER_BUILD)/moduledefs.collected: $(HEADER_BUILD)/moduledefs.split
 	$(ECHO) "GEN $@"
 	$(Q)$(PYTHON) $(PY_SRC)/makeqstrdefs.py cat module _ $(HEADER_BUILD)/module $@
 
+# Module definitions via MP_REGISTER_ROOT_POINTER.
+$(HEADER_BUILD)/root_pointers.split: $(HEADER_BUILD)/qstr.i.last
+	$(ECHO) "GEN $@"
+	$(Q)$(PYTHON) $(PY_SRC)/makeqstrdefs.py split root_pointer $< $(HEADER_BUILD)/root_pointer _
+	$(Q)$(TOUCH) $@
+
+$(HEADER_BUILD)/root_pointers.collected: $(HEADER_BUILD)/root_pointers.split
+	$(ECHO) "GEN $@"
+	$(Q)$(PYTHON) $(PY_SRC)/makeqstrdefs.py cat root_pointer _ $(HEADER_BUILD)/root_pointer $@
+
 # Compressed error strings.
 $(HEADER_BUILD)/compressed.split: $(HEADER_BUILD)/qstr.i.last
 	$(ECHO) "GEN $@"
@@ -165,7 +175,7 @@ endif
 
 ifneq ($(FROZEN_MANIFEST),)
 # to build frozen_content.c from a manifest
-$(BUILD)/frozen_content.c: FORCE $(BUILD)/genhdr/qstrdefs.generated.h | $(MICROPY_MPYCROSS_DEPENDENCY)
+$(BUILD)/frozen_content.c: FORCE $(BUILD)/genhdr/qstrdefs.generated.h $(BUILD)/genhdr/root_pointers.h | $(MICROPY_MPYCROSS_DEPENDENCY)
 	$(Q)$(MAKE_MANIFEST) -o $@ -v "MPY_DIR=$(TOP)" -v "MPY_LIB_DIR=$(MPY_LIB_DIR)" -v "PORT_DIR=$(shell pwd)" -v "BOARD_DIR=$(BOARD_DIR)" -b "$(BUILD)" $(if $(MPY_CROSS_FLAGS),-f"$(MPY_CROSS_FLAGS)",) --mpy-tool-flags="$(MPY_TOOL_FLAGS)" $(FROZEN_MANIFEST)
 endif
 

Diff for: py/mpstate.h

@@ -192,6 +192,13 @@ typedef struct _mp_state_vm_t {
     // include any root pointers defined by a port
     MICROPY_PORT_ROOT_POINTERS
 
+    // Include any root pointers registered with MP_REGISTER_ROOT_POINTER().
+    #ifndef NO_QSTR
+    // Only include root pointer definitions when not doing qstr extraction, because
+    // the qstr extraction stage also generates the root pointers header file.
+    #include "genhdr/root_pointers.h"
+    #endif
+
     // root pointers for extmod
 
     #if MICROPY_REPL_EVENT_DRIVEN

Diff for: py/obj.h

@@ -416,13 +416,18 @@ typedef struct _mp_rom_obj_t { mp_const_obj_t o; } mp_rom_obj_t;
 #define MP_DEFINE_CONST_STATICMETHOD_OBJ(obj_name, fun_name) const mp_rom_obj_static_class_method_t obj_name = {{&mp_type_staticmethod}, fun_name}
 #define MP_DEFINE_CONST_CLASSMETHOD_OBJ(obj_name, fun_name) const mp_rom_obj_static_class_method_t obj_name = {{&mp_type_classmethod}, fun_name}
 
+#ifndef NO_QSTR
+
 // Declare a module as a builtin, processed by makemoduledefs.py
 // param module_name: MP_QSTR_<module name>
 // param obj_module: mp_obj_module_t instance
-
-#ifndef NO_QSTR
 #define MP_REGISTER_MODULE(module_name, obj_module)
-#endif
+
+// Declare a root pointer (to avoid garbage collection of a global static variable).
+// param variable_declaration: a valid C variable declaration
+#define MP_REGISTER_ROOT_POINTER(variable_declaration)
+
+#endif // NO_QSTR
 
 // Underlying map/hash table implementation (not dict object or map function)
 

Diff for: py/py.mk

@@ -218,6 +218,11 @@ $(HEADER_BUILD)/moduledefs.h: $(HEADER_BUILD)/moduledefs.collected
 	@$(ECHO) "GEN $@"
 	$(Q)$(PYTHON) $(PY_SRC)/makemoduledefs.py $< > $@
 
+# build a list of registered root pointers for py/mpstate.h.
+$(HEADER_BUILD)/root_pointers.h: $(HEADER_BUILD)/root_pointers.collected $(PY_SRC)/make_root_pointers.py
+	@$(ECHO) "GEN $@"
+	$(Q)$(PYTHON) $(PY_SRC)/make_root_pointers.py $< > $@
+
 # Standard C functions like memset need to be compiled with special flags so
 # the compiler does not optimise these functions in terms of themselves.
 CFLAGS_BUILTIN ?= -ffreestanding -fno-builtin -fno-lto