Merge pull request micropython#1057 from tannewt/flexible_heap

Add basic memory allocation outside Python runtime

Author: dhalbert

main.c

@@ -45,12 +45,14 @@
 
 #include "mpconfigboard.h"
 #include "supervisor/cpu.h"
+#include "supervisor/memory.h"
 #include "supervisor/port.h"
 #include "supervisor/filesystem.h"
 // TODO(tannewt): Figure out how to choose language at compile time.
 #include "supervisor/messages/en-US.h"
 #include "supervisor/shared/autoreload.h"
 #include "supervisor/shared/rgb_led_status.h"
+#include "supervisor/shared/stack.h"
 #include "supervisor/serial.h"
 
 void do_str(const char *src, mp_parse_input_kind_t input_kind) {
@@ -73,12 +75,21 @@ void do_str(const char *src, mp_parse_input_kind_t input_kind) {
     }
 }
 
-static char heap[PORT_HEAP_SIZE];
-
-void reset_mp(void) {
+void start_mp(supervisor_allocation* heap) {
     reset_status_led();
     autoreload_stop();
 
+    // Stack limit should be less than real stack size, so we have a chance
+    // to recover from limit hit.  (Limit is measured in bytes.)
+    mp_stack_ctrl_init();
+    mp_stack_set_limit(stack_alloc->length - 1024);
+
+#if MICROPY_MAX_STACK_USAGE
+    // _ezero (same as _ebss) is an int, so start 4 bytes above it.
+    mp_stack_set_bottom(stack_alloc->ptr);
+    mp_stack_fill_with_sentinel();
+#endif
+
     // Sync the file systems in case any used RAM from the GC to cache. As soon
     // as we re-init the GC all bets are off on the cache.
     filesystem_flush();
@@ -87,7 +98,7 @@ void reset_mp(void) {
     readline_init0();
 
     #if MICROPY_ENABLE_GC
-    gc_init(heap, heap + sizeof(heap));
+    gc_init(heap->ptr, heap->ptr + heap->length / 4);
     #endif
     mp_init();
     mp_obj_list_init(mp_sys_path, 0);
@@ -99,6 +110,11 @@ void reset_mp(void) {
 
     mp_obj_list_init(mp_sys_argv, 0);
 }
+
+void stop_mp(void) {
+
+}
+
 #define STRING_LIST(...) {__VA_ARGS__, ""}
 
 // Look for the first file that exists in the list of filenames, using mp_import_stat().
@@ -124,7 +140,7 @@ bool maybe_run_list(const char ** filenames, pyexec_result_t* exec_result) {
     return true;
 }
 
-bool start_mp(safe_mode_t safe_mode) {
+bool run_code_py(safe_mode_t safe_mode) {
     bool serial_connected_at_start = serial_connected();
     #ifdef CIRCUITPY_AUTORELOAD_DELAY_MS
     if (serial_connected_at_start) {
@@ -155,14 +171,20 @@ bool start_mp(safe_mode_t safe_mode) {
         const char *supported_filenames[] = STRING_LIST("code.txt", "code.py", "main.py", "main.txt");
         const char *double_extension_filenames[] = STRING_LIST("code.txt.py", "code.py.txt", "code.txt.txt","code.py.py",
                                                     "main.txt.py", "main.py.txt", "main.txt.txt","main.py.py");
-        reset_mp();
+
+        stack_resize();
+        filesystem_flush();
+        supervisor_allocation* heap = allocate_remaining_memory();
+        start_mp(heap);
         found_main = maybe_run_list(supported_filenames, &result);
         if (!found_main){
             found_main = maybe_run_list(double_extension_filenames, &result);
             if (found_main) {
                 serial_write(MSG_DOUBLE_FILE_EXTENSION);
             }
         }
+        stop_mp();
+        free_memory(heap);
 
         reset_port();
         reset_board();
@@ -291,6 +313,12 @@ void __attribute__ ((noinline)) run_boot_py(safe_mode_t safe_mode) {
         }
         #endif
 
+        stack_init();
+        // TODO(tannewt): Allocate temporary space to hold custom usb descriptors.
+        filesystem_flush();
+        supervisor_allocation* heap = allocate_remaining_memory();
+        start_mp(heap);
+
         // TODO(tannewt): Re-add support for flashing boot error output.
         bool found_boot = maybe_run_list(boot_py_filenames, NULL);
         (void) found_boot;
@@ -306,27 +334,41 @@ void __attribute__ ((noinline)) run_boot_py(safe_mode_t safe_mode) {
         // Reset to remove any state that boot.py setup. It should only be used to
         // change internal state that's not in the heap.
         reset_port();
-        reset_mp();
+        reset_board();
+        stop_mp();
+        free_memory(heap);
+    }
+}
+
+int run_repl(void) {
+    int exit_code = PYEXEC_FORCED_EXIT;
+    stack_resize();
+    filesystem_flush();
+    supervisor_allocation* heap = allocate_remaining_memory();
+    start_mp(heap);
+    autoreload_suspend();
+    new_status_color(REPL_RUNNING);
+    if (pyexec_mode_kind == PYEXEC_MODE_RAW_REPL) {
+        exit_code = pyexec_raw_repl();
+    } else {
+        exit_code = pyexec_friendly_repl();
     }
+    reset_port();
+    reset_board();
+    stop_mp();
+    free_memory(heap);
+    autoreload_resume();
+    return exit_code;
 }
 
 int __attribute__((used)) main(void) {
+    memory_init();
+
     // initialise the cpu and peripherals
     safe_mode_t safe_mode = port_init();
 
     rgb_led_status_init();
 
-    // Stack limit should be less than real stack size, so we have a chance
-    // to recover from limit hit.  (Limit is measured in bytes.)
-    mp_stack_set_top((char*)&_estack);
-    mp_stack_set_limit((char*)&_estack - (char*)&_ebss - 1024);
-
-#if MICROPY_MAX_STACK_USAGE
-    // _ezero (same as _ebss) is an int, so start 4 bytes above it.
-    mp_stack_set_bottom(&_ezero + 1);
-    mp_stack_fill_with_sentinel();
-#endif
-
     // Create a new filesystem only if we're not in a safe mode.
     // A power brownout here could make it appear as if there's
     // no SPI flash filesystem, and we might erase the existing one.
@@ -335,7 +377,6 @@ int __attribute__((used)) main(void) {
     // Reset everything and prep MicroPython to run boot.py.
     reset_port();
     reset_board();
-    reset_mp();
 
     // Turn on autoreload by default but before boot.py in case it wants to change it.
     autoreload_enable();
@@ -355,24 +396,14 @@ int __attribute__((used)) main(void) {
     bool first_run = true;
     for (;;) {
         if (!skip_repl) {
-            reset_mp();
-            autoreload_suspend();
-            new_status_color(REPL_RUNNING);
-            if (pyexec_mode_kind == PYEXEC_MODE_RAW_REPL) {
-                exit_code = pyexec_raw_repl();
-            } else {
-                exit_code = pyexec_friendly_repl();
-            }
-            autoreload_resume();
-            reset_port();
-            reset_board();
+            exit_code = run_repl();
         }
         if (exit_code == PYEXEC_FORCED_EXIT) {
             if (!first_run) {
                 serial_write(MSG_SOFT_REBOOT MSG_NEWLINE);
             }
             first_run = false;
-            skip_repl = start_mp(safe_mode);
+            skip_repl = run_code_py(safe_mode);
         } else if (exit_code != 0) {
             break;
         }

ports/atmel-samd/Makefile

@@ -288,7 +288,8 @@ SRC_C = \
 	lib/libc/string0.c \
 	lib/mp-readline/readline.c \
 	$(BUILD)/autogen_usb_descriptor.c \
-	freetouch/adafruit_ptc.c
+	freetouch/adafruit_ptc.c \
+	supervisor/shared/memory.c
 
 # Choose which flash filesystem impl to use.
 # (Right now INTERNAL_FLASH_FILESYSTEM and SPI_FLASH_FILESYSTEM are mutually exclusive.

ports/atmel-samd/external_flash/external_flash.c

@@ -37,6 +37,7 @@
 #include "py/runtime.h"
 #include "lib/oofatfs/ff.h"
 #include "shared-bindings/microcontroller/__init__.h"
+#include "supervisor/memory.h"
 #include "supervisor/shared/rgb_led_status.h"
 
 #include "hal_gpio.h"
@@ -59,6 +60,8 @@ static const external_flash_device* flash_device = NULL;
 // cache.
 static uint32_t dirty_mask;
 
+static supervisor_allocation* supervisor_cache = NULL;
+
 // Wait until both the write enable and write in progress bits have cleared.
 static bool wait_for_flash_ready(void) {
     uint8_t read_status_response[1] = {0x00};
@@ -308,6 +311,23 @@ static bool flush_scratch_flash(void) {
 static bool allocate_ram_cache(void) {
     uint8_t blocks_per_sector = SPI_FLASH_ERASE_SIZE / FILESYSTEM_BLOCK_SIZE;
     uint8_t pages_per_block = FILESYSTEM_BLOCK_SIZE / SPI_FLASH_PAGE_SIZE;
+
+    uint32_t table_size = blocks_per_sector * pages_per_block * sizeof(uint32_t);
+    // Attempt to allocate outside the heap first.
+    supervisor_cache = allocate_memory(table_size + SPI_FLASH_ERASE_SIZE, false);
+    if (supervisor_cache != NULL) {
+        MP_STATE_VM(flash_ram_cache) = (uint8_t **) supervisor_cache->ptr;
+        uint8_t* page_start = (uint8_t *) supervisor_cache->ptr + table_size;
+
+        for (uint8_t i = 0; i < blocks_per_sector; i++) {
+            for (uint8_t j = 0; j < pages_per_block; j++) {
+                uint32_t offset = i * pages_per_block + j;
+                MP_STATE_VM(flash_ram_cache)[offset] = page_start + offset * SPI_FLASH_PAGE_SIZE;
+            }
+        }
+        return true;
+    }
+
     MP_STATE_VM(flash_ram_cache) = m_malloc_maybe(blocks_per_sector * pages_per_block * sizeof(uint32_t), false);
     if (MP_STATE_VM(flash_ram_cache) == NULL) {
         return false;
@@ -383,14 +403,19 @@ static bool flush_ram_cache(bool keep_cache) {
             write_flash(current_sector + (i * pages_per_block + j) * SPI_FLASH_PAGE_SIZE,
                         MP_STATE_VM(flash_ram_cache)[i * pages_per_block + j],
                         SPI_FLASH_PAGE_SIZE);
-            if (!keep_cache) {
+            if (!keep_cache && supervisor_cache == NULL) {
                 m_free(MP_STATE_VM(flash_ram_cache)[i * pages_per_block + j]);
             }
         }
     }
     // We're done with the cache for now so give it back.
     if (!keep_cache) {
-        m_free(MP_STATE_VM(flash_ram_cache));
+        if (supervisor_cache != NULL) {
+            free_memory(supervisor_cache);
+            supervisor_cache = NULL;
+        } else {
+            m_free(MP_STATE_VM(flash_ram_cache));
+        }
         MP_STATE_VM(flash_ram_cache) = NULL;
     }
     return true;

ports/atmel-samd/mpconfigport.h

@@ -338,4 +338,8 @@ void run_background_tasks(void);
 #define CIRCUITPY_AUTORELOAD_DELAY_MS 500
 #define CIRCUITPY_BOOT_OUTPUT_FILE "/boot_out.txt"
 
+// TODO(tannewt): Make this 6k+ for any non-express M4 boards because they cache sectors on the
+// stack.
+#define CIRCUITPY_DEFAULT_STACK_SIZE 2048
+
 #endif  // __INCLUDED_MPCONFIGPORT_H

ports/nrf/Makefile

@@ -119,6 +119,7 @@ SRC_C += \
 	lib/utils/sys_stdio_mphal.c \
 	lib/libc/string0.c \
 	lib/mp-readline/readline.c \
+	supervisor/shared/memory.c
 
 DRIVERS_SRC_C += $(addprefix modules/,\
 	ubluepy/modubluepy.c \

ports/nrf/mpconfigport.h

@@ -242,5 +242,6 @@ void run_background_tasks(void);
 #define MICROPY_VM_HOOK_RETURN  run_background_tasks();
 
 //#define CIRCUITPY_BOOT_OUTPUT_FILE "/boot_out.txt"
+#define CIRCUITPY_DEFAULT_STACK_SIZE 2048
 
 #endif

shared-bindings/supervisor/__init__.c

@@ -23,17 +23,17 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
- #include "py/obj.h"
- #include "py/runtime.h"
- #include "py/reload.h"
+#include "py/obj.h"
+#include "py/runtime.h"
+#include "py/reload.h"
 
- #include "lib/utils/interrupt_char.h"
- #include "supervisor/shared/autoreload.h"
+#include "lib/utils/interrupt_char.h"
+#include "supervisor/shared/autoreload.h"
+#include "supervisor/shared/rgb_led_status.h"
+#include "supervisor/shared/stack.h"
 
- #include "supervisor/shared/rgb_led_status.h"
- 
- #include "shared-bindings/supervisor/__init__.h"
- #include "shared-bindings/supervisor/Runtime.h"
+#include "shared-bindings/supervisor/__init__.h"
+#include "shared-bindings/supervisor/Runtime.h"
 
 //| :mod:`supervisor` --- Supervisor settings
 //| =================================================
@@ -107,6 +107,21 @@ STATIC mp_obj_t supervisor_reload(void) {
 }
 MP_DEFINE_CONST_FUN_OBJ_0(supervisor_reload_obj, supervisor_reload);
 
+//| .. method:: set_next_stack_limit(size)
+//|
+//|   Set the size of the stack for the next vm run. If its too large, the default will be used.
+//|
+STATIC mp_obj_t supervisor_set_next_stack_limit(mp_obj_t size_obj) {
+    mp_int_t size = mp_obj_get_int(size_obj);
+
+    if (size < 256) {
+        mp_raise_ValueError("Stack size must be at least 256");
+    }
+    set_next_stack_size(size);
+
+    return mp_const_none;
+}
+MP_DEFINE_CONST_FUN_OBJ_1(supervisor_set_next_stack_limit_obj, supervisor_set_next_stack_limit);
 
 STATIC const mp_rom_map_elem_t supervisor_module_globals_table[] = {
     { MP_ROM_QSTR(MP_QSTR___name__), MP_ROM_QSTR(MP_QSTR_supervisor) },
@@ -115,6 +130,8 @@ STATIC const mp_rom_map_elem_t supervisor_module_globals_table[] = {
     { MP_OBJ_NEW_QSTR(MP_QSTR_set_rgb_status_brightness),  MP_ROM_PTR(&supervisor_set_rgb_status_brightness_obj) },
     { MP_ROM_QSTR(MP_QSTR_runtime),  MP_ROM_PTR(&common_hal_supervisor_runtime_obj) },
     { MP_ROM_QSTR(MP_QSTR_reload),  MP_ROM_PTR(&supervisor_reload_obj) },
+    { MP_ROM_QSTR(MP_QSTR_set_next_stack_limit),  MP_ROM_PTR(&supervisor_set_next_stack_limit_obj) },
+
 };
 
 STATIC MP_DEFINE_CONST_DICT(supervisor_module_globals, supervisor_module_globals_table);